fix yaml float - string conversion

test against python 3.10

.github/workflows/github-actions.yml

@@ -0,0 +1,143 @@
+name: MongoengineCI
+on:
+  # All PR
+  pull_request:
+  # master branch merge
+  push:
+    branches:
+      - master
+  # release tags
+  create:
+    tags:
+      - 'v[0-9]+\.[0-9]+\.[0-9]+*'
+env:
+  MONGODB_3_6: 3.6.14
+  MONGODB_4_0: 4.0.23
+  MONGODB_4_2: 4.2
+  MONGODB_4_4: 4.4
+
+  PYMONGO_3_4: 3.4
+  PYMONGO_3_6: 3.6
+  PYMONGO_3_9: 3.9
+  PYMONGO_3_11: 3.11
+
+  MAIN_PYTHON_VERSION: 3.7
+
+jobs:
+  linting:
+    # Run pre-commit (https://pre-commit.com/)
+    # which runs pre-configured linter & autoformatter
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - run: bash .github/workflows/install_ci_python_dep.sh
+    - run: pre-commit run -a
+
+  test:
+    # Test suite run against recent python versions
+    # and against a few combination of MongoDB and pymongo
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
+        MONGODB: [$MONGODB_4_0]
+        PYMONGO: [$PYMONGO_3_11]
+        include:
+          - python-version: 3.7
+            MONGODB: $MONGODB_3_6
+            PYMONGO: $PYMONGO_3_9
+          - python-version: 3.7
+            MONGODB: $MONGODB_4_2
+            PYMONGO: $PYMONGO_3_6
+          - python-version: 3.7
+            MONGODB: $MONGODB_4_4
+            PYMONGO: $PYMONGO_3_11
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: install mongo and ci dependencies
+      run: |
+        bash .github/workflows/install_mongo.sh ${{ matrix.MONGODB }}
+        bash .github/workflows/install_ci_python_dep.sh
+        bash .github/workflows/start_mongo.sh ${{ matrix.MONGODB }}
+    - name: tox dry-run (to pre-install venv)
+      run: tox -e $(echo py${{ matrix.python-version }}-mg${{ matrix.PYMONGO }} | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
+    - name: Run test suite
+      run: tox -e $(echo py${{ matrix.python-version }}-mg${{ matrix.PYMONGO }} | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
+    - name: Send coverage to Coveralls
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COVERALLS_SERVICE_NAME: github
+      if: ${{ matrix.python-version == env.MAIN_PYTHON_VERSION }}
+      run: coveralls
+
+  build_doc_dryrun:
+    # ensures that readthedocs can be built continuously
+    # to avoid that it breaks when new releases are being created
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: install python dep
+      run: |
+        pip install -e .
+        pip install -r docs/requirements.txt
+    - name: build doc
+      run: |
+        cd docs
+        make html-readthedocs
+
+  build-n-publish-dummy:
+    runs-on: ubuntu-latest
+    needs: [linting, test, build_doc_dryrun]
+    if: github.event_name != 'pull_request'
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: build dummy wheel for test-pypi
+      run: |
+        pip install wheel
+        python setup.py egg_info -b ".dev`date '+%Y%m%d%H%M%S'`" build sdist bdist_wheel
+#    - name: publish test-pypi
+#      # Although working and recommended, test-pypi has a limit
+#      # in the size of projects so it's better to avoid publishing
+#      # until there is a way to garbage collect these dummy releases
+#      uses: pypa/gh-action-pypi-publish@master
+#      with:
+#        password: ${{ secrets.test_pypi_token }}
+#        repository_url: https://test.pypi.org/legacy/
+
+  build-n-publish:
+    runs-on: ubuntu-latest
+    needs: [linting, test, build_doc_dryrun, build-n-publish-dummy]
+    if: github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    # todo separate build from publish
+    # https://stackoverflow.com/questions/59349905/which-properties-does-github-event-in-a-github-workflow-have
+    - name: build dummy wheel for test-pypi
+      run: |
+        pip install wheel
+        python setup.py sdist bdist_wheel
+    - name: publish pypi
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        password: ${{ secrets.pypi_token }}

.github/workflows/install_ci_python_dep.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+pip install --upgrade pip
+pip install coveralls
+pip install pre-commit
+pip install tox

.github/workflows/install_mongo.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+MONGODB=$1
+
+# Mongo > 4.0 follows different name convention for download links
+mongo_build=mongodb-linux-x86_64-${MONGODB}
+
+if [[ "$MONGODB" == *"4.2"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+elif [[ "$MONGODB" == *"4.4"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+fi
+
+wget http://fastdl.mongodb.org/linux/$mongo_build.tgz
+tar xzf $mongo_build.tgz
+
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+$mongodb_dir/bin/mongod --version

.github/workflows/start_mongo.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+MONGODB=$1
+
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+
+mkdir $mongodb_dir/data
+$mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork
+mongo --eval 'db.version();'    # Make sure mongo is awake

.landscape.yml

@@ -1,17 +0,0 @@
-pylint:
-    disable:
-        # We use this a lot (e.g. via document._meta)
-        - protected-access
-
-    options:
-        additional-builtins:
-            # add long as valid built-ins.
-            - long
-
-pyflakes:
-    disable:
-        # undefined variables are already covered by pylint (and exclude long)
-        - F821
-
-ignore-paths:
-    - benchmark.py

.pre-commit-config.yaml

@@ -1,12 +1,26 @@
 fail_fast: false
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
   - repo: https://github.com/ambv/black
-      rev: 19.10b0
+    rev: 21.5b2
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
-      rev: 3.8.0a2
+    rev: 3.9.2
     hooks:
       - id: flake8
-          additional_dependencies:
+  - repo: https://github.com/asottile/pyupgrade
-            - flake8-import-order
+    rev: v2.19.1
+    hooks:
+      - id: pyupgrade
+        args: [--py36-plus]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.8.0
+    hooks:
+      - id: isort

.readthedocs.yml

@@ -0,0 +1,20 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt
+    # docs/conf.py is importing mongoengine
+    # so mongoengine needs to be installed as well
+    - method: setuptools
+      path: .

.travis.yml

@@ -1,107 +0,0 @@
-# For full coverage, we'd have to test all supported Python, MongoDB, and
-# PyMongo combinations. However, that would result in an overly long build
-# with a very large number of jobs, hence we only test a subset of all the
-# combinations.
-# * Python3.7, MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-# Other combinations are tested. See below for the details or check the travis jobs
-
-# We should periodically check MongoDB Server versions supported by MongoDB
-# Inc., add newly released versions to the test matrix, and remove versions
-# which have reached their End of Life. See:
-# 1. https://www.mongodb.com/support-policy.
-# 2. https://docs.mongodb.com/ecosystem/drivers/driver-compatibility-reference/#python-driver-compatibility
-#
-# Reminder: Update README.rst if you change MongoDB versions we test.
-
-language: python
-dist: xenial
-python:
-- 3.5
-- 3.6
-- 3.7
-- 3.8
-- pypy3
-
-env:
-  global:
-    - MONGODB_3_4=3.4.17
-    - MONGODB_3_6=3.6.12
-    - MONGODB_4_0=4.0.13
-
-    - PYMONGO_3_4=3.4
-    - PYMONGO_3_6=3.6
-    - PYMONGO_3_9=3.9
-    - PYMONGO_3_10=3.10
-
-    - MAIN_PYTHON_VERSION=3.7
-  matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
-
-matrix:
-  # Finish the build as soon as one job fails
-  fast_finish: true
-
-  include:
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10}
-  - python: 3.8
-    env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_10}
-
-install:
-  # Install Mongo
-  - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB}.tgz
-  - tar xzf mongodb-linux-x86_64-${MONGODB}.tgz
-  - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --version
-  # Install Python dependencies.
-  - pip install --upgrade pip
-  - pip install coveralls
-  - pip install pre-commit
-  - pip install tox
-  # tox dryrun to setup the tox venv (we run a mock test).
-  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
-
-before_script:
-  - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data
-  - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
-  # Run pre-commit hooks (black, flake8, etc) on entire codebase
-  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
-  - mongo --eval 'db.version();'    # Make sure mongo is awake
-
-script:
-  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
-
-after_success:
-- - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi
-
-notifications:
-  irc: irc.freenode.org#mongoengine
-
-# Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
-branches:
-  only:
-  - master
-  - /^v.*$/
-
-# Whenever a new release is created via GitHub, publish it on PyPI.
-deploy:
-  provider: pypi
-  user: the_drow
-  password:
-    secure: QMyatmWBnC6ZN3XLW2+fTBDU4LQcp1m/LjR2/0uamyeUzWKdlOoh/Wx5elOgLwt/8N9ppdPeG83ose1jOz69l5G0MUMjv8n/RIcMFSpCT59tGYqn3kh55b0cIZXFT9ar+5cxlif6a5rS72IHm5li7QQyxexJIII6Uxp0kpvUmek=
-
-  # Create a source distribution and a pure python wheel for faster installs.
-  distributions: "sdist bdist_wheel"
-
-  # Only deploy on tagged commits (aka GitHub releases) and only for the parent
-  # repo's builds running Python v3.7 along with PyMongo v3.x and MongoDB v3.4.
-  # We run Travis against many different Python, PyMongo, and MongoDB versions
-  # and we don't want the deploy to occur multiple times).
-  on:
-    tags: true
-    repo: MongoEngine/mongoengine
-    condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4})
-    python: 3.7

AUTHORS

@@ -257,3 +257,9 @@ that much better:
  * Matthew Simpson (https://github.com/mcsimps2)
  * Leonardo Domingues (https://github.com/leodmgs)
  * Agustin Barto (https://github.com/abarto)
+ * Stankiewicz Mateusz (https://github.com/mas15)
+ * Felix Schultheiß (https://github.com/felix-smashdocs)
+ * Jan Stein (https://github.com/janste63)
+ * Timothé Perez (https://github.com/AchilleAsh)
+ * oleksandr-l5 (https://github.com/oleksandr-l5)
+ * Ido Shraga (https://github.com/idoshr)

CONTRIBUTING.rst

@@ -35,8 +35,8 @@ Travis runs the tests against the main Python 3.x versions.
 Style Guide
 -----------
 
-MongoEngine's codebase is formatted with `black <https://github.com/python/black>`_, other tools like
+MongoEngine's codebase is auto-formatted with `black <https://github.com/python/black>`_, imports are ordered with `isort <https://pycqa.github.io/isort/>`_
-flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
+and other tools like flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
 
 To install all development tools, simply run the following commands:
 
@@ -58,6 +58,10 @@ To enable ``pre-commit`` simply run:
 See the ``.pre-commit-config.yaml`` configuration file for more information
 on how it works.
 
+pre-commit will now run upon every commit and will reject anything that doesn't comply.
+
+You can also run all the checks with ``pre-commit run -a``, this is what is used in the CI.
+
 Testing
 -------
 

README.rst

@@ -12,9 +12,8 @@ MongoEngine
 .. image:: https://coveralls.io/repos/github/MongoEngine/mongoengine/badge.svg?branch=master
   :target: https://coveralls.io/github/MongoEngine/mongoengine?branch=master
 
-.. image:: https://landscape.io/github/MongoEngine/mongoengine/master/landscape.svg?style=flat
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
-  :target: https://landscape.io/github/MongoEngine/mongoengine/master
+  :target: https://github.com/ambv/black
-  :alt: Code Health
 
 About
 =====

benchmarks/test_basic_doc_ops.py

@@ -45,7 +45,7 @@ def test_basic():
 
     print(
         "Doc setattr: %.3fus"
-        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)
+        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)  # noqa B010
     )
 
     print("Doc to mongo: %.3fus" % (timeit(b.to_mongo, 1000) * 10 ** 6))

benchmarks/test_inserts.py

@@ -31,7 +31,7 @@ myNoddys = noddy.find()
     print("-" * 100)
     print("PyMongo: Creating 10000 dictionaries.")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 from pymongo import MongoClient, WriteConcern
@@ -54,7 +54,7 @@ myNoddys = noddy.find()
     print("-" * 100)
     print('PyMongo: Creating 10000 dictionaries (write_concern={"w": 0}).')
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     setup = """
 from pymongo import MongoClient
@@ -84,7 +84,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print("MongoEngine: Creating 10000 dictionaries.")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -102,7 +102,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print("MongoEngine: Creating 10000 dictionaries (using a single field assignment).")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -118,7 +118,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print('MongoEngine: Creating 10000 dictionaries (write_concern={"w": 0}).')
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -136,7 +136,7 @@ myNoddys = Noddy.objects()
         'MongoEngine: Creating 10000 dictionaries (write_concern={"w": 0}, validate=False).'
     )
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -154,7 +154,7 @@ myNoddys = Noddy.objects()
         'MongoEngine: Creating 10000 dictionaries (force_insert=True, write_concern={"w": 0}, validate=False).'
     )
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
 
 if __name__ == "__main__":

docs/Makefile

@@ -33,8 +33,14 @@ clean:
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
+	@echo "Build finished. Check $(BUILDDIR)/html/index.html"
+
+html-readthedocs:
+	$(SPHINXBUILD) -T -E -b readthedocs $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
+
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo

docs/apireference.rst

@@ -75,6 +75,7 @@ Fields
 .. autoclass:: mongoengine.fields.StringField
 .. autoclass:: mongoengine.fields.URLField
 .. autoclass:: mongoengine.fields.EmailField
+.. autoclass:: mongoengine.fields.EnumField
 .. autoclass:: mongoengine.fields.IntField
 .. autoclass:: mongoengine.fields.LongField
 .. autoclass:: mongoengine.fields.FloatField

docs/changelog.rst

@@ -1,4 +1,5 @@
 
+
 =========
 Changelog
 =========
@@ -6,6 +7,49 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- EnumField improvements: now `choices` limits the values of an enum to allow
+- Fix deepcopy of EmbeddedDocument #2202
+- Fix error when using precision=0 with DecimalField #2535
+- Add support for regex and whole word text search query #2568
+
+Changes in 0.23.1
+===========
+- Bug fix: ignore LazyReferenceFields when clearing _changed_fields #2484
+- Improve connection doc #2481
+
+Changes in 0.23.0
+=================
+- Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471
+- Add MongoDB 4.2 and 4.4 to CI
+- Add support for allowDiskUse on querysets #2468
+
+Changes in 0.22.1
+=================
+- Declare that Py3.5 is not supported in package metadata #2449
+- Moved CI from Travis to Github-Actions
+
+Changes in 0.22.0
+=================
+- Fix LazyReferenceField dereferencing in embedded documents #2426
+- Fix regarding the recent use of Cursor.__spec in .count() that was interfering with mongomock #2425
+- Drop support for Python 3.5 by introducing f-strings in the codebase
+
+Changes in 0.21.0
+=================
+- Bug fix in DynamicDocument which is not parsing known fields in constructor like Document do #2412
+- When using pymongo >= 3.7, make use of Collection.count_documents instead of Collection.count
+    and Cursor.count that got deprecated in pymongo >= 3.7.
+    This should have a negative impact on performance of count see Issue #2219
+- Fix a bug that made the queryset drop the read_preference after clone().
+- Remove Py3.5 from CI as it reached EOL and add Python 3.9
+- Fix some issues related with db_field/field conflict in constructor #2414
+- BREAKING CHANGE: Fix the behavior of Doc.objects.limit(0) which should return all documents (similar to mongodb) #2311
+- Bug fix in ListField when updating the first item, it was saving the whole list, instead of
+    just replacing the first item (as usually done when updating 1 item of the list) #2392
+- Add EnumField: ``mongoengine.fields.EnumField``
+- Refactoring - Remove useless code related to Document.__only_fields and Queryset.only_fields
+- Fix query transformation regarding special operators #2365
+- Bug Fix: Document.save() fails when shard_key is not _id #2154
 
 Changes in 0.20.0
 =================
@@ -28,7 +72,7 @@ Changes in 0.20.0
 
 Changes in 0.19.1
 =================
-- Requires Pillow < 7.0.0 as it dropped Python2 support
+- Tests require Pillow < 7.0.0 as it dropped Python2 support
 - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of
     pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079
 
@@ -456,9 +500,6 @@ Changes in 0.8.3
 - Document.select_related() now respects ``db_alias`` (#377)
 - Reload uses shard_key if applicable (#384)
 - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396)
-
-  **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3
-
 - Fixed pickling dynamic documents ``_dynamic_fields`` (#387)
 - Fixed ListField setslice and delslice dirty tracking (#390)
 - Added Django 1.5 PY3 support (#392)

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # MongoEngine documentation build configuration file, created by
 # sphinx-quickstart on Sun Nov 22 18:14:13 2009.
@@ -27,7 +26,7 @@ sys.path.insert(0, os.path.abspath(".."))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ["sphinx.ext.autodoc", "sphinx.ext.todo"]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.todo", "readthedocs_ext.readthedocs"]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
@@ -42,8 +41,8 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = u"MongoEngine"
+project = "MongoEngine"
-copyright = u"2009, MongoEngine Authors"
+copyright = "2009, MongoEngine Authors"  # noqa: A001
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/guide/connecting.rst

@@ -5,7 +5,7 @@ Connecting to MongoDB
 =====================
 
 Connections in MongoEngine are registered globally and are identified with aliases.
-If no `alias` is provided during the connection, it will use "default" as alias.
+If no ``alias`` is provided during the connection, it will use "default" as alias.
 
 To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect`
 function. The first argument is the name of the database to connect to::
@@ -14,25 +14,66 @@ function. The first argument is the name of the database to connect to::
     connect('project1')
 
 By default, MongoEngine assumes that the :program:`mongod` instance is running
-on **localhost** on port **27017**. If MongoDB is running elsewhere, you should
+on **localhost** on port **27017**.
-provide the :attr:`host` and :attr:`port` arguments to
-:func:`~mongoengine.connect`::
 
-    connect('project1', host='192.168.1.35', port=12345)
+If MongoDB is running elsewhere, you need to provide details on how to connect. There are two ways of
+doing this. Using a connection string in URI format (**this is the preferred method**) or individual attributes
+provided as keyword arguments.
+
+Connect with URI string
+=======================
+
+When using a connection string in URI format you should specify the connection details
+as the :attr:`host` to :func:`~mongoengine.connect`. In a web application context for instance, the URI
+is typically read from the config file::
+
+        connect(host="mongodb://127.0.0.1:27017/my_db")
+
+If the database requires authentication, you can specify it in the
+URI. As each database can have its own users configured, you need to tell MongoDB
+where to look for the user you are working with, that's what the ``?authSource=admin`` bit
+of the MongoDB connection string is for::
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against the 'admin' database (by default as authSource isn't provided)
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db")
+
+    # Equivalent to previous connection but explicitly states that
+    # it should use admin as the authentication source database
+    connect(host="mongodb://my_user:my_password@hostname:port/my_db?authSource=admin")
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against that same database
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=my_db")
+
+The URI string can also be used to configure advanced parameters like ssl, replicaSet, etc. For more
+information or example about URI string, you can refer to the `official doc <https://docs.mongodb.com/manual/reference/connection-string/>`_::
+
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=admin&ssl=true&replicaSet=globaldb")
+
+.. note:: URI containing SRV records (e.g "mongodb+srv://server.example.com/") can be used as well
+
+Connect with keyword attributes
+===============================
+
+The second option for specifying the connection details is to provide the information as keyword
+attributes to :func:`~mongoengine.connect`::
+
+    connect('my_db', host='127.0.0.1', port=27017)
 
 If the database requires authentication, :attr:`username`, :attr:`password`
 and :attr:`authentication_source` arguments should be provided::
 
-    connect('project1', username='webapp', password='pwd123', authentication_source='admin')
+    connect('my_db', username='my_user', password='my_password', authentication_source='admin')
 
-URI style connections are also supported -- just supply the URI as
+The set of attributes that :func:`~mongoengine.connect` recognizes includes but is not limited to:
-the :attr:`host` to
+:attr:`host`, :attr:`port`, :attr:`read_preference`, :attr:`username`, :attr:`password`, :attr:`authentication_source`, :attr:`authentication_mechanism`,
-:func:`~mongoengine.connect`::
+:attr:`replicaset`, :attr:`tls`, etc. Most of the parameters accepted by `pymongo.MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_
-
+can be used with :func:`~mongoengine.connect` and will simply be forwarded when instantiating the `pymongo.MongoClient`.
-    connect('project1', host='mongodb://localhost/database_name')
 
 .. note:: Database, username and password from URI string overrides
-    corresponding parameters in :func:`~mongoengine.connect`: ::
+    corresponding parameters in :func:`~mongoengine.connect`, this should
+    obviously be avoided: ::
 
         connect(
             db='test',
@@ -41,28 +82,19 @@ the :attr:`host` to
             host='mongodb://admin:qwerty@localhost/production'
         )
 
-    will establish connection to ``production`` database using
+    will establish connection to ``production`` database using ``admin`` username and ``qwerty`` password.
-    ``admin`` username and ``qwerty`` password.
 
 .. note:: Calling :func:`~mongoengine.connect` without argument will establish
     a connection to the "test" database by default
 
-Replica Sets
+Read Preferences
-============
+================
 
-MongoEngine supports connecting to replica sets::
+As stated above, Read preferences are supported through the connection but also via individual
-
-    from mongoengine import connect
-
-    # Regular connect
-    connect('dbname', replicaset='rs-name')
-
-    # MongoDB URI-style connect
-    connect(host='mongodb://localhost/dbname?replicaSet=rs-name')
-
-Read preferences are supported through the connection or via individual
 queries by passing the read_preference ::
 
+    from pymongo import ReadPreference
+
     Bar.objects().read_preference(ReadPreference.PRIMARY)
     Bar.objects(read_preference=ReadPreference.PRIMARY)
 

docs/guide/defining-documents.rst

@@ -27,6 +27,8 @@ objects** as class attributes to the document class::
 As BSON (the binary format for storing data in mongodb) is order dependent,
 documents are serialized based on their field order.
 
+.. _dynamic-document-schemas:
+
 Dynamic document schemas
 ========================
 One of the benefits of MongoDB is dynamic schemas for a collection, whilst data
@@ -76,6 +78,7 @@ are as follows:
 * :class:`~mongoengine.fields.EmailField`
 * :class:`~mongoengine.fields.EmbeddedDocumentField`
 * :class:`~mongoengine.fields.EmbeddedDocumentListField`
+* :class:`~mongoengine.fields.EnumField`
 * :class:`~mongoengine.fields.FileField`
 * :class:`~mongoengine.fields.FloatField`
 * :class:`~mongoengine.fields.GenericEmbeddedDocumentField`
@@ -230,6 +233,9 @@ document class as the first argument::
     comment2 = Comment(content='Nice article!')
     page = Page(comments=[comment1, comment2])
 
+Embedded documents can also leverage the flexibility of :ref:`dynamic-document-schemas:`
+by inheriting :class:`~mongoengine.DynamicEmbeddedDocument`.
+
 Dictionary Fields
 -----------------
 Often, an embedded document may be used instead of a dictionary – generally
@@ -289,12 +295,12 @@ as the constructor's argument::
         content = StringField()
 
 
-.. _one-to-many-with-listfields:
+.. _many-to-many-with-listfields:
 
-One to Many with ListFields
+Many to Many with ListFields
 '''''''''''''''''''''''''''
 
-If you are implementing a one to many relationship via a list of references,
+If you are implementing a many to many relationship via a list of references,
 then the references are stored as DBRefs and to query you need to pass an
 instance of the object to the query::
 
@@ -335,7 +341,6 @@ supplying the :attr:`reverse_delete_rule` attributes on the
 :class:`ReferenceField` definition, like this::
 
     class ProfilePage(Document):
-        ...
         employee = ReferenceField('Employee', reverse_delete_rule=mongoengine.CASCADE)
 
 The declaration in this example means that when an :class:`Employee` object is
@@ -426,28 +431,15 @@ either a single field name, or a list or tuple of field names::
         first_name = StringField()
         last_name = StringField(unique_with='first_name')
 
-Skipping Document validation on save
-------------------------------------
-You can also skip the whole document validation process by setting
-``validate=False`` when calling the :meth:`~mongoengine.document.Document.save`
-method::
-
-    class Recipient(Document):
-        name = StringField()
-        email = EmailField()
-
-    recipient = Recipient(name='admin', email='root@localhost')
-    recipient.save()               # will raise a ValidationError while
-    recipient.save(validate=False) # won't
 
 Document collections
 ====================
 Document classes that inherit **directly** from :class:`~mongoengine.Document`
 will have their own **collection** in the database. The name of the collection
-is by default the name of the class, converted to lowercase (so in the example
+is by default the name of the class converted to snake_case (e.g if your Document class
-above, the collection would be called `page`). If you need to change the name
+is named `CompanyUser`, the corresponding collection would be `company_user`). If you need
-of the collection (e.g. to use MongoEngine with an existing database), then
+to change the name of the collection (e.g. to use MongoEngine with an existing database),
-create a class dictionary attribute called :attr:`meta` on your document, and
+then create a class dictionary attribute called :attr:`meta` on your document, and
 set :attr:`collection` to the name of the collection that you want your
 document class to use::
 
@@ -485,7 +477,7 @@ dictionary containing a full index definition.
 
 A direction may be specified on fields by prefixing the field name with a
 **+** (for ascending) or a **-** sign (for descending). Note that direction
-only matters on multi-field indexes. Text indexes may be specified by prefixing
+only matters on compound indexes. Text indexes may be specified by prefixing
 the field name with a **$**. Hashed indexes may be specified by prefixing
 the field name with a **#**::
 
@@ -496,14 +488,14 @@ the field name with a **#**::
         created = DateTimeField()
         meta = {
             'indexes': [
-                'title',
+                'title',   # single-field index
                 '$title',  # text index
                 '#title',  # hashed index
-                ('title', '-rating'),
+                ('title', '-rating'),  # compound index
-                ('category', '_cls'),
+                ('category', '_cls'),  # compound index
                 {
                     'fields': ['created'],
-                    'expireAfterSeconds': 3600
+                    'expireAfterSeconds': 3600  # ttl index
                 }
             ]
         }
@@ -636,8 +628,8 @@ point. To create a geospatial index you must prefix the field with the
             ],
         }
 
-Time To Live indexes
+Time To Live (TTL) indexes
---------------------
+--------------------------
 
 A special index type that allows you to automatically expire data from a
 collection after a given period. See the official

docs/guide/document-instances.rst

@@ -41,35 +41,6 @@ already exist, then any changes will be updated atomically.  For example::
 .. seealso::
     :ref:`guide-atomic-updates`
 
-Pre save data validation and cleaning
--------------------------------------
-MongoEngine allows you to create custom cleaning rules for your documents when
-calling :meth:`~mongoengine.Document.save`.  By providing a custom
-:meth:`~mongoengine.Document.clean` method you can do any pre validation / data
-cleaning.
-
-This might be useful if you want to ensure a default value based on other
-document values for example::
-
-    class Essay(Document):
-        status = StringField(choices=('Published', 'Draft'), required=True)
-        pub_date = DateTimeField()
-
-        def clean(self):
-            """Ensures that only published essays have a `pub_date` and
-            automatically sets `pub_date` if essay is published and `pub_date`
-            is not set"""
-            if self.status == 'Draft' and self.pub_date is not None:
-                msg = 'Draft entries should not have a publication date.'
-                raise ValidationError(msg)
-            # Set the pub_date for published items if not set.
-            if self.status == 'Published' and self.pub_date is None:
-                self.pub_date = datetime.now()
-
-.. note::
-    Cleaning is only called if validation is turned on and when calling
-    :meth:`~mongoengine.Document.save`.
-
 Cascading Saves
 ---------------
 If your document contains :class:`~mongoengine.fields.ReferenceField` or

docs/guide/gridfs.rst

@@ -2,8 +2,6 @@
 GridFS
 ======
 
-.. versionadded:: 0.4
-
 Writing
 -------
 

docs/guide/index.rst

@@ -10,8 +10,10 @@ User Guide
    defining-documents
    document-instances
    querying
+   validation
    gridfs
    signals
    text-indexes
+   migration
    logging-monitoring
    mongomock

docs/guide/migration.rst

@@ -0,0 +1,308 @@
+===================
+Documents migration
+===================
+
+The structure of your documents and their associated mongoengine schemas are likely
+to change over the lifetime of an application. This section provides guidance and
+recommendations on how to deal with migrations.
+
+Due to the very flexible nature of mongodb, migrations of models aren't trivial and
+for people that know about `alembic` for `sqlalchemy`, there is unfortunately no equivalent
+library that will manage the migration in an automatic fashion for mongoengine.
+
+Example 1: Addition of a field
+==============================
+
+Let's start by taking a simple example of a model change and review the different option you
+have to deal with the migration.
+
+Let's assume we start with the following schema and save an instance:
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField()
+
+    User(name="John Doe").save()
+
+    # print the objects as they exist in mongodb
+    print(User.objects().as_pymongo())    # [{u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John Doe'}]
+
+On the next version of your application, let's now assume that a new field `enabled` gets added to the
+existing ``User`` model with a `default=True`. Thus you simply update the ``User`` class to the following:
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField(required=True)
+        enabled = BooleanField(default=True)
+
+Without applying any migration, we now reload an object from the database into the ``User`` class
+and checks its `enabled` attribute:
+
+.. code-block:: python
+
+    assert User.objects.count() == 1
+    user = User.objects().first()
+    assert user.enabled is True
+    assert User.objects(enabled=True).count() == 0    # uh?
+    assert User.objects(enabled=False).count() == 0   # uh?
+
+    # this is consistent with what we have in the database
+    # in fact, 'enabled' does not exist
+    print(User.objects().as_pymongo().first())    # {u'_id': ObjectId('5d06b9c3d7c1f18db3e7c874'), u'name': u'John'}
+    assert User.objects(enabled=None).count() == 1
+
+As you can see, even if the document wasn't updated, mongoengine applies the default value seamlessly when it
+loads the pymongo dict into a ``User`` instance. At first sight it looks like you don't need to migrate the
+existing documents when adding new fields but this actually leads to inconsistencies when it comes to querying.
+
+In fact, when querying, mongoengine isn't trying to account for the default value of the new field and so
+if you don't actually migrate the existing documents, you are taking a risk that querying/updating
+will be missing relevant record.
+
+When adding fields/modifying default values, you can use any of the following to do the migration
+as a standalone script:
+
+.. code-block:: python
+
+    # Use mongoengine to set a default value for a given field
+    User.objects().update(enabled=True)
+    # or use pymongo
+    user_coll = User._get_collection()
+    user_coll.update_many({}, {'$set': {'enabled': True}})
+
+
+Example 2: Inheritance change
+=============================
+
+Let's consider the following example:
+
+.. code-block:: python
+
+    class Human(Document):
+        name = StringField()
+        meta = {"allow_inheritance": True}
+
+    class Jedi(Human):
+        dark_side = BooleanField()
+        light_saber_color = StringField()
+
+    Jedi(name="Darth Vader", dark_side=True, light_saber_color="red").save()
+    Jedi(name="Obi Wan Kenobi", dark_side=False, light_saber_color="blue").save()
+
+    assert Human.objects.count() == 2
+    assert Jedi.objects.count() == 2
+
+    # Let's check how these documents got stored in mongodb
+    print(Jedi.objects.as_pymongo())
+    # [
+    #   {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'},
+    #   {'_id': ObjectId('5fac4ac4f61d7fb06046e0fa'), '_cls': 'Human.Jedi', 'name': 'Obi Wan Kenobi', 'dark_side': False, 'light_saber_color': 'blue'}
+    # ]
+
+As you can observe, when you use inheritance, MongoEngine stores a field named '_cls' behind the scene to keep
+track of the Document class.
+
+Let's now take the scenario that you want to refactor the inheritance schema and:
+- Have the Jedi's with dark_side=True/False become GoodJedi's/DarkSith
+- get rid of the 'dark_side' field
+
+move to the following schemas:
+
+.. code-block:: python
+
+    # unchanged
+    class Human(Document):
+        name = StringField()
+        meta = {"allow_inheritance": True}
+
+    # attribute 'dark_side' removed
+    class GoodJedi(Human):
+        light_saber_color = StringField()
+
+    # new class
+    class BadSith(Human):
+        light_saber_color = StringField()
+
+MongoEngine doesn't know about the change or how to map them with the existing data
+so if you don't apply any migration, you will observe a strange behavior, as if the collection was suddenly
+empty.
+
+.. code-block:: python
+
+    # As a reminder, the documents that we inserted
+    # have the _cls field = 'Human.Jedi'
+
+    # Following has no match
+    # because the query that is used behind the scene is
+    # filtering on {'_cls': 'Human.GoodJedi'}
+    assert GoodJedi.objects().count() == 0
+
+    # Following has also no match
+    # because it is filtering on {'_cls': {'$in': ('Human', 'Human.GoodJedi', 'Human.BadSith')}}
+    # which has no match
+    assert Human.objects.count() == 0
+    assert Human.objects.first() is None
+
+    # If we bypass MongoEngine and make use of underlying driver (PyMongo)
+    # we can see that the documents are there
+    humans_coll = Human._get_collection()
+    assert humans_coll.count_documents({}) == 2
+    # print first document
+    print(humans_coll.find_one())
+    # {'_id': ObjectId('5fac4aaaf61d7fb06046e0f9'), '_cls': 'Human.Jedi', 'name': 'Darth Vader', 'dark_side': True, 'light_saber_color': 'red'}
+
+As you can see, first obvious problem is that we need to modify '_cls' values based on existing values of
+'dark_side' documents.
+
+.. code-block:: python
+
+    humans_coll = Human._get_collection()
+    old_class = 'Human.Jedi'
+    good_jedi_class = 'Human.GoodJedi'
+    bad_sith_class = 'Human.BadSith'
+    humans_coll.update_many({'_cls': old_class, 'dark_side': False}, {'$set': {'_cls': good_jedi_class}})
+    humans_coll.update_many({'_cls': old_class, 'dark_side': True}, {'$set': {'_cls': bad_sith_class}})
+
+Let's now check if querying improved in MongoEngine:
+
+.. code-block:: python
+
+    assert GoodJedi.objects().count() == 1  # Hoorah!
+    assert BadSith.objects().count() == 1   # Hoorah!
+    assert Human.objects.count() == 2       # Hoorah!
+
+    # let's now check that documents load correctly
+    jedi = GoodJedi.objects().first()
+    # raises FieldDoesNotExist: The fields "{'dark_side'}" do not exist on the document "Human.GoodJedi"
+
+In fact we only took care of renaming the _cls values but we havn't removed the 'dark_side' fields
+which does not exist anymore on the GoodJedi's and BadSith's models.
+Let's remove the field from the collections:
+
+.. code-block:: python
+
+    humans_coll = Human._get_collection()
+    humans_coll.update_many({}, {'$unset': {'dark_side': 1}})
+
+.. note:: We did this migration in 2 different steps for the sake of example but it could have been combined
+    with the migration of the _cls fields: ::
+
+        humans_coll.update_many(
+            {'_cls': old_class, 'dark_side': False},
+            {
+                '$set': {'_cls': good_jedi_class},
+                '$unset': {'dark_side': 1}
+            }
+        )
+
+
+And verify that the documents now load correctly:
+
+.. code-block:: python
+
+    jedi = GoodJedi.objects().first()
+    assert jedi.name == "Obi Wan Kenobi"
+
+    sith = BadSith.objects().first()
+    assert sith.name == "Darth Vader"
+
+
+An other way of dealing with this migration is to iterate over
+the documents and update/replace them one by one. This is way slower but
+it is often useful for complex migrations of Document models.
+
+.. code-block:: python
+
+    for doc in humans_coll.find():
+        if doc['_cls'] == 'Human.Jedi':
+            doc['_cls'] =  'Human.BadSith' if doc['dark_side'] else 'Human.GoodJedi'
+            doc.pop('dark_side')
+            humans_coll.replace_one({'_id': doc['_id']}, doc)
+
+.. warning:: Be aware of this `flaw <https://groups.google.com/g/mongodb-user/c/AFC1ia7MHzk>`_ if you modify documents while iterating
+
+Example 4: Index removal
+========================
+
+If you remove an index from your Document class, or remove an indexed Field from your Document class,
+you'll need to manually drop the corresponding index. MongoEngine will not do that for you.
+
+The way to deal with this case is to identify the name of the index to drop with `index_information()`, and then drop
+it with `drop_index()`
+
+Let's for instance assume that you start with the following Document class
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField(index=True)
+
+        meta = {"indexes": ["name"]}
+
+    User(name="John Doe").save()
+
+As soon as you start interacting with the Document collection (when `.save()` is called in this case),
+it would create the following indexes:
+
+.. code-block:: python
+
+    print(User._get_collection().index_information())
+    # {
+    #  '_id_': {'key': [('_id', 1)], 'v': 2},
+    #  'name_1': {'background': False, 'key': [('name', 1)], 'v': 2},
+    # }
+
+Thus: '_id' which is the default index and 'name_1' which is our custom index.
+If you would remove the 'name' field or its index, you would have to call:
+
+.. code-block:: python
+
+    User._get_collection().drop_index('name_1')
+
+.. note:: When adding new fields or new indexes, MongoEngine will take care of creating them
+    (unless `auto_create_index` is disabled) ::
+
+Recommendations
+===============
+
+- Write migration scripts whenever you do changes to the model schemas
+- Using :class:`~mongoengine.DynamicDocument` or ``meta = {"strict": False}`` may help to avoid some migrations or to have the 2 versions of your application to co-exist.
+- Write post-processing checks to verify that migrations script worked. See below
+
+Post-processing checks
+======================
+
+The following recipe can be used to sanity check a Document collection after you applied migration.
+It does not make any assumption on what was migrated, it will fetch 1000 objects randomly and
+run some quick checks on the documents to make sure the document looks OK. As it is, it will fail
+on the first occurrence of an error but this is something that can be adapted based on your needs.
+
+.. code-block:: python
+
+    def get_random_oids(collection, sample_size):
+        pipeline = [{"$project": {'_id': 1}}, {"$sample": {"size": sample_size}}]
+        return [s['_id'] for s in collection.aggregate(pipeline)]
+
+    def get_random_documents(DocCls, sample_size):
+        doc_collection = DocCls._get_collection()
+        random_oids = get_random_oids(doc_collection, sample_size)
+        return DocCls.objects(id__in=random_oids)
+
+    def check_documents(DocCls, sample_size):
+        for doc in get_random_documents(DocCls, sample_size):
+            # general validation (types and values)
+            doc.validate()
+
+            # load all subfields,
+            # this may trigger additional queries if you have ReferenceFields
+            # so it may be slow
+            for field in doc._fields:
+                try:
+                    getattr(doc, field)
+                except Exception:
+                    LOG.warning(f"Could not load field {field} in Document {doc.id}")
+                    raise
+
+    check_documents(Human, sample_size=1000)

docs/guide/querying.rst

@@ -86,6 +86,10 @@ expressions:
 * ``istartswith`` -- string field starts with value (case insensitive)
 * ``endswith`` -- string field ends with value
 * ``iendswith`` -- string field ends with value (case insensitive)
+* ``wholeword`` -- string field contains whole word
+* ``iwholeword`` -- string field contains whole word (case insensitive)
+* ``regex`` -- string field match by regex
+* ``iregex`` -- string field match by regex (case insensitive)
 * ``match``  -- performs an $elemMatch so you can match an entire document within an array
 
 
@@ -239,7 +243,7 @@ Limiting and skipping results
 Just as with traditional ORMs, you may limit the number of results returned or
 skip a number or results in you query.
 :meth:`~mongoengine.queryset.QuerySet.limit` and
-:meth:`~mongoengine.queryset.QuerySet.skip` and methods are available on
+:meth:`~mongoengine.queryset.QuerySet.skip` methods are available on
 :class:`~mongoengine.queryset.QuerySet` objects, but the `array-slicing` syntax
 is preferred for achieving this::
 
@@ -543,7 +547,10 @@ Documents may be updated atomically by using the
 There are several different "modifiers" that you may use with these methods:
 
 * ``set`` -- set a particular value
+* ``set_on_insert`` -- set only if this is new document  `need to add upsert=True`_
 * ``unset`` -- delete a particular value (since MongoDB v1.3)
+* ``max`` -- update only if value is bigger
+* ``min`` -- update only if value is smaller
 * ``inc`` -- increment a value by a given amount
 * ``dec`` -- decrement a value by a given amount
 * ``push`` -- append a value to a list
@@ -552,6 +559,7 @@ There are several different "modifiers" that you may use with these methods:
 * ``pull`` -- remove a value from a list
 * ``pull_all`` -- remove several values from a list
 * ``add_to_set`` -- add value to a list only if its not in the list already
+* ``rename`` -- rename the key name
 
 .. _depending on the value: http://docs.mongodb.org/manual/reference/operator/update/pop/
 
@@ -609,7 +617,7 @@ to push values with index::
 .. note::
     Currently only top level lists are handled, future versions of mongodb /
     pymongo plan to support nested positional operators.  See `The $ positional
-    operator <http://www.mongodb.org/display/DOCS/Updating#Updating-The%24positionaloperator>`_.
+    operator <https://docs.mongodb.com/manual/tutorial/update-documents/#Updating-The%24positionaloperator>`_.
 
 Server-side javascript execution
 ================================

docs/guide/validation.rst

@@ -0,0 +1,122 @@
+====================
+Document Validation
+====================
+
+By design, MongoEngine strictly validates the documents right before they are inserted in MongoDB
+and makes sure they are consistent with the fields defined in your models.
+
+MongoEngine makes the assumption that the documents that exists in the DB are compliant with the schema.
+This means that Mongoengine will not validate a document when an object is loaded from the DB into an instance
+of your model but this operation may fail under some circumstances (e.g. if there is a field in
+the document fetched from the database that is not defined in your model).
+
+
+Built-in validation
+===================
+
+Mongoengine provides different fields that encapsulate the corresponding validation
+out of the box. Validation runs when calling `.validate()` or `.save()`
+
+.. code-block:: python
+
+    from mongoengine import Document, EmailField
+
+    class User(Document):
+        email = EmailField()
+        age = IntField(min_value=0, max_value=99)
+
+    user = User(email='invalid@', age=24)
+    user.validate()     # raises ValidationError (Invalid email address: ['email'])
+    user.save()         # raises ValidationError (Invalid email address: ['email'])
+
+    user2 = User(email='john.doe@garbage.com', age=1000)
+    user2.save()        # raises ValidationError (Integer value is too large: ['age'])
+
+Custom validation
+=================
+
+The following feature can be used to customize the validation:
+
+* Field `validation` parameter
+
+.. code-block:: python
+
+    def not_john_doe(name):
+        if name == 'John Doe':
+            raise ValidationError("John Doe is not a valid name")
+
+    class Person(Document):
+        full_name = StringField(validation=not_john_doe)
+
+    Person(full_name='Billy Doe').save()
+    Person(full_name='John Doe').save()  # raises ValidationError (John Doe is not a valid name)
+
+
+* Document `clean` method
+
+This method is called as part of :meth:`~mongoengine.document.Document.save` and should be used to provide
+custom model validation and/or to modify some of the field values prior to validation.
+For instance, you could use it to automatically provide a value for a field, or to do validation
+that requires access to more than a single field.
+
+.. code-block:: python
+
+    class Essay(Document):
+        status = StringField(choices=('Published', 'Draft'), required=True)
+        pub_date = DateTimeField()
+
+        def clean(self):
+            # Validate that only published essays have a `pub_date`
+            if self.status == 'Draft' and self.pub_date is not None:
+                raise ValidationError('Draft entries should not have a publication date.')
+            # Set the pub_date for published items if not set.
+            if self.status == 'Published' and self.pub_date is None:
+                self.pub_date = datetime.now()
+
+.. note::
+    Cleaning is only called if validation is turned on and when calling
+    :meth:`~mongoengine.Document.save`.
+
+* Adding custom Field classes
+
+We recommend as much as possible to use fields provided by MongoEngine. However, it is also possible
+to subclass a Field and encapsulate some validation by overriding the `validate` method
+
+.. code-block:: python
+
+    class AgeField(IntField):
+
+        def validate(self, value):
+            super(AgeField, self).validate(value)     # let IntField.validate run first
+            if value == 60:
+                self.error('60 is not allowed')
+
+    class Person(Document):
+        age = AgeField(min_value=0, max_value=99)
+
+    Person(age=20).save()   # passes
+    Person(age=1000).save() # raises ValidationError (Integer value is too large: ['age'])
+    Person(age=60).save()   # raises ValidationError (Person:None) (60 is not allowed: ['age'])
+
+
+.. note::
+
+   When overriding `validate`, use `self.error("your-custom-error")` instead of raising ValidationError explicitly,
+   it will provide a better context with the error message
+
+Skipping validation
+====================
+
+Although discouraged as it allows to violate fields constraints, if for some reason you need to disable
+the validation and cleaning of a document when you call :meth:`~mongoengine.document.Document.save`, you can use `.save(validate=False)`.
+
+.. code-block:: python
+
+    class Person(Document):
+        age = IntField(max_value=100)
+
+    Person(age=1000).save()    # raises ValidationError (Integer value is too large)
+
+    Person(age=1000).save(validate=False)
+    person = Person.objects.first()
+    assert person.age == 1000

docs/requirements.txt

@@ -0,0 +1,3 @@
+Sphinx==3.3.0
+sphinx-rtd-theme==0.5.0
+readthedocs-sphinx-ext==2.1.1

mongoengine/__init__.py

@@ -1,22 +1,23 @@
 # Import submodules so that we can expose their __all__
-from mongoengine import connection
+from mongoengine import (
-from mongoengine import document
+    connection,
-from mongoengine import errors
+    document,
-from mongoengine import fields
+    errors,
-from mongoengine import queryset
+    fields,
-from mongoengine import signals
+    queryset,
+    signals,
+)
 
 # Import everything from each submodule so that it can be accessed via
 # mongoengine, e.g. instead of `from mongoengine.connection import connect`,
 # users can simply use `from mongoengine import connect`, or even
 # `from mongoengine import *` and then `connect('testdb')`.
-from mongoengine.connection import *
+from mongoengine.connection import *  # noqa: F401
-from mongoengine.document import *
+from mongoengine.document import *  # noqa: F401
-from mongoengine.errors import *
+from mongoengine.errors import *  # noqa: F401
-from mongoengine.fields import *
+from mongoengine.fields import *  # noqa: F401
-from mongoengine.queryset import *
+from mongoengine.queryset import *  # noqa: F401
-from mongoengine.signals import *
+from mongoengine.signals import *  # noqa: F401
-
 
 __all__ = (
     list(document.__all__)
@@ -28,7 +29,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 20, 0)
+VERSION = (0, 23, 1)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -67,11 +67,11 @@ class BaseDict(dict):
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -97,7 +97,7 @@ class BaseDict(dict):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
             if key:
-                self._instance._mark_as_changed("{}.{}".format(self._name, key))
+                self._instance._mark_as_changed(f"{self._name}.{key}")
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -133,12 +133,12 @@ class BaseList(list):
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             # Replace dict by BaseDict
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
             # Replace list by BaseList
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -179,10 +179,8 @@ class BaseList(list):
 
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
-            if key:
+            if key is not None:
-                self._instance._mark_as_changed(
+                self._instance._mark_as_changed(f"{self._name}.{key % len(self)}")
-                    "{}.{}".format(self._name, key % len(self))
-                )
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -215,7 +213,7 @@ class EmbeddedDocumentList(BaseList):
         Filters the list by only including embedded documents with the
         given keyword arguments.
 
-        This method only supports simple comparison (e.g: .filter(name='John Doe'))
+        This method only supports simple comparison (e.g. .filter(name='John Doe'))
         and does not support operators like __gte, __lte, __icontains like queryset.filter does
 
         :param kwargs: The keyword arguments corresponding to the fields to
@@ -429,7 +427,7 @@ class StrictDict:
 
                 def __repr__(self):
                     return "{%s}" % ", ".join(
-                        '"{!s}": {!r}'.format(k, v) for k, v in self.items()
+                        f'"{k!s}": {v!r}' for k, v in self.items()
                     )
 
             cls._classes[allowed_keys] = SpecificStrictDict
@@ -472,4 +470,4 @@ class LazyReference(DBRef):
             raise AttributeError()
 
     def __repr__(self):
-        return "<LazyReference({}, {!r})>".format(self.document_type, self.pk)
+        return f"<LazyReference({self.document_type}, {self.pk!r})>"

mongoengine/base/document.py

@@ -1,10 +1,9 @@
 import copy
-
 import numbers
 from functools import partial
 
-from bson import DBRef, ObjectId, SON, json_util
 import pymongo
+from bson import SON, DBRef, ObjectId, json_util
 
 from mongoengine import signals
 from mongoengine.base.common import get_document
@@ -64,8 +63,6 @@ class BaseDocument:
             It may contain additional reserved keywords, e.g. "__auto_convert".
         :param __auto_convert: If True, supplied values will be converted
             to Python-type values via each field's `to_python` method.
-        :param __only_fields: A set of fields that have been loaded for
-            this document. Empty if all fields have been loaded.
         :param _created: Indicates whether this is a brand new document
             or whether it's already been persisted before. Defaults to true.
         """
@@ -80,8 +77,6 @@ class BaseDocument:
 
         __auto_convert = values.pop("__auto_convert", True)
 
-        __only_fields = set(values.pop("__only_fields", values))
-
         _created = values.pop("_created", True)
 
         signals.pre_init.send(self.__class__, document=self, values=values)
@@ -93,9 +88,7 @@ class BaseDocument:
                 list(self._fields.keys()) + ["id", "pk", "_cls", "_text_score"]
             )
             if _undefined_fields:
-                msg = ('The fields "{}" do not exist on the document "{}"').format(
+                msg = f'The fields "{_undefined_fields}" do not exist on the document "{self._class_name}"'
-                    _undefined_fields, self._class_name
-                )
                 raise FieldDoesNotExist(msg)
 
         if self.STRICT and not self._dynamic:
@@ -105,37 +98,32 @@ class BaseDocument:
 
         self._dynamic_fields = SON()
 
-        # Assign default values to the instance.
+        # Assign default values for fields
-        # We set default values only for fields loaded from DB. See
+        # not set in the constructor
-        # https://github.com/mongoengine/mongoengine/issues/399 for more info.
+        for field_name in self._fields:
-        for key, field in self._fields.items():
+            if field_name in values:
-            if self._db_field_map.get(key, key) in __only_fields:
                 continue
-            value = getattr(self, key, None)
+            value = getattr(self, field_name, None)
-            setattr(self, key, value)
+            setattr(self, field_name, value)
 
         if "_cls" not in values:
             self._cls = self._class_name
 
-        # Set passed values after initialisation
+        # Set actual values
-        if self._dynamic:
         dynamic_data = {}
-            for key, value in values.items():
-                if key in self._fields or key == "_id":
-                    setattr(self, key, value)
-                else:
-                    dynamic_data[key] = value
-        else:
         FileField = _import_class("FileField")
         for key, value in values.items():
-                key = self._reverse_db_field_map.get(key, key)
-                if key in self._fields or key in ("id", "pk", "_cls"):
-                    if __auto_convert and value is not None:
             field = self._fields.get(key)
+            if field or key in ("id", "pk", "_cls"):
+                if __auto_convert and value is not None:
                     if field and not isinstance(field, FileField):
                         value = field.to_python(value)
                 setattr(self, key, value)
             else:
+                if self._dynamic:
+                    dynamic_data[key] = value
+                else:
+                    # For strict Document
                     self._data[key] = value
 
         # Set any get_<field>_display methods
@@ -167,7 +155,7 @@ class BaseDocument:
         # Handle dynamic data only if an initialised dynamic document
         if self._dynamic and not self._dynamic_lock:
 
-            if not hasattr(self, name) and not name.startswith("_"):
+            if name not in self._fields_ordered and not name.startswith("_"):
                 DynamicField = _import_class("DynamicField")
                 field = DynamicField(db_field=name, null=True)
                 field.name = name
@@ -240,10 +228,10 @@ class BaseDocument:
                 setattr(self, k, data[k])
         if "_fields_ordered" in data:
             if self._dynamic:
-                setattr(self, "_fields_ordered", data["_fields_ordered"])
+                self._fields_ordered = data["_fields_ordered"]
             else:
                 _super_fields_ordered = type(self)._fields_ordered
-                setattr(self, "_fields_ordered", _super_fields_ordered)
+                self._fields_ordered = _super_fields_ordered
 
         dynamic_fields = data.get("_dynamic_fields") or SON()
         for k in dynamic_fields.keys():
@@ -253,8 +241,7 @@ class BaseDocument:
         return iter(self._fields_ordered)
 
     def __getitem__(self, name):
-        """Dictionary-style field access, return a field's value if present.
+        """Dictionary-style field access, return a field's value if present."""
-        """
         try:
             if name in self._fields_ordered:
                 return getattr(self, name)
@@ -263,8 +250,7 @@ class BaseDocument:
         raise KeyError(name)
 
     def __setitem__(self, name, value):
-        """Dictionary-style field access, set a field's value.
+        """Dictionary-style field access, set a field's value."""
-        """
         # Ensure that the field exists before settings its value
         if not self._dynamic and name not in self._fields:
             raise KeyError(name)
@@ -286,7 +272,7 @@ class BaseDocument:
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = "[Bad Unicode data]"
         repr_type = str if u is None else type(u)
-        return repr_type("<{}: {}>".format(self.__class__.__name__, u))
+        return repr_type(f"<{self.__class__.__name__}: {u}>")
 
     def __str__(self):
         # TODO this could be simpler?
@@ -314,7 +300,8 @@ class BaseDocument:
 
     def clean(self):
         """
-        Hook for doing document level data cleaning before validation is run.
+        Hook for doing document level data cleaning (usually validation or assignment)
+        before validation is run.
 
         Any ValidationError raised by this method will not be associated with
         a particular field; it will have a special-case association with the
@@ -441,7 +428,7 @@ class BaseDocument:
                 pk = self.pk
             elif self._instance and hasattr(self._instance, "pk"):
                 pk = self._instance.pk
-            message = "ValidationError ({}:{}) ".format(self._class_name, pk)
+            message = f"ValidationError ({self._class_name}:{pk}) "
             raise ValidationError(message, errors=errors)
 
     def to_json(self, *args, **kwargs):
@@ -514,7 +501,7 @@ class BaseDocument:
         if "." in key:
             key, rest = key.split(".", 1)
             key = self._db_field_map.get(key, key)
-            key = "{}.{}".format(key, rest)
+            key = f"{key}.{rest}"
         else:
             key = self._db_field_map.get(key, key)
 
@@ -537,6 +524,9 @@ class BaseDocument:
         """Using _get_changed_fields iterate and remove any fields that
         are marked as changed.
         """
+        ReferenceField = _import_class("ReferenceField")
+        GenericReferenceField = _import_class("GenericReferenceField")
+
         for changed in self._get_changed_fields():
             parts = changed.split(".")
             data = self
@@ -549,7 +539,8 @@ class BaseDocument:
                 elif isinstance(data, dict):
                     data = data.get(part, None)
                 else:
-                    data = getattr(data, part, None)
+                    field_name = data._reverse_db_field_map.get(part, part)
+                    data = getattr(data, field_name, None)
 
                 if not isinstance(data, LazyReference) and hasattr(
                     data, "_changed_fields"
@@ -558,10 +549,40 @@ class BaseDocument:
                         continue
 
                     data._changed_fields = []
+                elif isinstance(data, (list, tuple, dict)):
+                    if hasattr(data, "field") and isinstance(
+                        data.field, (ReferenceField, GenericReferenceField)
+                    ):
+                        continue
+                    BaseDocument._nestable_types_clear_changed_fields(data)
 
         self._changed_fields = []
 
-    def _nestable_types_changed_fields(self, changed_fields, base_key, data):
+    @staticmethod
+    def _nestable_types_clear_changed_fields(data):
+        """Inspect nested data for changed fields
+
+        :param data: data to inspect for changes
+        """
+        Document = _import_class("Document")
+
+        # Loop list / dict fields as they contain documents
+        # Determine the iterator to use
+        if not hasattr(data, "items"):
+            iterator = enumerate(data)
+        else:
+            iterator = data.items()
+
+        for _index_or_key, value in iterator:
+            if hasattr(value, "_get_changed_fields") and not isinstance(
+                value, Document
+            ):  # don't follow references
+                value._clear_changed_fields()
+            elif isinstance(value, (list, tuple, dict)):
+                BaseDocument._nestable_types_clear_changed_fields(value)
+
+    @staticmethod
+    def _nestable_types_changed_fields(changed_fields, base_key, data):
         """Inspect nested data for changed fields
 
         :param changed_fields: Previously collected changed fields
@@ -576,7 +597,7 @@ class BaseDocument:
             iterator = data.items()
 
         for index_or_key, value in iterator:
-            item_key = "{}{}.".format(base_key, index_or_key)
+            item_key = f"{base_key}{index_or_key}."
             # don't check anything lower if this key is already marked
             # as changed.
             if item_key[:-1] in changed_fields:
@@ -584,15 +605,18 @@ class BaseDocument:
 
             if hasattr(value, "_get_changed_fields"):
                 changed = value._get_changed_fields()
-                changed_fields += ["{}{}".format(item_key, k) for k in changed if k]
+                changed_fields += [f"{item_key}{k}" for k in changed if k]
             elif isinstance(value, (list, tuple, dict)):
-                self._nestable_types_changed_fields(changed_fields, item_key, value)
+                BaseDocument._nestable_types_changed_fields(
+                    changed_fields, item_key, value
+                )
 
     def _get_changed_fields(self):
-        """Return a list of all fields that have explicitly been changed.
+        """Return a list of all fields that have explicitly been changed."""
-        """
         EmbeddedDocument = _import_class("EmbeddedDocument")
+        LazyReferenceField = _import_class("LazyReferenceField")
         ReferenceField = _import_class("ReferenceField")
+        GenericLazyReferenceField = _import_class("GenericLazyReferenceField")
         GenericReferenceField = _import_class("GenericReferenceField")
         SortedListField = _import_class("SortedListField")
 
@@ -615,10 +639,16 @@ class BaseDocument:
             if isinstance(data, EmbeddedDocument):
                 # Find all embedded fields that have been changed
                 changed = data._get_changed_fields()
-                changed_fields += ["{}{}".format(key, k) for k in changed if k]
+                changed_fields += [f"{key}{k}" for k in changed if k]
             elif isinstance(data, (list, tuple, dict)):
                 if hasattr(field, "field") and isinstance(
-                    field.field, (ReferenceField, GenericReferenceField)
+                    field.field,
+                    (
+                        LazyReferenceField,
+                        ReferenceField,
+                        GenericLazyReferenceField,
+                        GenericReferenceField,
+                    ),
                 ):
                     continue
                 elif isinstance(field, SortedListField) and field._ordering:
@@ -721,11 +751,8 @@ class BaseDocument:
         return cls._meta.get("collection", None)
 
     @classmethod
-    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
+    def _from_son(cls, son, _auto_dereference=True, created=False):
-        """Create an instance of a Document (subclass) from a PyMongo SON."""
+        """Create an instance of a Document (subclass) from a PyMongo SON (dict)"""
-        if not only_fields:
-            only_fields = []
-
         if son and not isinstance(son, dict):
             raise ValueError(
                 "The source SON object needs to be of type 'dict' but a '%s' was found"
@@ -738,6 +765,8 @@ class BaseDocument:
 
         # Convert SON to a data dict, making sure each key is a string and
         # corresponds to the right db field.
+        # This is needed as _from_son is currently called both from BaseDocument.__init__
+        # and from EmbeddedDocumentField.to_python
         data = {}
         for key, value in son.items():
             key = str(key)
@@ -768,11 +797,10 @@ class BaseDocument:
                     errors_dict[field_name] = e
 
         if errors_dict:
-            errors = "\n".join(
+            errors = "\n".join([f"Field '{k}' - {v}" for k, v in errors_dict.items()])
-                ["Field '{}' - {}".format(k, v) for k, v in errors_dict.items()]
-            )
             msg = "Invalid data to create a `{}` instance.\n{}".format(
-                cls._class_name, errors,
+                cls._class_name,
+                errors,
             )
             raise InvalidDocumentError(msg)
 
@@ -780,9 +808,7 @@ class BaseDocument:
         if cls.STRICT:
             data = {k: v for k, v in data.items() if k in cls._fields}
 
-        obj = cls(
+        obj = cls(__auto_convert=False, _created=created, **data)
-            __auto_convert=False, _created=created, __only_fields=only_fields, **data
-        )
         obj._changed_fields = []
         if not _auto_dereference:
             obj._fields = fields
@@ -942,10 +968,7 @@ class BaseDocument:
                     unique_fields += unique_with
 
                 # Add the new index to the list
-                fields = [
+                fields = [(f"{namespace}{f}", pymongo.ASCENDING) for f in unique_fields]
-                    ("{}{}".format(namespace, f), pymongo.ASCENDING)
-                    for f in unique_fields
-                ]
                 index = {"fields": fields, "unique": True, "sparse": sparse}
                 unique_indexes.append(index)
 
@@ -981,9 +1004,7 @@ class BaseDocument:
             "PolygonField",
         )
 
-        geo_field_types = tuple(
+        geo_field_types = tuple(_import_class(field) for field in geo_field_type_names)
-            [_import_class(field) for field in geo_field_type_names]
-        )
 
         for field in cls._fields.values():
             if not isinstance(field, geo_field_types):
@@ -1001,7 +1022,7 @@ class BaseDocument:
             elif field._geo_index:
                 field_name = field.db_field
                 if parent_field:
-                    field_name = "{}.{}".format(parent_field, field_name)
+                    field_name = f"{parent_field}.{field_name}"
                 geo_indices.append({"fields": [(field_name, field._geo_index)]})
 
         return geo_indices
@@ -1139,8 +1160,7 @@ class BaseDocument:
 
     @classmethod
     def _translate_field_name(cls, field, sep="."):
-        """Translate a field attribute name to a database field name.
+        """Translate a field attribute name to a database field name."""
-        """
         parts = field.split(sep)
         parts = [f.db_field for f in cls._lookup_field(parts)]
         return ".".join(parts)

mongoengine/base/fields.py

@@ -1,12 +1,15 @@
 import operator
-import warnings
 import weakref
 
-from bson import DBRef, ObjectId, SON
 import pymongo
+from bson import SON, DBRef, ObjectId
 
 from mongoengine.base.common import UPDATE_OPERATORS
-from mongoengine.base.datastructures import BaseDict, BaseList, EmbeddedDocumentList
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    EmbeddedDocumentList,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import DeprecatedError, ValidationError
 
@@ -16,11 +19,9 @@ __all__ = ("BaseField", "ComplexBaseField", "ObjectIdField", "GeoJsonBaseField")
 class BaseField:
     """A base class for fields in a MongoDB document. Instances of this class
     may be added to subclasses of `Document` to define a document's schema.
-
-    .. versionchanged:: 0.5 - added verbose and help text
     """
 
-    name = None
+    name = None  # set in TopLevelDocumentMetaclass
     _geo_index = False
     _auto_gen = False  # Call `generate` to generate a value
     _auto_dereference = True
@@ -43,7 +44,7 @@ class BaseField:
         choices=None,
         null=False,
         sparse=False,
-        **kwargs
+        **kwargs,
     ):
         """
         :param db_field: The database field to store this field in
@@ -120,8 +121,7 @@ class BaseField:
             BaseField.creation_counter += 1
 
     def __get__(self, instance, owner):
-        """Descriptor for retrieving a value from a field in a document.
+        """Descriptor for retrieving a value from a field in a document."""
-        """
         if instance is None:
             # Document class being used rather than a document object
             return self
@@ -265,11 +265,22 @@ class ComplexBaseField(BaseField):
     Allows for nesting of embedded documents inside complex types.
     Handles the lazy dereferencing of a queryset by lazily dereferencing all
     items in a list / dict rather than one at a time.
-
-    .. versionadded:: 0.5
     """
 
-    field = None
+    def __init__(self, field=None, **kwargs):
+        self.field = field
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def _lazy_load_refs(instance, name, ref_values, *, max_depth):
+        _dereference = _import_class("DeReference")()
+        documents = _dereference(
+            ref_values,
+            max_depth=max_depth,
+            instance=instance,
+            name=name,
+        )
+        return documents
 
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references."""
@@ -288,19 +299,15 @@ class ComplexBaseField(BaseField):
             or isinstance(self.field, (GenericReferenceField, ReferenceField))
         )
 
-        _dereference = _import_class("DeReference")()
-
         if (
             instance._initialised
             and dereference
             and instance._data.get(self.name)
             and not getattr(instance._data[self.name], "_dereferenced", False)
         ):
-            instance._data[self.name] = _dereference(
+            ref_values = instance._data.get(self.name)
-                instance._data.get(self.name),
+            instance._data[self.name] = self._lazy_load_refs(
-                max_depth=1,
+                ref_values=ref_values, instance=instance, name=self.name, max_depth=1
-                instance=instance,
-                name=self.name,
             )
             if hasattr(instance._data[self.name], "_dereferenced"):
                 instance._data[self.name]._dereferenced = True
@@ -326,7 +333,9 @@ class ComplexBaseField(BaseField):
             and isinstance(value, (BaseList, BaseDict))
             and not value._dereferenced
         ):
-            value = _dereference(value, max_depth=1, instance=instance, name=self.name)
+            value = self._lazy_load_refs(
+                ref_values=value, instance=instance, name=self.name, max_depth=1
+            )
             value._dereferenced = True
             instance._data[self.name] = value
 
@@ -469,9 +478,7 @@ class ComplexBaseField(BaseField):
 
             if errors:
                 field_class = self.field.__class__.__name__
-                self.error(
+                self.error(f"Invalid {field_class} item ({value})", errors=errors)
-                    "Invalid {} item ({})".format(field_class, value), errors=errors
-                )
         # Don't allow empty values if required
         if self.required and not value:
             self.error("Field is required and cannot be empty")
@@ -520,10 +527,7 @@ class ObjectIdField(BaseField):
 
 
 class GeoJsonBaseField(BaseField):
-    """A geo json field storing a geojson style object.
+    """A geo json field storing a geojson style object."""
-
-    .. versionadded:: 0.8
-    """
 
     _geo_index = pymongo.GEOSPHERE
     _type = "GeoBase"
@@ -543,7 +547,7 @@ class GeoJsonBaseField(BaseField):
         if isinstance(value, dict):
             if set(value.keys()) == {"type", "coordinates"}:
                 if value["type"] != self._type:
-                    self.error('{} type must be "{}"'.format(self._name, self._type))
+                    self.error(f'{self._name} type must be "{self._type}"')
                 return self.validate(value["coordinates"])
             else:
                 self.error(

mongoengine/base/metaclasses.py

@@ -2,7 +2,11 @@ import itertools
 import warnings
 
 from mongoengine.base.common import _document_registry
-from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
+from mongoengine.base.fields import (
+    BaseField,
+    ComplexBaseField,
+    ObjectIdField,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidDocumentError
 from mongoengine.queryset import (
@@ -12,7 +16,6 @@ from mongoengine.queryset import (
     QuerySetManager,
 )
 
-
 __all__ = ("DocumentMetaclass", "TopLevelDocumentMetaclass")
 
 
@@ -337,7 +340,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         # allow_inheritance to False. If the base Document allows inheritance,
         # none of its subclasses can override allow_inheritance to False.
         simple_class = all(
-            [b._meta.get("abstract") for b in flattened_bases if hasattr(b, "_meta")]
+            b._meta.get("abstract") for b in flattened_bases if hasattr(b, "_meta")
         )
         if (
             not simple_class
@@ -439,8 +442,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         id_basename, id_db_basename, i = ("auto_id", "_auto_id", 0)
         for i in itertools.count():
-            id_name = "{}_{}".format(id_basename, i)
+            id_name = f"{id_basename}_{i}"
-            id_db_name = "{}_{}".format(id_db_basename, i)
+            id_db_name = f"{id_db_basename}_{i}"
             if id_name not in existing_fields and id_db_name not in existing_db_fields:
                 return id_name, id_db_name
 

mongoengine/connection.py

@@ -54,28 +54,26 @@ def _get_connection_settings(
     password=None,
     authentication_source=None,
     authentication_mechanism=None,
-    **kwargs
+    **kwargs,
 ):
     """Get the connection settings as a dict
 
-    : param db: the name of the database to use, for compatibility with connect
+    :param db: the name of the database to use, for compatibility with connect
-    : param name: the name of the specific database to use
+    :param name: the name of the specific database to use
-    : param host: the host name of the: program: `mongod` instance to connect to
+    :param host: the host name of the: program: `mongod` instance to connect to
-    : param port: the port that the: program: `mongod` instance is running on
+    :param port: the port that the: program: `mongod` instance is running on
-    : param read_preference: The read preference for the collection
+    :param read_preference: The read preference for the collection
-    : param username: username to authenticate with
+    :param username: username to authenticate with
-    : param password: password to authenticate with
+    :param password: password to authenticate with
-    : param authentication_source: database to authenticate against
+    :param authentication_source: database to authenticate against
-    : param authentication_mechanism: database authentication mechanisms.
+    :param authentication_mechanism: database authentication mechanisms.
         By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
         MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
-    : param is_mock: explicitly use mongomock for this connection
+    :param is_mock: explicitly use mongomock for this connection
         (can also be done by using `mongomock: // ` as db host prefix)
-    : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
-
-    .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = {
         "name": name or db or DEFAULT_DATABASE_NAME,
@@ -179,30 +177,27 @@ def register_connection(
     password=None,
     authentication_source=None,
     authentication_mechanism=None,
-    **kwargs
+    **kwargs,
 ):
     """Register the connection settings.
 
-    : param alias: the name that will be used to refer to this connection
+    :param alias: the name that will be used to refer to this connection throughout MongoEngine
-        throughout MongoEngine
+    :param db: the name of the database to use, for compatibility with connect
-    : param db: the name of the database to use, for compatibility with connect
+    :param name: the name of the specific database to use
-    : param name: the name of the specific database to use
+    :param host: the host name of the: program: `mongod` instance to connect to
-    : param host: the host name of the: program: `mongod` instance to connect to
+    :param port: the port that the: program: `mongod` instance is running on
-    : param port: the port that the: program: `mongod` instance is running on
+    :param read_preference: The read preference for the collection
-    : param read_preference: The read preference for the collection
+    :param username: username to authenticate with
-    : param username: username to authenticate with
+    :param password: password to authenticate with
-    : param password: password to authenticate with
+    :param authentication_source: database to authenticate against
-    : param authentication_source: database to authenticate against
+    :param authentication_mechanism: database authentication mechanisms.
-    : param authentication_mechanism: database authentication mechanisms.
         By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
         MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
-    : param is_mock: explicitly use mongomock for this connection
+    :param is_mock: explicitly use mongomock for this connection
         (can also be done by using `mongomock: // ` as db host prefix)
-    : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
-
-    .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = _get_connection_settings(
         db=db,
@@ -214,15 +209,15 @@ def register_connection(
         password=password,
         authentication_source=authentication_source,
         authentication_mechanism=authentication_mechanism,
-        **kwargs
+        **kwargs,
     )
     _connection_settings[alias] = conn_settings
 
 
 def disconnect(alias=DEFAULT_CONNECTION_NAME):
     """Close the connection with a given alias."""
-    from mongoengine.base.common import _get_documents_by_db
     from mongoengine import Document
+    from mongoengine.base.common import _get_documents_by_db
 
     if alias in _connections:
         get_connection(alias=alias).close()
@@ -317,7 +312,7 @@ def _create_connection(alias, connection_class, **connection_settings):
     try:
         return connection_class(**connection_settings)
     except Exception as e:
-        raise ConnectionFailure("Cannot connect to database {} :\n{}".format(alias, e))
+        raise ConnectionFailure(f"Cannot connect to database {alias} :\n{e}")
 
 
 def _find_existing_connection(connection_settings):
@@ -386,8 +381,6 @@ def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs):
 
     See the docstring for `register_connection` for more details about all
     supported kwargs.
-
-    .. versionchanged:: 0.6 - added multiple database support.
     """
     if alias in _connections:
         prev_conn_setting = _connection_settings[alias]

mongoengine/context_managers.py

@@ -177,15 +177,28 @@ class query_counter:
     This was designed for debugging purpose. In fact it is a global counter so queries issued by other threads/processes
     can interfere with it
 
+    Usage:
+
+    .. code-block:: python
+
+        class User(Document):
+            name = StringField()
+
+        with query_counter() as q:
+            user = User(name='Bob')
+            assert q == 0       # no query fired yet
+            user.save()
+            assert q == 1       # 1 query was fired, an 'insert'
+            user_bis = User.objects().first()
+            assert q == 2       # a 2nd query was fired, a 'find_one'
+
     Be aware that:
-    - Iterating over large amount of documents (>101) makes pymongo issue `getmore` queries to fetch the next batch of
+
-        documents (https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches)
+    - Iterating over large amount of documents (>101) makes pymongo issue `getmore` queries to fetch the next batch of documents (https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches)
     - Some queries are ignored by default by the counter (killcursors, db.system.indexes)
     """
 
     def __init__(self, alias=DEFAULT_CONNECTION_NAME):
-        """Construct the query_counter
-        """
         self.db = get_db(alias=alias)
         self.initial_profiling_level = None
         self._ctx_query_counter = 0  # number of queries issued by the context

mongoengine/dereference.py

@@ -1,4 +1,4 @@
-from bson import DBRef, SON
+from bson import SON, DBRef
 
 from mongoengine.base import (
     BaseDict,
@@ -10,7 +10,12 @@ from mongoengine.base import (
 from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.fields import DictField, ListField, MapField, ReferenceField
+from mongoengine.fields import (
+    DictField,
+    ListField,
+    MapField,
+    ReferenceField,
+)
 from mongoengine.queryset import QuerySet
 
 
@@ -51,10 +56,10 @@ class DeReference:
                 doc_type = doc_type.document_type
                 is_list = not hasattr(items, "items")
 
-                if is_list and all([i.__class__ == doc_type for i in items]):
+                if is_list and all(i.__class__ == doc_type for i in items):
                     return items
                 elif not is_list and all(
-                    [i.__class__ == doc_type for i in items.values()]
+                    i.__class__ == doc_type for i in items.values()
                 ):
                     return items
                 elif not field.dbref:
@@ -157,8 +162,7 @@ class DeReference:
         return reference_map
 
     def _fetch_objects(self, doc_type=None):
-        """Fetch all references and convert to their document objects
+        """Fetch all references and convert to their document objects"""
-        """
         object_map = {}
         for collection, dbrefs in self.reference_map.items():
 
@@ -272,12 +276,12 @@ class DeReference:
                             (v["_ref"].collection, v["_ref"].id), v
                         )
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                        item_name = "{}.{}.{}".format(name, k, field_name)
+                        item_name = f"{name}.{k}.{field_name}"
                         data[k]._data[field_name] = self._attach_objects(
                             v, depth, instance=instance, name=item_name
                         )
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                item_name = "{}.{}".format(name, k) if name else name
+                item_name = f"{name}.{k}" if name else name
                 data[k] = self._attach_objects(
                     v, depth - 1, instance=instance, name=item_name
                 )

mongoengine/document.py

@@ -1,8 +1,7 @@
 import re
-import warnings
 
-from bson.dbref import DBRef
 import pymongo
+from bson.dbref import DBRef
 from pymongo.read_preferences import ReadPreference
 
 from mongoengine import signals
@@ -17,14 +16,23 @@ from mongoengine.base import (
 )
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from mongoengine.context_managers import set_write_concern, switch_collection, switch_db
+from mongoengine.context_managers import (
+    set_write_concern,
+    switch_collection,
+    switch_db,
+)
 from mongoengine.errors import (
     InvalidDocumentError,
     InvalidQueryError,
     SaveConditionError,
 )
 from mongoengine.pymongo_support import list_collection_names
-from mongoengine.queryset import NotUniqueError, OperationError, QuerySet, transform
+from mongoengine.queryset import (
+    NotUniqueError,
+    OperationError,
+    QuerySet,
+    transform,
+)
 
 __all__ = (
     "Document",
@@ -91,6 +99,15 @@ class EmbeddedDocument(BaseDocument, metaclass=DocumentMetaclass):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __getstate__(self):
+        data = super().__getstate__()
+        data["_instance"] = None
+        return data
+
+    def __setstate__(self, state):
+        super().__setstate__(state)
+        self._instance = state["_instance"]
+
     def to_mongo(self, *args, **kwargs):
         data = super().to_mongo(*args, **kwargs)
 
@@ -110,7 +127,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     By default, the MongoDB collection used to store documents created using a
     :class:`~mongoengine.Document` subclass will be the name of the subclass
-    converted to lowercase. A different collection may be specified by
+    converted to snake_case. A different collection may be specified by
     providing :attr:`collection` to the :attr:`meta` dictionary in the class
     definition.
 
@@ -118,7 +135,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     create a specialised version of the document that will be stored in the
     same collection. To facilitate this behaviour a `_cls`
     field is added to documents (hidden though the MongoEngine interface).
-    To enable this behaviourset :attr:`allow_inheritance` to ``True`` in the
+    To enable this behaviour set :attr:`allow_inheritance` to ``True`` in the
     :attr:`meta` dictionary.
 
     A :class:`~mongoengine.Document` may use a **Capped Collection** by
@@ -324,7 +341,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
         _refs=None,
         save_condition=None,
         signal_kwargs=None,
-        **kwargs
+        **kwargs,
     ):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
@@ -367,15 +384,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             meta['cascade'] = True.  Also you can pass different kwargs to
             the cascade save using cascade_kwargs which overwrites the
             existing kwargs with custom values.
-        .. versionchanged:: 0.8.5
-            Optional save_condition that only overwrites existing documents
-            if the condition is satisfied in the current db record.
-        .. versionchanged:: 0.10
-            :class:`OperationError` exception raised if save_condition fails.
-        .. versionchanged:: 0.10.1
-            :class: save_condition failure now raises a `SaveConditionError`
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         signal_kwargs = signal_kwargs or {}
 
@@ -464,9 +472,9 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             # insert_one will provoke UniqueError alongside save does not
             # therefore, it need to catch and call replace_one.
             if "_id" in doc:
-                raw_object = wc_collection.find_one_and_replace(
+                select_dict = {"_id": doc["_id"]}
-                    {"_id": doc["_id"]}, doc
+                select_dict = self._integrate_shard_key(doc, select_dict)
-                )
+                raw_object = wc_collection.find_one_and_replace(select_dict, doc)
                 if raw_object:
                     return doc["_id"]
 
@@ -489,6 +497,23 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         return update_doc
 
+    def _integrate_shard_key(self, doc, select_dict):
+        """Integrates the collection's shard key to the `select_dict`, which will be used for the query.
+        The value from the shard key is taken from the `doc` and finally the select_dict is returned.
+        """
+
+        # Need to add shard key to query, or you get an error
+        shard_key = self._meta.get("shard_key", tuple())
+        for k in shard_key:
+            path = self._lookup_field(k.split("."))
+            actual_key = [p.db_field for p in path]
+            val = doc
+            for ak in actual_key:
+                val = val[ak]
+            select_dict[".".join(actual_key)] = val
+
+        return select_dict
+
     def _save_update(self, doc, save_condition, write_concern):
         """Update an existing document.
 
@@ -504,15 +529,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         select_dict["_id"] = object_id
 
-        # Need to add shard key to query, or you get an error
+        select_dict = self._integrate_shard_key(doc, select_dict)
-        shard_key = self._meta.get("shard_key", tuple())
-        for k in shard_key:
-            path = self._lookup_field(k.split("."))
-            actual_key = [p.db_field for p in path]
-            val = doc
-            for ak in actual_key:
-                val = val[ak]
-            select_dict[".".join(actual_key)] = val
 
         update_doc = self._get_update_doc()
         if update_doc:
@@ -555,7 +572,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             if not getattr(ref, "_changed_fields", True):
                 continue
 
-            ref_id = "{},{}".format(ref.__class__.__name__, str(ref._data))
+            ref_id = f"{ref.__class__.__name__},{str(ref._data)}"
             if ref and ref_id not in _refs:
                 _refs.append(ref_id)
                 kwargs["_refs"] = _refs
@@ -566,7 +583,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     def _qs(self):
         """Return the default queryset corresponding to this document."""
         if not hasattr(self, "__objects"):
-            self.__objects = QuerySet(self, self._get_collection())
+            self.__objects = QuerySet(self.__class__, self._get_collection())
         return self.__objects
 
     @property
@@ -621,9 +638,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             For example, ``save(..., w: 2, fsync: True)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
-
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         signal_kwargs = signal_kwargs or {}
         signals.pre_delete.send(self.__class__, document=self, **signal_kwargs)
@@ -639,7 +653,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
                 write_concern=write_concern, _from_doc_delete=True
             )
         except pymongo.errors.OperationFailure as err:
-            message = "Could not delete document (%s)" % err.message
+            message = "Could not delete document (%s)" % err.args
             raise OperationError(message)
         signals.post_delete.send(self.__class__, document=self, **signal_kwargs)
 
@@ -705,8 +719,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     def select_related(self, max_depth=1):
         """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to
         a maximum depth in order to cut down the number queries to mongodb.
-
-        .. versionadded:: 0.5
         """
         DeReference = _import_class("DeReference")
         DeReference()([self], max_depth + 1)
@@ -717,10 +729,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         :param fields: (optional) args list of fields to reload
         :param max_depth: (optional) depth of dereferencing to follow
-
-        .. versionadded:: 0.1.2
-        .. versionchanged:: 0.6  Now chainable
-        .. versionchanged:: 0.9  Can provide specific fields to reload
         """
         max_depth = 1
         if fields and isinstance(fields[0], int):
@@ -822,9 +830,6 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         Raises :class:`OperationError` if the document has no collection set
         (i.g. if it is `abstract`)
-
-        .. versionchanged:: 0.10.7
-            :class:`OperationError` exception raised if no collection available
         """
         coll_name = cls._get_collection_name()
         if not coll_name:
@@ -870,6 +875,10 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         Global defaults can be set in the meta - see :doc:`guide/defining-documents`
 
+        By default, this will get called automatically upon first interaction with the
+        Document collection (query, save, etc) so unless you disabled `auto_create_index`, you
+        shouldn't have to call this manually.
+
         .. note:: You can disable automatic index creation by setting
                   `auto_create_index` to False in the documents meta data
         """
@@ -919,8 +928,10 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     @classmethod
     def list_indexes(cls):
-        """ Lists all of the indexes that should be created for given
+        """Lists all indexes that should be created for the Document collection.
-        collection. It includes all the indexes from super- and sub-classes.
+        It includes all the indexes from super- and sub-classes.
+
+        Note that it will only return the indexes' fields, not the indexes' options
         """
         if cls._meta.get("abstract"):
             return []
@@ -984,7 +995,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     @classmethod
     def compare_indexes(cls):
-        """ Compares the indexes defined in MongoEngine with the ones
+        """Compares the indexes defined in MongoEngine with the ones
         existing in the database. Returns any missing/extra indexes.
         """
 
@@ -1079,8 +1090,6 @@ class MapReduceDocument:
                 an ``ObjectId`` found in the given ``collection``,
                 the object can be accessed via the ``object`` property.
     :param value: The result(s) for this key.
-
-    .. versionadded:: 0.3
     """
 
     def __init__(self, document, collection, key, value):

mongoengine/errors.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 
-
 __all__ = (
     "NotRegistered",
     "InvalidDocumentError",
@@ -18,11 +17,15 @@ __all__ = (
 )
 
 
-class NotRegistered(Exception):
+class MongoEngineException(Exception):
     pass
 
 
-class InvalidDocumentError(Exception):
+class NotRegistered(MongoEngineException):
+    pass
+
+
+class InvalidDocumentError(MongoEngineException):
     pass
 
 
@@ -30,19 +33,19 @@ class LookUpError(AttributeError):
     pass
 
 
-class DoesNotExist(Exception):
+class DoesNotExist(MongoEngineException):
     pass
 
 
-class MultipleObjectsReturned(Exception):
+class MultipleObjectsReturned(MongoEngineException):
     pass
 
 
-class InvalidQueryError(Exception):
+class InvalidQueryError(MongoEngineException):
     pass
 
 
-class OperationError(Exception):
+class OperationError(MongoEngineException):
     pass
 
 
@@ -58,7 +61,7 @@ class SaveConditionError(OperationError):
     pass
 
 
-class FieldDoesNotExist(Exception):
+class FieldDoesNotExist(MongoEngineException):
     """Raised when trying to set a field
     not declared in a :class:`~mongoengine.Document`
     or an :class:`~mongoengine.EmbeddedDocument`.
@@ -94,7 +97,7 @@ class ValidationError(AssertionError):
         return str(self.message)
 
     def __repr__(self):
-        return "{}({},)".format(self.__class__.__name__, self.message)
+        return f"{self.__class__.__name__}({self.message},)"
 
     def __getattribute__(self, name):
         message = super().__getattribute__(name)
@@ -102,7 +105,7 @@ class ValidationError(AssertionError):
             if self.field_name:
                 message = "%s" % message
             if self.errors:
-                message = "{}({})".format(message, self._format_errors())
+                message = f"{message}({self._format_errors()})"
         return message
 
     def _get_message(self):
@@ -147,16 +150,16 @@ class ValidationError(AssertionError):
             elif isinstance(value, dict):
                 value = " ".join([generate_key(v, k) for k, v in value.items()])
 
-            results = "{}.{}".format(prefix, value) if prefix else value
+            results = f"{prefix}.{value}" if prefix else value
             return results
 
         error_dict = defaultdict(list)
         for k, v in self.to_dict().items():
             error_dict[generate_key(v)].append(k)
-        return " ".join(["{}: {}".format(k, v) for k, v in error_dict.items()])
+        return " ".join([f"{k}: {v}" for k, v in error_dict.items()])
 
 
-class DeprecatedError(Exception):
+class DeprecatedError(MongoEngineException):
     """Raise when a user uses a feature that has been Deprecated"""
 
     pass

mongoengine/fields.py

@@ -1,5 +1,6 @@
 import datetime
 import decimal
+import inspect
 import itertools
 import re
 import socket
@@ -8,10 +9,10 @@ import uuid
 from io import BytesIO
 from operator import itemgetter
 
-from bson import Binary, DBRef, ObjectId, SON
-from bson.int64 import Int64
 import gridfs
 import pymongo
+from bson import SON, Binary, DBRef, ObjectId
+from bson.int64 import Int64
 from pymongo import ReturnDocument
 
 try:
@@ -21,7 +22,6 @@ except ImportError:
 else:
     import dateutil.parser
 
-
 from mongoengine.base import (
     BaseDocument,
     BaseField,
@@ -35,8 +35,11 @@ from mongoengine.base.utils import LazyRegexCompiler
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
+from mongoengine.errors import (
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
+    DoesNotExist,
+    InvalidQueryError,
+    ValidationError,
+)
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
 from mongoengine.queryset.transform import STRING_OPERATORS
@@ -87,6 +90,7 @@ __all__ = (
     "PolygonField",
     "SequenceField",
     "UUIDField",
+    "EnumField",
     "MultiPointField",
     "MultiLineStringField",
     "MultiPolygonField",
@@ -100,6 +104,12 @@ class StringField(BaseField):
     """A unicode string field."""
 
     def __init__(self, regex=None, max_length=None, min_length=None, **kwargs):
+        """
+        :param regex: (optional) A string pattern that will be applied during validation
+        :param max_length: (optional) A max length that will be applied during validation
+        :param min_length: (optional) A min length that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.regex = re.compile(regex) if regex else None
         self.max_length = max_length
         self.min_length = min_length
@@ -147,7 +157,14 @@ class StringField(BaseField):
                 regex = r"%s$"
             elif op == "exact":
                 regex = r"^%s$"
+            elif op == "wholeword":
+                regex = r"\b%s\b"
+            elif op == "regex":
+                regex = value
 
+            if op == "regex":
+                value = re.compile(regex, flags)
+            else:
                 # escape unsafe characters which could lead to a re.error
                 value = re.escape(value)
                 value = re.compile(regex % value, flags)
@@ -155,10 +172,7 @@ class StringField(BaseField):
 
 
 class URLField(StringField):
-    """A field that validates input as an URL.
+    """A field that validates input as an URL."""
-
-    .. versionadded:: 0.3
-    """
 
     _URL_REGEX = LazyRegexCompiler(
         r"^(?:[a-z0-9\.\-]*)://"  # scheme is validated separately
@@ -173,6 +187,11 @@ class URLField(StringField):
     _URL_SCHEMES = ["http", "https", "ftp", "ftps"]
 
     def __init__(self, url_regex=None, schemes=None, **kwargs):
+        """
+        :param url_regex: (optional) Overwrite the default regex used for validation
+        :param schemes: (optional) Overwrite the default URL schemes that are allowed
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
+        """
         self.url_regex = url_regex or self._URL_REGEX
         self.schemes = schemes or self._URL_SCHEMES
         super().__init__(**kwargs)
@@ -181,18 +200,15 @@ class URLField(StringField):
         # Check first if the scheme is valid
         scheme = value.split("://")[0].lower()
         if scheme not in self.schemes:
-            self.error("Invalid scheme {} in URL: {}".format(scheme, value))
+            self.error(f"Invalid scheme {scheme} in URL: {value}")
 
         # Then check full URL
         if not self.url_regex.match(value):
-            self.error("Invalid URL: {}".format(value))
+            self.error(f"Invalid URL: {value}")
 
 
 class EmailField(StringField):
-    """A field that validates input as an email address.
+    """A field that validates input as an email address."""
-
-    .. versionadded:: 0.4
-    """
 
     USER_REGEX = LazyRegexCompiler(
         # `dot-atom` defined in RFC 5322 Section 3.2.3.
@@ -226,18 +242,13 @@ class EmailField(StringField):
         allow_utf8_user=False,
         allow_ip_domain=False,
         *args,
-        **kwargs
+        **kwargs,
     ):
-        """Initialize the EmailField.
+        """
-
+        :param domain_whitelist: (optional) list of valid domain names applied during validation
-        Args:
+        :param allow_utf8_user: Allow user part of the email to contain utf8 char
-            domain_whitelist (list) - list of otherwise invalid domain
+        :param allow_ip_domain: Allow domain part of the email to be an IPv4 or IPv6 address
-                                      names which you'd like to support.
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
-            allow_utf8_user (bool) - if True, the user part of the email
-                                     address can contain UTF8 characters.
-                                     False by default.
-            allow_ip_domain (bool) - if True, the domain part of the email
-                                     can be a valid IPv4 or IPv6 address.
         """
         self.domain_whitelist = domain_whitelist or []
         self.allow_utf8_user = allow_utf8_user
@@ -309,6 +320,11 @@ class IntField(BaseField):
     """32-bit integer field."""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -342,6 +358,11 @@ class LongField(BaseField):
     """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)"""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -378,6 +399,11 @@ class FloatField(BaseField):
     """Floating point number field."""
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
+        """
+        :param min_value: (optional) A min value that will be applied during validation
+        :param max_value: (optional) A max value that will be applied during validation
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
+        """
         self.min_value, self.max_value = min_value, max_value
         super().__init__(**kwargs)
 
@@ -414,9 +440,6 @@ class FloatField(BaseField):
 class DecimalField(BaseField):
     """Fixed-point decimal number field. Stores the value as a float by default unless `force_string` is used.
     If using floats, beware of Decimal to float conversion (potential precision loss)
-
-    .. versionchanged:: 0.8
-    .. versionadded:: 0.3
     """
 
     def __init__(
@@ -426,14 +449,14 @@ class DecimalField(BaseField):
         force_string=False,
         precision=2,
         rounding=decimal.ROUND_HALF_UP,
-        **kwargs
+        **kwargs,
     ):
         """
-        :param min_value: Validation rule for the minimum acceptable value.
+        :param min_value: (optional) A min value that will be applied during validation
-        :param max_value: Validation rule for the maximum acceptable value.
+        :param max_value: (optional) A max value that will be applied during validation
         :param force_string: Store the value as a string (instead of a float).
          Be aware that this affects query sorting and operation like lte, gte (as string comparison is applied)
-         and some query operator won't work (e.g: inc, dec)
+         and some query operator won't work (e.g. inc, dec)
         :param precision: Number of decimal places to store.
         :param rounding: The rounding rule from the python decimal library:
 
@@ -447,11 +470,15 @@ class DecimalField(BaseField):
             - decimal.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
 
             Defaults to: ``decimal.ROUND_HALF_UP``
-
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
         """
         self.min_value = min_value
         self.max_value = max_value
         self.force_string = force_string
+
+        if precision < 0 or not isinstance(precision, int):
+            self.error("precision must be a positive integer")
+
         self.precision = precision
         self.rounding = rounding
 
@@ -466,9 +493,12 @@ class DecimalField(BaseField):
             value = decimal.Decimal("%s" % value)
         except (TypeError, ValueError, decimal.InvalidOperation):
             return value
+        if self.precision > 0:
             return value.quantize(
                 decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding
             )
+        else:
+            return value.quantize(decimal.Decimal(), rounding=self.rounding)
 
     def to_mongo(self, value):
         if value is None:
@@ -497,15 +527,12 @@ class DecimalField(BaseField):
 
 
 class BooleanField(BaseField):
-    """Boolean field type.
+    """Boolean field type."""
-
-    .. versionadded:: 0.1.2
-    """
 
     def to_python(self, value):
         try:
             value = bool(value)
-        except ValueError:
+        except (ValueError, TypeError):
             pass
         return value
 
@@ -545,12 +572,13 @@ class DateTimeField(BaseField):
         if callable(value):
             return value()
 
-        if not isinstance(value, str):
+        if isinstance(value, str):
+            return self._parse_datetime(value)
+        else:
             return None
 
-        return self._parse_datetime(value)
+    @staticmethod
-
+    def _parse_datetime(value):
-    def _parse_datetime(self, value):
         # Attempt to parse a datetime from a string
         value = value.strip()
         if not value:
@@ -626,13 +654,12 @@ class ComplexDateTimeField(StringField):
     keyword when initializing the field.
 
     Note: To default the field to the current datetime, use: DateTimeField(default=datetime.utcnow)
-
-    .. versionadded:: 0.5
     """
 
     def __init__(self, separator=",", **kwargs):
         """
         :param separator: Allows to customize the separator used for storage (default ``,``)
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.StringField`
         """
         self.separator = separator
         self.format = separator.join(["%Y", "%m", "%d", "%H", "%M", "%S", "%f"])
@@ -773,6 +800,9 @@ class EmbeddedDocumentField(BaseField):
 
     def prepare_query_value(self, op, value):
         if value is not None and not isinstance(value, self.document_type):
+            # Short circuit for special operators, returning them as is
+            if isinstance(value, dict) and all(k.startswith("$") for k in value.keys()):
+                return value
             try:
                 value = self.document_type._from_son(value)
             except ValueError:
@@ -844,8 +874,7 @@ class DynamicField(BaseField):
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
     def to_mongo(self, value, use_db_field=True, fields=None):
-        """Convert a Python type to a MongoDB compatible type.
+        """Convert a Python type to a MongoDB compatible type."""
-        """
 
         if isinstance(value, str):
             return value
@@ -903,17 +932,16 @@ class ListField(ComplexBaseField):
     """A list field that wraps a standard field, allowing multiple instances
     of the field to be used as a list in the database.
 
-    If using with ReferenceFields see: :ref:`one-to-many-with-listfields`
+    If using with ReferenceFields see: :ref:`many-to-many-with-listfields`
 
     .. note::
         Required means it cannot be empty - as the default for ListFields is []
     """
 
     def __init__(self, field=None, max_length=None, **kwargs):
-        self.field = field
         self.max_length = max_length
         kwargs.setdefault("default", lambda: [])
-        super().__init__(**kwargs)
+        super().__init__(field=field, **kwargs)
 
     def __get__(self, instance, owner):
         if instance is None:
@@ -972,16 +1000,13 @@ class EmbeddedDocumentListField(ListField):
     .. note::
         The only valid list values are subclasses of
         :class:`~mongoengine.EmbeddedDocument`.
-
-    .. versionadded:: 0.9
     """
 
     def __init__(self, document_type, **kwargs):
         """
         :param document_type: The type of
          :class:`~mongoengine.EmbeddedDocument` the list will hold.
-        :param kwargs: Keyword arguments passed directly into the parent
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.ListField`
-         :class:`~mongoengine.ListField`.
         """
         super().__init__(field=EmbeddedDocumentField(document_type), **kwargs)
 
@@ -996,19 +1021,11 @@ class SortedListField(ListField):
         save the whole list then other processes trying to save the whole list
         as well could overwrite changes.  The safest way to append to a list is
         to perform a push operation.
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.6 - added reverse keyword
     """
 
-    _ordering = None
-    _order_reverse = False
-
     def __init__(self, field, **kwargs):
-        if "ordering" in kwargs.keys():
+        self._ordering = kwargs.pop("ordering", None)
-            self._ordering = kwargs.pop("ordering")
+        self._order_reverse = kwargs.pop("reverse", False)
-        if "reverse" in kwargs.keys():
-            self._order_reverse = kwargs.pop("reverse")
         super().__init__(field, **kwargs)
 
     def to_mongo(self, value, use_db_field=True, fields=None):
@@ -1029,17 +1046,6 @@ def key_not_string(d):
             return True
 
 
-def key_has_dot_or_dollar(d):
-    """Helper function to recursively determine if any key in a
-    dictionary contains a dot or a dollar sign.
-    """
-    for k, v in d.items():
-        if ("." in k or k.startswith("$")) or (
-            isinstance(v, dict) and key_has_dot_or_dollar(v)
-        ):
-            return True
-
-
 def key_starts_with_dollar(d):
     """Helper function to recursively determine if any key in a
     dictionary starts with a dollar
@@ -1055,17 +1061,13 @@ class DictField(ComplexBaseField):
 
     .. note::
         Required means it cannot be empty - as the default for DictFields is {}
-
-    .. versionadded:: 0.3
-    .. versionchanged:: 0.5 - Can now handle complex / varying types of data
     """
 
     def __init__(self, field=None, *args, **kwargs):
-        self.field = field
         self._auto_dereference = False
 
         kwargs.setdefault("default", lambda: {})
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, field=field, **kwargs)
 
     def validate(self, value):
         """Make sure that a list of valid fields is being used."""
@@ -1091,16 +1093,7 @@ class DictField(ComplexBaseField):
         return DictField(db_field=member_name)
 
     def prepare_query_value(self, op, value):
-        match_operators = [
+        match_operators = [*STRING_OPERATORS]
-            "contains",
-            "icontains",
-            "startswith",
-            "istartswith",
-            "endswith",
-            "iendswith",
-            "exact",
-            "iexact",
-        ]
 
         if op in match_operators and isinstance(value, str):
             return StringField().prepare_query_value(op, value)
@@ -1121,8 +1114,6 @@ class MapField(DictField):
     """A field that maps a name to a specified field type. Similar to
     a DictField, except the 'value' of each item must match the specified
     field type.
-
-    .. versionadded:: 0.5
     """
 
     def __init__(self, field=None, *args, **kwargs):
@@ -1170,8 +1161,6 @@ class ReferenceField(BaseField):
             org = ReferenceField('Org', reverse_delete_rule=CASCADE)
 
         User.register_delete_rule(Org, 'owner', DENY)
-
-    .. versionchanged:: 0.5 added `reverse_delete_rule`
     """
 
     def __init__(
@@ -1179,10 +1168,12 @@ class ReferenceField(BaseField):
     ):
         """Initialises the Reference Field.
 
+        :param document_type: The type of Document that will be referenced
         :param dbref:  Store the reference as :class:`~pymongo.dbref.DBRef`
-          or as the :class:`~pymongo.objectid.ObjectId`.id .
+          or as the :class:`~pymongo.objectid.ObjectId`.
         :param reverse_delete_rule: Determines what to do when the referring
           object is deleted
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
 
         .. note ::
             A reference to an abstract document type is always stored as a
@@ -1211,6 +1202,14 @@ class ReferenceField(BaseField):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         """Descriptor to allow lazy dereferencing."""
         if instance is None:
@@ -1218,20 +1217,17 @@ class ReferenceField(BaseField):
             return self
 
         # Get value from document instance if available
-        value = instance._data.get(self.name)
+        ref_value = instance._data.get(self.name)
         auto_dereference = instance._fields[self.name]._auto_dereference
         # Dereference DBRefs
-        if auto_dereference and isinstance(value, DBRef):
+        if auto_dereference and isinstance(ref_value, DBRef):
-            if hasattr(value, "cls"):
+            if hasattr(ref_value, "cls"):
                 # Dereference using the class type specified in the reference
-                cls = get_document(value.cls)
+                cls = get_document(ref_value.cls)
             else:
                 cls = self.document_type
-            dereferenced = cls._get_db().dereference(value)
+
-            if dereferenced is None:
+            instance._data[self.name] = self._lazy_load_ref(cls, ref_value)
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = cls._from_son(dereferenced)
 
         return super().__get__(instance, owner)
 
@@ -1304,24 +1300,22 @@ class ReferenceField(BaseField):
 
 
 class CachedReferenceField(BaseField):
-    """
+    """A referencefield with cache fields to purpose pseudo-joins"""
-    A referencefield with cache fields to purpose pseudo-joins
-
-    .. versionadded:: 0.9
-    """
 
     def __init__(self, document_type, fields=None, auto_sync=True, **kwargs):
         """Initialises the Cached Reference Field.
 
+        :param document_type: The type of Document that will be referenced
         :param fields:  A list of fields to be cached in document
-        :param auto_sync: if True documents are auto updated.
+        :param auto_sync: if True documents are auto updated
+        :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
         """
         if fields is None:
             fields = []
 
         # XXX ValidationError raised outside of the "validate" method.
-        if not isinstance(document_type, str) and not issubclass(
+        if not isinstance(document_type, str) and not (
-            document_type, Document
+            inspect.isclass(document_type) and issubclass(document_type, Document)
         ):
             self.error(
                 "Argument to CachedReferenceField constructor must be a"
@@ -1343,7 +1337,7 @@ class CachedReferenceField(BaseField):
             return None
 
         update_kwargs = {
-            "set__{}__{}".format(self.name, key): val
+            f"set__{self.name}__{key}": val
             for key, val in document._delta()[0].items()
             if key in self.fields
         }
@@ -1372,6 +1366,14 @@ class CachedReferenceField(BaseField):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         if instance is None:
             # Document class being used rather than a document object
@@ -1383,11 +1385,7 @@ class CachedReferenceField(BaseField):
 
         # Dereference DBRefs
         if auto_dereference and isinstance(value, DBRef):
-            dereferenced = self.document_type._get_db().dereference(value)
+            instance._data[self.name] = self._lazy_load_ref(self.document_type, value)
-            if dereferenced is None:
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = self.document_type._from_son(dereferenced)
 
         return super().__get__(instance, owner)
 
@@ -1482,8 +1480,6 @@ class GenericReferenceField(BaseField):
           it.
 
         * You can use the choices param to limit the acceptable Document types
-
-    .. versionadded:: 0.3
     """
 
     def __init__(self, *args, **kwargs):
@@ -1514,6 +1510,14 @@ class GenericReferenceField(BaseField):
             value = value._class_name
         super()._validate_choices(value)
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         if instance is None:
             return self
@@ -1521,12 +1525,9 @@ class GenericReferenceField(BaseField):
         value = instance._data.get(self.name)
 
         auto_dereference = instance._fields[self.name]._auto_dereference
-        if auto_dereference and isinstance(value, (dict, SON)):
+        if auto_dereference and isinstance(value, dict):
-            dereferenced = self.dereference(value)
+            doc_cls = get_document(value["_cls"])
-            if dereferenced is None:
+            instance._data[self.name] = self._lazy_load_ref(doc_cls, value["_ref"])
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = dereferenced
 
         return super().__get__(instance, owner)
 
@@ -1545,14 +1546,6 @@ class GenericReferenceField(BaseField):
                 " saved to the database"
             )
 
-    def dereference(self, value):
-        doc_cls = get_document(value["_cls"])
-        reference = value["_ref"]
-        doc = doc_cls._get_db().dereference(reference)
-        if doc is not None:
-            doc = doc_cls._from_son(doc)
-        return doc
-
     def to_mongo(self, document):
         if document is None:
             return None
@@ -1619,17 +1612,80 @@ class BinaryField(BaseField):
         return super().prepare_query_value(op, self.to_mongo(value))
 
 
+class EnumField(BaseField):
+    """Enumeration Field. Values are stored underneath as is,
+    so it will only work with simple types (str, int, etc) that
+    are bson encodable
+
+    Example usage:
+
+    .. code-block:: python
+
+        class Status(Enum):
+            NEW = 'new'
+            ONGOING = 'ongoing'
+            DONE = 'done'
+
+        class ModelWithEnum(Document):
+            status = EnumField(Status, default=Status.NEW)
+
+        ModelWithEnum(status='done')
+        ModelWithEnum(status=Status.DONE)
+
+    Enum fields can be searched using enum or its value:
+
+    .. code-block:: python
+
+        ModelWithEnum.objects(status='new').count()
+        ModelWithEnum.objects(status=Status.NEW).count()
+
+    The values can be restricted to a subset of the enum by using the ``choices`` parameter:
+
+    .. code-block:: python
+
+        class ModelWithEnum(Document):
+            status = EnumField(Status, choices=[Status.NEW, Status.DONE])
+    """
+
+    def __init__(self, enum, **kwargs):
+        self._enum_cls = enum
+        if kwargs.get("choices"):
+            invalid_choices = []
+            for choice in kwargs["choices"]:
+                if not isinstance(choice, enum):
+                    invalid_choices.append(choice)
+            if invalid_choices:
+                raise ValueError("Invalid choices: %r" % invalid_choices)
+        else:
+            kwargs["choices"] = list(self._enum_cls)  # Implicit validator
+        super().__init__(**kwargs)
+
+    def __set__(self, instance, value):
+        is_legal_value = value is None or isinstance(value, self._enum_cls)
+        if not is_legal_value:
+            try:
+                value = self._enum_cls(value)
+            except Exception:
+                pass
+        return super().__set__(instance, value)
+
+    def to_mongo(self, value):
+        if isinstance(value, self._enum_cls):
+            return value.value
+        return value
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+        return super().prepare_query_value(op, self.to_mongo(value))
+
+
 class GridFSError(Exception):
     pass
 
 
 class GridFSProxy:
-    """Proxy object to handle writing and reading of files to and from GridFS
+    """Proxy object to handle writing and reading of files to and from GridFS"""
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.5 - added optional size param to read
-    .. versionchanged:: 0.6 - added collection name param
-    """
 
     _fs = None
 
@@ -1687,12 +1743,12 @@ class GridFSProxy:
         return self.__copy__()
 
     def __repr__(self):
-        return "<{}: {}>".format(self.__class__.__name__, self.grid_id)
+        return f"<{self.__class__.__name__}: {self.grid_id}>"
 
     def __str__(self):
         gridout = self.get()
-        filename = getattr(gridout, "filename") if gridout else "<no file>"
+        filename = gridout.filename if gridout else "<no file>"
-        return "<{}: {} ({})>".format(self.__class__.__name__, filename, self.grid_id)
+        return f"<{self.__class__.__name__}: {filename} ({self.grid_id})>"
 
     def __eq__(self, other):
         if isinstance(other, GridFSProxy):
@@ -1791,12 +1847,7 @@ class GridFSProxy:
 
 
 class FileField(BaseField):
-    """A GridFS storage field.
+    """A GridFS storage field."""
-
-    .. versionadded:: 0.4
-    .. versionchanged:: 0.5 added optional size param for read
-    .. versionchanged:: 0.6 added db_alias for multidb support
-    """
 
     proxy_class = GridFSProxy
 
@@ -1878,11 +1929,7 @@ class FileField(BaseField):
 
 
 class ImageGridFsProxy(GridFSProxy):
-    """
+    """Proxy for ImageField"""
-    Proxy for ImageField
-
-    versionadded: 0.6
-    """
 
     def put(self, file_obj, **kwargs):
         """
@@ -2016,8 +2063,6 @@ class ImageField(FileField):
     :param size: max size to store images, provided as (width, height, force)
         if larger, it will be automatically resized (ex: size=(800, 600, True))
     :param thumbnail_size: size to generate a thumbnail, provided as (width, height, force)
-
-    .. versionadded:: 0.6
     """
 
     proxy_class = ImageGridFsProxy
@@ -2042,7 +2087,7 @@ class ImageField(FileField):
 
 class SequenceField(BaseField):
     """Provides a sequential counter see:
-     http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers
+     https://docs.mongodb.com/manual/reference/method/ObjectId/#ObjectIDs-SequenceNumbers
 
     .. note::
 
@@ -2065,9 +2110,6 @@ class SequenceField(BaseField):
         In case the counter is defined in the abstract document, it will be
         common to all inherited documents and the default sequence name will
         be the class name of the abstract document.
-
-    .. versionadded:: 0.5
-    .. versionchanged:: 0.8 added `value_decorator`
     """
 
     _auto_gen = True
@@ -2081,7 +2123,7 @@ class SequenceField(BaseField):
         sequence_name=None,
         value_decorator=None,
         *args,
-        **kwargs
+        **kwargs,
     ):
         self.collection_name = collection_name or self.COLLECTION_NAME
         self.db_alias = db_alias or DEFAULT_CONNECTION_NAME
@@ -2096,7 +2138,7 @@ class SequenceField(BaseField):
         Generate and Increment the counter
         """
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
 
         counter = collection.find_one_and_update(
@@ -2110,7 +2152,7 @@ class SequenceField(BaseField):
     def set_next_value(self, value):
         """Helper method to set the next sequence value"""
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
         counter = collection.find_one_and_update(
             filter={"_id": sequence_id},
@@ -2127,7 +2169,7 @@ class SequenceField(BaseField):
         as it is only fixed on set.
         """
         sequence_name = self.get_sequence_name()
-        sequence_id = "{}.{}".format(sequence_name, self.name)
+        sequence_id = f"{sequence_name}.{self.name}"
         collection = get_db(alias=self.db_alias)[self.collection_name]
         data = collection.find_one({"_id": sequence_id})
 
@@ -2180,10 +2222,7 @@ class SequenceField(BaseField):
 
 
 class UUIDField(BaseField):
-    """A UUID field.
+    """A UUID field."""
-
-    .. versionadded:: 0.6
-    """
 
     _binary = None
 
@@ -2192,9 +2231,6 @@ class UUIDField(BaseField):
         Store UUID data in the database
 
         :param binary: if False store as a string.
-
-        .. versionchanged:: 0.8.0
-        .. versionchanged:: 0.6.19
         """
         self._binary = binary
         super().__init__(**kwargs)
@@ -2239,8 +2275,6 @@ class GeoPointField(BaseField):
         representing a geo point. It admits 2d indexes but not "2dsphere" indexes
         in MongoDB > 2.4 which are more natural for modeling geospatial points.
         See :ref:`geospatial-indexes`
-
-    .. versionadded:: 0.4
     """
 
     _geo_index = pymongo.GEO2D
@@ -2272,8 +2306,6 @@ class PointField(GeoJsonBaseField):
     to set the value.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "Point"
@@ -2292,8 +2324,6 @@ class LineStringField(GeoJsonBaseField):
     You can either pass a dict with the full information or a list of points.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "LineString"
@@ -2315,8 +2345,6 @@ class PolygonField(GeoJsonBaseField):
     holes.
 
     Requires mongodb >= 2.4
-
-    .. versionadded:: 0.8
     """
 
     _type = "Polygon"
@@ -2336,8 +2364,6 @@ class MultiPointField(GeoJsonBaseField):
     to set the value.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiPoint"
@@ -2357,8 +2383,6 @@ class MultiLineStringField(GeoJsonBaseField):
     You can either pass a dict with the full information or a list of points.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiLineString"
@@ -2385,8 +2409,6 @@ class MultiPolygonField(GeoJsonBaseField):
     of Polygons.
 
     Requires mongodb >= 2.6
-
-    .. versionadded:: 0.9
     """
 
     _type = "MultiPolygon"
@@ -2399,8 +2421,6 @@ class LazyReferenceField(BaseField):
     Instead, access will return a :class:`~mongoengine.base.LazyReference` class
     instance, allowing access to `pk` or manual dereference by using
     ``fetch()`` method.
-
-    .. versionadded:: 0.15
     """
 
     def __init__(
@@ -2409,7 +2429,7 @@ class LazyReferenceField(BaseField):
         passthrough=False,
         dbref=False,
         reverse_delete_rule=DO_NOTHING,
-        **kwargs
+        **kwargs,
     ):
         """Initialises the Reference Field.
 
@@ -2419,7 +2439,7 @@ class LazyReferenceField(BaseField):
           object is deleted
         :param passthrough: When trying to access unknown fields, the
           :class:`~mongoengine.base.datastructure.LazyReference` instance will
-          automatically call `fetch()` and try to retrive the field on the fetched
+          automatically call `fetch()` and try to retrieve the field on the fetched
           document. Note this only work getting field (not setting or deleting).
         """
         # XXX ValidationError raised outside of the "validate" method.
@@ -2503,6 +2523,7 @@ class LazyReferenceField(BaseField):
         if not isinstance(value, (DBRef, Document, EmbeddedDocument)):
             collection = self.document_type._get_collection_name()
             value = DBRef(collection, self.document_type.id.to_python(value))
+            value = self.build_lazyref(value)
         return value
 
     def validate(self, value):
@@ -2563,8 +2584,6 @@ class GenericLazyReferenceField(GenericReferenceField):
           it.
 
         * You can use the choices param to limit the acceptable Document types
-
-    .. versionadded:: 0.15
     """
 
     def __init__(self, *args, **kwargs):

mongoengine/mongodb_support.py

@@ -3,11 +3,12 @@ Helper functions, constants, and types to aid with MongoDB version support
 """
 from mongoengine.connection import get_connection
 
-
 # Constant that can be used to compare the version retrieved with
 # get_mongodb_version()
 MONGODB_34 = (3, 4)
 MONGODB_36 = (3, 6)
+MONGODB_42 = (4, 2)
+MONGODB_44 = (4, 4)
 
 
 def get_mongodb_version():

mongoengine/pymongo_support.py

@@ -2,6 +2,7 @@
 Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support.
 """
 import pymongo
+from pymongo.errors import OperationFailure
 
 _PYMONGO_37 = (3, 7)
 
@@ -10,13 +11,40 @@ PYMONGO_VERSION = tuple(pymongo.version_tuple[:2])
 IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37
 
 
-def count_documents(collection, filter):
+def count_documents(
+    collection, filter, skip=None, limit=None, hint=None, collation=None
+):
     """Pymongo>3.7 deprecates count in favour of count_documents"""
+    if limit == 0:
+        return 0  # Pymongo raises an OperationFailure if called with limit=0
+
+    kwargs = {}
+    if skip is not None:
+        kwargs["skip"] = skip
+    if limit is not None:
+        kwargs["limit"] = limit
+    if hint not in (-1, None):
+        kwargs["hint"] = hint
+    if collation is not None:
+        kwargs["collation"] = collation
+
+    # count_documents appeared in pymongo 3.7
     if IS_PYMONGO_GTE_37:
-        return collection.count_documents(filter)
+        try:
-    else:
+            return collection.count_documents(filter=filter, **kwargs)
-        count = collection.find(filter).count()
+        except OperationFailure:
-    return count
+            # OperationFailure - accounts for some operators that used to work
+            # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere)
+            # fallback to deprecated Cursor.count
+            # Keeping this should be reevaluated the day pymongo removes .count entirely
+            pass
+
+    cursor = collection.find(filter)
+    for option, option_value in kwargs.items():
+        cursor_method = getattr(cursor, option)
+        cursor = cursor_method(option_value)
+    with_limit_and_skip = "skip" in kwargs or "limit" in kwargs
+    return cursor.count(with_limit_and_skip=with_limit_and_skip)
 
 
 def list_collection_names(db, include_system_collections=False):

mongoengine/queryset/base.py

@@ -2,13 +2,12 @@ import copy
 import itertools
 import re
 import warnings
-
 from collections.abc import Mapping
 
-from bson import SON, json_util
-from bson.code import Code
 import pymongo
 import pymongo.errors
+from bson import SON, json_util
+from bson.code import Code
 from pymongo.collection import ReturnDocument
 from pymongo.common import validate_read_preference
 from pymongo.read_concern import ReadConcern
@@ -29,11 +28,11 @@ from mongoengine.errors import (
     NotUniqueError,
     OperationError,
 )
+from mongoengine.pymongo_support import count_documents
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
 
-
 __all__ = ("BaseQuerySet", "DO_NOTHING", "NULLIFY", "CASCADE", "DENY", "PULL")
 
 # Delete rules
@@ -63,6 +62,7 @@ class BaseQuerySet:
         self._ordering = None
         self._snapshot = False
         self._timeout = True
+        self._allow_disk_use = False
         self._read_preference = None
         self._read_concern = None
         self._iter = False
@@ -83,13 +83,20 @@ class BaseQuerySet:
         self._cursor_obj = None
         self._limit = None
         self._skip = None
+
         self._hint = -1  # Using -1 as None is a valid value for hint
         self._collation = None
         self._batch_size = None
-        self.only_fields = []
         self._max_time_ms = None
         self._comment = None
 
+        # Hack - As people expect cursor[5:5] to return
+        # an empty result set. It's hard to do that right, though, because the
+        # server uses limit(0) to mean 'no limit'. So we set _empty
+        # in that case and check for it when iterating. We also unset
+        # it anytime we change _limit. Inspired by how it is done in pymongo.Cursor
+        self._empty = False
+
     def __call__(self, q_obj=None, **query):
         """Filter the selected documents by calling the
         :class:`~mongoengine.queryset.QuerySet` with a query.
@@ -162,6 +169,7 @@ class BaseQuerySet:
         [<User: User object>, <User: User object>]
         """
         queryset = self.clone()
+        queryset._empty = False
 
         # Handle a slice
         if isinstance(key, slice):
@@ -169,6 +177,8 @@ class BaseQuerySet:
             queryset._skip, queryset._limit = key.start, key.stop
             if key.start and key.stop:
                 queryset._limit = key.stop - key.start
+            if queryset._limit == 0:
+                queryset._empty = True
 
             # Allow further QuerySet modifications to be performed
             return queryset
@@ -180,7 +190,6 @@ class BaseQuerySet:
                     queryset._document._from_son(
                         queryset._cursor[key],
                         _auto_dereference=self._auto_dereference,
-                        only_fields=self.only_fields,
                     )
                 )
 
@@ -190,7 +199,6 @@ class BaseQuerySet:
             return queryset._document._from_son(
                 queryset._cursor[key],
                 _auto_dereference=self._auto_dereference,
-                only_fields=self.only_fields,
             )
 
         raise TypeError("Provide a slice or an integer index")
@@ -249,8 +257,6 @@ class BaseQuerySet:
         `DocumentName.MultipleObjectsReturned` exception if multiple results
         and :class:`~mongoengine.queryset.DoesNotExist` or
         `DocumentName.DoesNotExist` if no results are found.
-
-        .. versionadded:: 0.3
         """
         queryset = self.clone()
         queryset = queryset.order_by().limit(2)
@@ -274,10 +280,7 @@ class BaseQuerySet:
         )
 
     def create(self, **kwargs):
-        """Create new object. Returns the saved object instance.
+        """Create new object. Returns the saved object instance."""
-
-        .. versionadded:: 0.4
-        """
         return self._document(**kwargs).save(force_insert=True)
 
     def first(self):
@@ -309,10 +312,6 @@ class BaseQuerySet:
 
         By default returns document instances, set ``load_bulk`` to False to
         return just ``ObjectIds``
-
-        .. versionadded:: 0.5
-        .. versionchanged:: 0.10.7
-            Add signal_kwargs argument
         """
         Document = _import_class("Document")
 
@@ -394,9 +393,36 @@ class BaseQuerySet:
             :meth:`skip` that has been applied to this cursor into account when
             getting the count
         """
-        if self._limit == 0 and with_limit_and_skip is False or self._none:
+        # mimic the fact that setting .limit(0) in pymongo sets no limit
+        # https://docs.mongodb.com/manual/reference/method/cursor.limit/#zero-value
+        if (
+            self._limit == 0
+            and with_limit_and_skip is False
+            or self._none
+            or self._empty
+        ):
             return 0
-        count = self._cursor.count(with_limit_and_skip=with_limit_and_skip)
+
+        kwargs = (
+            {"limit": self._limit, "skip": self._skip} if with_limit_and_skip else {}
+        )
+
+        if self._limit == 0:
+            # mimic the fact that historically .limit(0) sets no limit
+            kwargs.pop("limit", None)
+
+        if self._hint not in (-1, None):
+            kwargs["hint"] = self._hint
+
+        if self._collation:
+            kwargs["collation"] = self._collation
+
+        count = count_documents(
+            collection=self._cursor.collection,
+            filter=self._query,
+            **kwargs,
+        )
+
         self._cursor_obj = None
         return count
 
@@ -498,7 +524,7 @@ class BaseQuerySet:
         write_concern=None,
         read_concern=None,
         full_result=False,
-        **update
+        **update,
     ):
         """Perform an atomic update on the fields matched by the query.
 
@@ -516,8 +542,6 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
 
         :returns the number of updated documents (unless ``full_result`` is True)
-
-        .. versionadded:: 0.2
         """
         if not update and not upsert:
             raise OperationError("No update parameters, would remove data")
@@ -569,8 +593,6 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
 
         :returns the new or overwritten document
-
-        .. versionadded:: 0.10.2
         """
 
         atomic_update = self.update(
@@ -579,7 +601,7 @@ class BaseQuerySet:
             write_concern=write_concern,
             read_concern=read_concern,
             full_result=True,
-            **update
+            **update,
         )
 
         if atomic_update.raw_result["updatedExisting"]:
@@ -604,14 +626,13 @@ class BaseQuerySet:
         :param update: Django-style update keyword arguments
             full_result
         :returns the number of updated documents (unless ``full_result`` is True)
-        .. versionadded:: 0.2
         """
         return self.update(
             upsert=upsert,
             multi=False,
             write_concern=write_concern,
             full_result=full_result,
-            **update
+            **update,
         )
 
     def modify(
@@ -636,8 +657,6 @@ class BaseQuerySet:
         :param new: return updated rather than original document
             (default ``False``)
         :param update: Django-style update keyword arguments
-
-        .. versionadded:: 0.9
         """
 
         if remove and new:
@@ -671,7 +690,7 @@ class BaseQuerySet:
                     upsert=upsert,
                     sort=sort,
                     return_document=return_doc,
-                    **self._cursor_args
+                    **self._cursor_args,
                 )
         except pymongo.errors.DuplicateKeyError as err:
             raise NotUniqueError("Update failed (%s)" % err)
@@ -680,12 +699,10 @@ class BaseQuerySet:
 
         if full_response:
             if result["value"] is not None:
-                result["value"] = self._document._from_son(
+                result["value"] = self._document._from_son(result["value"])
-                    result["value"], only_fields=self.only_fields
-                )
         else:
             if result is not None:
-                result = self._document._from_son(result, only_fields=self.only_fields)
+                result = self._document._from_son(result)
 
         return result
 
@@ -695,8 +712,6 @@ class BaseQuerySet:
         `None` if no document exists with that id.
 
         :param object_id: the value for the id of the document to look up
-
-        .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set
         """
         queryset = self.clone()
         if not queryset._query_obj.empty:
@@ -705,22 +720,18 @@ class BaseQuerySet:
         return queryset.filter(pk=object_id).first()
 
     def in_bulk(self, object_ids):
-        """"Retrieve a set of documents by their ids.
+        """Retrieve a set of documents by their ids.
 
         :param object_ids: a list or tuple of ObjectId's
         :rtype: dict of ObjectId's as keys and collection-specific
                 Document subclasses as values.
-
-        .. versionadded:: 0.3
         """
         doc_map = {}
 
         docs = self._collection.find({"_id": {"$in": object_ids}}, **self._cursor_args)
         if self._scalar:
             for doc in docs:
-                doc_map[doc["_id"]] = self._get_scalar(
+                doc_map[doc["_id"]] = self._get_scalar(self._document._from_son(doc))
-                    self._document._from_son(doc, only_fields=self.only_fields)
-                )
         elif self._as_pymongo:
             for doc in docs:
                 doc_map[doc["_id"]] = doc
@@ -728,14 +739,15 @@ class BaseQuerySet:
             for doc in docs:
                 doc_map[doc["_id"]] = self._document._from_son(
                     doc,
-                    only_fields=self.only_fields,
                     _auto_dereference=self._auto_dereference,
                 )
 
         return doc_map
 
     def none(self):
-        """Helper that just returns a list"""
+        """Returns a queryset that never returns any objects and no query will be executed when accessing the results
+        inspired by django none() https://docs.djangoproject.com/en/dev/ref/models/querysets/#none
+        """
         queryset = self.clone()
         queryset._none = True
         return queryset
@@ -755,8 +767,6 @@ class BaseQuerySet:
         evaluated against if you are using more than one database.
 
         :param alias: The database alias
-
-        .. versionadded:: 0.9
         """
 
         with switch_db(self._document, alias) as cls:
@@ -788,17 +798,19 @@ class BaseQuerySet:
             "_ordering",
             "_snapshot",
             "_timeout",
+            "_allow_disk_use",
             "_read_preference",
+            "_read_concern",
             "_iter",
             "_scalar",
             "_as_pymongo",
             "_limit",
             "_skip",
+            "_empty",
             "_hint",
             "_collation",
             "_auto_dereference",
             "_search_text",
-            "only_fields",
             "_max_time_ms",
             "_comment",
             "_batch_size",
@@ -817,8 +829,6 @@ class BaseQuerySet:
         """Handles dereferencing of :class:`~bson.dbref.DBRef` objects or
         :class:`~bson.object_id.ObjectId` a maximum depth in order to cut down
         the number queries to mongodb.
-
-        .. versionadded:: 0.5
         """
         # Make select related work the same for querysets
         max_depth += 1
@@ -834,6 +844,7 @@ class BaseQuerySet:
         """
         queryset = self.clone()
         queryset._limit = n
+        queryset._empty = False  # cancels the effect of empty
 
         # If a cursor object has already been created, apply the limit to it.
         if queryset._cursor_obj:
@@ -866,8 +877,6 @@ class BaseQuerySet:
 
         Hinting will not do anything if the corresponding index does not exist.
         The last hint applied to this cursor takes precedence over all others.
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         queryset._hint = index
@@ -929,10 +938,6 @@ class BaseQuerySet:
 
         .. note:: This is a command and won't take ordering or limit into
            account.
-
-        .. versionadded:: 0.4
-        .. versionchanged:: 0.5 - Fixed handling references
-        .. versionchanged:: 0.6 - Improved db_field refrence handling
         """
         queryset = self.clone()
 
@@ -996,12 +1001,8 @@ class BaseQuerySet:
         field filters.
 
         :param fields: fields to include
-
-        .. versionadded:: 0.3
-        .. versionchanged:: 0.5 - Added subfield support
         """
         fields = {f: QueryFieldList.ONLY for f in fields}
-        self.only_fields = list(fields.keys())
         return self.fields(True, **fields)
 
     def exclude(self, *fields):
@@ -1018,8 +1019,6 @@ class BaseQuerySet:
         field filters.
 
         :param fields: fields to exclude
-
-        .. versionadded:: 0.5
         """
         fields = {f: QueryFieldList.EXCLUDE for f in fields}
         return self.fields(**fields)
@@ -1046,8 +1045,6 @@ class BaseQuerySet:
 
         :param kwargs: A set of keyword arguments identifying what to
             include, exclude, or slice.
-
-        .. versionadded:: 0.5
         """
 
         # Check for an operator and transform to mongo-style if there is
@@ -1089,8 +1086,6 @@ class BaseQuerySet:
         .exclude(). ::
 
             post = BlogPost.objects.exclude('comments').all_fields()
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         queryset._loaded_fields = QueryFieldList(
@@ -1163,9 +1158,6 @@ class BaseQuerySet:
         """Enable or disable snapshot mode when querying.
 
         :param enabled: whether or not snapshot mode is enabled
-
-        ..versionchanged:: 0.5 - made chainable
-        .. deprecated:: Ignored with PyMongo 3+
         """
         msg = "snapshot is deprecated as it has no impact when using PyMongo 3+."
         warnings.warn(msg, DeprecationWarning)
@@ -1173,12 +1165,20 @@ class BaseQuerySet:
         queryset._snapshot = enabled
         return queryset
 
+    def allow_disk_use(self, enabled):
+        """Enable or disable the use of temporary files on disk while processing a blocking sort operation.
+         (To store data exceeding the 100 megabyte system memory limit)
+
+        :param enabled: whether or not temporary files on disk are used
+        """
+        queryset = self.clone()
+        queryset._allow_disk_use = enabled
+        return queryset
+
     def timeout(self, enabled):
         """Enable or disable the default mongod timeout when querying. (no_cursor_timeout option)
 
         :param enabled: whether or not the timeout is used
-
-        ..versionchanged:: 0.5 - made chainable
         """
         queryset = self.clone()
         queryset._timeout = enabled
@@ -1203,7 +1203,7 @@ class BaseQuerySet:
             preference.
         """
         if read_concern is not None and not isinstance(read_concern, Mapping):
-            raise TypeError("%r is not a valid read concern." % (read_concern,))
+            raise TypeError(f"{read_concern!r} is not a valid read concern.")
 
         queryset = self.clone()
         queryset._read_concern = (
@@ -1266,10 +1266,7 @@ class BaseQuerySet:
     def from_json(self, json_data):
         """Converts json data to unsaved objects"""
         son_data = json_util.loads(json_data)
-        return [
+        return [self._document._from_son(data) for data in son_data]
-            self._document._from_son(data, only_fields=self.only_fields)
-            for data in son_data
-        ]
 
     def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
         """Perform a aggregate function based in your queryset params
@@ -1280,7 +1277,6 @@ class BaseQuerySet:
             parameter will be removed shortly
         :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call
             See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate
-        .. versionadded:: 0.9
         """
         using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)
         user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline)
@@ -1311,10 +1307,11 @@ class BaseQuerySet:
         final_pipeline = initial_pipeline + user_pipeline
 
         collection = self._collection
-        if self._read_preference is not None:
+        if self._read_preference is not None or self._read_concern is not None:
             collection = self._collection.with_options(
-                read_preference=self._read_preference
+                read_preference=self._read_preference, read_concern=self._read_concern
             )
+
         return collection.aggregate(final_pipeline, cursor={}, **kwargs)
 
     # JS functionality
@@ -1351,32 +1348,23 @@ class BaseQuerySet:
             Map/Reduce changed in server version **>= 1.7.4**. The PyMongo
             :meth:`~pymongo.collection.Collection.map_reduce` helper requires
             PyMongo version **>= 1.11**.
-
-        .. versionchanged:: 0.5
-           - removed ``keep_temp`` keyword argument, which was only relevant
-             for MongoDB server versions older than 1.7.4
-
-        .. versionadded:: 0.3
         """
         queryset = self.clone()
 
         MapReduceDocument = _import_class("MapReduceDocument")
 
-        if not hasattr(self._collection, "map_reduce"):
-            raise NotImplementedError("Requires MongoDB >= 1.7.1")
-
         map_f_scope = {}
         if isinstance(map_f, Code):
             map_f_scope = map_f.scope
             map_f = str(map_f)
-        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope)
+        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None)
 
         reduce_f_scope = {}
         if isinstance(reduce_f, Code):
             reduce_f_scope = reduce_f.scope
             reduce_f = str(reduce_f)
         reduce_f_code = queryset._sub_js_fields(reduce_f)
-        reduce_f = Code(reduce_f_code, reduce_f_scope)
+        reduce_f = Code(reduce_f_code, reduce_f_scope or None)
 
         mr_args = {"query": queryset._query}
 
@@ -1386,7 +1374,7 @@ class BaseQuerySet:
                 finalize_f_scope = finalize_f.scope
                 finalize_f = str(finalize_f)
             finalize_f_code = queryset._sub_js_fields(finalize_f)
-            finalize_f = Code(finalize_f_code, finalize_f_scope)
+            finalize_f = Code(finalize_f_code, finalize_f_scope or None)
             mr_args["finalize"] = finalize_f
 
         if scope:
@@ -1493,8 +1481,6 @@ class BaseQuerySet:
         .. note:: When using this mode of query, the database will call your
                   function, or evaluate your predicate clause, for each object
                   in the collection.
-
-        .. versionadded:: 0.5
         """
         queryset = self.clone()
         where_clause = queryset._sub_js_fields(where_clause)
@@ -1571,9 +1557,6 @@ class BaseQuerySet:
         :param field: the field to use
         :param normalize: normalize the results so they add to 1.0
         :param map_reduce: Use map_reduce over exec_js
-
-        .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded
-                            document lookups
         """
         if map_reduce:
             return self._item_frequencies_map_reduce(field, normalize=normalize)
@@ -1582,9 +1565,8 @@ class BaseQuerySet:
     # Iterator helpers
 
     def __next__(self):
-        """Wrap the result in a :class:`~mongoengine.Document` object.
+        """Wrap the result in a :class:`~mongoengine.Document` object."""
-        """
+        if self._none or self._empty:
-        if self._limit == 0 or self._none:
             raise StopIteration
 
         raw_doc = next(self._cursor)
@@ -1595,7 +1577,6 @@ class BaseQuerySet:
         doc = self._document._from_son(
             raw_doc,
             _auto_dereference=self._auto_dereference,
-            only_fields=self.only_fields,
         )
 
         if self._scalar:
@@ -1603,13 +1584,8 @@ class BaseQuerySet:
 
         return doc
 
-    next = __next__  # For Python2 support
-
     def rewind(self):
-        """Rewind the cursor to its unevaluated state.
+        """Rewind the cursor to its unevaluated state."""
-
-        .. versionadded:: 0.3
-        """
         self._iter = False
         self._cursor.rewind()
 
@@ -1635,6 +1611,9 @@ class BaseQuerySet:
         if not self._timeout:
             cursor_args["no_cursor_timeout"] = True
 
+        if self._allow_disk_use:
+            cursor_args["allow_disk_use"] = True
+
         if self._loaded_fields:
             cursor_args[fields_name] = self._loaded_fields.as_dict()
 

mongoengine/queryset/manager.py

@@ -1,4 +1,5 @@
 from functools import partial
+
 from mongoengine.queryset.queryset import QuerySet
 
 __all__ = ("queryset_manager", "QuerySetManager")

mongoengine/queryset/queryset.py

@@ -1,11 +1,11 @@
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (
-    BaseQuerySet,
     CASCADE,
     DENY,
     DO_NOTHING,
     NULLIFY,
     PULL,
+    BaseQuerySet,
 )
 
 __all__ = (
@@ -144,15 +144,13 @@ class QuerySet(BaseQuerySet):
             return super().count(with_limit_and_skip)
 
         if self._len is None:
+            # cache the length
             self._len = super().count(with_limit_and_skip)
 
         return self._len
 
     def no_cache(self):
-        """Convert to a non-caching queryset
+        """Convert to a non-caching queryset"""
-
-        .. versionadded:: 0.8.3 Convert to non caching queryset
-        """
         if self._result_cache is not None:
             raise OperationError("QuerySet already cached")
 
@@ -163,17 +161,11 @@ class QuerySetNoCache(BaseQuerySet):
     """A non caching QuerySet"""
 
     def cache(self):
-        """Convert to a caching queryset
+        """Convert to a caching queryset"""
-
-        .. versionadded:: 0.8.3 Convert to caching queryset
-        """
         return self._clone_into(QuerySet(self._document, self._collection))
 
     def __repr__(self):
-        """Provides the string representation of the QuerySet
+        """Provides the string representation of the QuerySet"""
-
-        .. versionchanged:: 0.6.13 Now doesnt modify the cursor
-        """
         if self._iter:
             return ".. queryset mid-iteration .."
 

mongoengine/queryset/transform.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
 
-from bson import ObjectId, SON
-from bson.dbref import DBRef
 import pymongo
+from bson import SON, ObjectId
+from bson.dbref import DBRef
 
 from mongoengine.base import UPDATE_OPERATORS
 from mongoengine.common import _import_class
@@ -51,6 +51,10 @@ STRING_OPERATORS = (
     "iendswith",
     "exact",
     "iexact",
+    "regex",
+    "iregex",
+    "wholeword",
+    "iwholeword",
 )
 CUSTOM_OPERATORS = ("match",)
 MATCH_OPERATORS = (

mongoengine/queryset/visitor.py

@@ -7,18 +7,20 @@ from mongoengine.queryset import transform
 __all__ = ("Q", "QNode")
 
 
+def warn_empty_is_deprecated():
+    msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
+    warnings.warn(msg, DeprecationWarning, stacklevel=2)
+
+
 class QNodeVisitor:
-    """Base visitor class for visiting Q-object nodes in a query tree.
+    """Base visitor class for visiting Q-object nodes in a query tree."""
-    """
 
     def visit_combination(self, combination):
-        """Called by QCombination objects.
+        """Called by QCombination objects."""
-        """
         return combination
 
     def visit_query(self, query):
-        """Called by (New)Q objects.
+        """Called by (New)Q objects."""
-        """
         return query
 
 
@@ -44,8 +46,7 @@ class SimplificationVisitor(QNodeVisitor):
         return combination
 
     def _query_conjunction(self, queries):
-        """Merges query dicts - effectively &ing them together.
+        """Merges query dicts - effectively &ing them together."""
-        """
         query_ops = set()
         combined_query = {}
         for query in queries:
@@ -98,19 +99,18 @@ class QNode:
         object.
         """
         # If the other Q() is empty, ignore it and just use `self`.
-        if getattr(other, "empty", True):
+        if not bool(other):
             return self
 
         # Or if this Q is empty, ignore it and just use `other`.
-        if self.empty:
+        if not bool(self):
             return other
 
         return QCombination(operation, [self, other])
 
     @property
     def empty(self):
-        msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
+        warn_empty_is_deprecated()
-        warnings.warn(msg, DeprecationWarning)
         return False
 
     def __or__(self, other):
@@ -152,8 +152,7 @@ class QCombination(QNode):
 
     @property
     def empty(self):
-        msg = "'empty' property is deprecated in favour of using 'not bool(filter)'"
+        warn_empty_is_deprecated()
-        warnings.warn(msg, DeprecationWarning)
         return not bool(self.children)
 
     def __eq__(self, other):
@@ -186,4 +185,5 @@ class Q(QNode):
 
     @property
     def empty(self):
+        warn_empty_is_deprecated()
         return not bool(self.query)

requirements-dev.txt

@@ -1,6 +1,5 @@
 black
 flake8
-flake8-import-order
 pre-commit
 pytest
 ipdb

requirements.txt

@@ -1,3 +0,0 @@
-pymongo>=3.4
-Sphinx==1.5.5
-sphinx-rtd-theme==0.2.4

setup.cfg

@@ -1,10 +1,18 @@
 [flake8]
-ignore=E501,F401,F403,F405,I201,I202,W504, W605, W503
+ignore=E501,F403,F405,I201,I202,W504,W605,W503,B007
 exclude=build,dist,docs,venv,venv3,.tox,.eggs,tests
 max-complexity=47
-application-import-names=mongoengine,tests
 
 [tool:pytest]
 # Limits the discovery to tests directory
 # avoids that it runs for instance the benchmark
 testpaths = tests
+
+[isort]
+known_first_party = mongoengine,tests
+default_section = THIRDPARTY
+multi_line_output = 3
+include_trailing_comma = True
+combine_as_imports = True
+line_length = 70
+ensure_newline_before_comments = 1

setup.py

@@ -7,7 +7,7 @@ from setuptools.command.test import test as TestCommand
 
 # Hack to silence atexit traceback in newer python versions
 try:
-    import multiprocessing
+    import multiprocessing  # noqa: F401
 except ImportError:
     pass
 
@@ -53,8 +53,8 @@ class PyTest(TestCommand):
 
     def run_tests(self):
         # import here, cause outside the eggs aren't loaded
-        from pkg_resources import _namespace_packages
         import pytest
+        from pkg_resources import _namespace_packages
 
         # Purge modules under test from sys.modules. The test loader will
         # re-import them from the build location. Required when 2to3 is used
@@ -98,10 +98,11 @@ CLASSIFIERS = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Database",
@@ -111,11 +112,11 @@ CLASSIFIERS = [
 extra_opts = {
     "packages": find_packages(exclude=["tests", "tests.*"]),
     "tests_require": [
-        "pytest<5.0",
+        "pytest",
         "pytest-cov",
-        "coverage<5.0",  # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls
+        "coverage",
         "blinker",
-        "Pillow>=2.0.0, <7.0.0",  # 7.0.0 dropped Python2 support
+        "Pillow>=7.0.0",
     ],
 }
 
@@ -140,7 +141,7 @@ setup(
     long_description=LONG_DESCRIPTION,
     platforms=["any"],
     classifiers=CLASSIFIERS,
-    python_requires=">=3.5",
+    python_requires=">=3.6",
     install_requires=["pymongo>=3.4, <4.0"],
     cmdclass={"test": PyTest},
     **extra_opts

tests/document/test_class_methods.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *
@@ -27,16 +26,14 @@ class TestClassMethods(unittest.TestCase):
             self.db.drop_collection(collection)
 
     def test_definition(self):
-        """Ensure that document may be defined using fields.
+        """Ensure that document may be defined using fields."""
-        """
         assert ["_cls", "age", "id", "name"] == sorted(self.Person._fields.keys())
         assert ["IntField", "ObjectIdField", "StringField", "StringField"] == sorted(
-            [x.__class__.__name__ for x in self.Person._fields.values()]
+            x.__class__.__name__ for x in self.Person._fields.values()
         )
 
     def test_get_db(self):
-        """Ensure that get_db returns the expected db.
+        """Ensure that get_db returns the expected db."""
-        """
         db = self.Person._get_db()
         assert self.db == db
 
@@ -48,15 +45,13 @@ class TestClassMethods(unittest.TestCase):
         assert collection_name == self.Person._get_collection_name()
 
     def test_get_collection(self):
-        """Ensure that get_collection returns the expected collection.
+        """Ensure that get_collection returns the expected collection."""
-        """
         collection_name = "person"
         collection = self.Person._get_collection()
         assert self.db[collection_name] == collection
 
     def test_drop_collection(self):
-        """Ensure that the collection may be dropped from the database.
+        """Ensure that the collection may be dropped from the database."""
-        """
         collection_name = "person"
         self.Person(name="Test").save()
         assert collection_name in list_collection_names(self.db)
@@ -78,7 +73,7 @@ class TestClassMethods(unittest.TestCase):
         assert self.Person._meta["delete_rules"] == {(Job, "employee"): NULLIFY}
 
     def test_compare_indexes(self):
-        """ Ensure that the indexes are properly created and that
+        """Ensure that the indexes are properly created and that
         compare_indexes identifies the missing/extra indexes
         """
 
@@ -111,7 +106,7 @@ class TestClassMethods(unittest.TestCase):
         }
 
     def test_compare_indexes_inheritance(self):
-        """ Ensure that the indexes are properly created and that
+        """Ensure that the indexes are properly created and that
         compare_indexes identifies the missing/extra indexes for subclassed
         documents (_cls included)
         """
@@ -151,7 +146,7 @@ class TestClassMethods(unittest.TestCase):
         }
 
     def test_compare_indexes_multiple_subclasses(self):
-        """ Ensure that compare_indexes behaves correctly if called from a
+        """Ensure that compare_indexes behaves correctly if called from a
         class, which base class has multiple subclasses
         """
 
@@ -182,7 +177,7 @@ class TestClassMethods(unittest.TestCase):
         assert BlogPostWithCustomField.compare_indexes() == {"missing": [], "extra": []}
 
     def test_compare_indexes_for_text_indexes(self):
-        """ Ensure that compare_indexes behaves correctly for text indexes """
+        """Ensure that compare_indexes behaves correctly for text indexes"""
 
         class Doc(Document):
             a = StringField()
@@ -204,7 +199,7 @@ class TestClassMethods(unittest.TestCase):
         assert actual == expected
 
     def test_list_indexes_inheritance(self):
-        """ ensure that all of the indexes are listed regardless of the super-
+        """ensure that all of the indexes are listed regardless of the super-
         or sub-class that we call it from
         """
 
@@ -236,7 +231,7 @@ class TestClassMethods(unittest.TestCase):
         assert BlogPost.list_indexes() == [
             [("_cls", 1), ("author", 1), ("tags", 1)],
             [("_cls", 1), ("author", 1), ("tags", 1), ("extra_text", 1)],
-            [(u"_id", 1)],
+            [("_id", 1)],
             [("_cls", 1)],
         ]
 
@@ -261,8 +256,7 @@ class TestClassMethods(unittest.TestCase):
         assert Vaccine._meta["delete_rules"][(Cat, "vaccine_made")] == PULL
 
     def test_collection_naming(self):
-        """Ensure that a collection with a specified name may be used.
+        """Ensure that a collection with a specified name may be used."""
-        """
 
         class DefaultNamingTest(Document):
             pass
@@ -294,7 +288,7 @@ class TestClassMethods(unittest.TestCase):
         assert "wibble" == InheritedAbstractNamingTest._get_collection_name()
 
         # Mixin tests
-        class BaseMixin(object):
+        class BaseMixin:
             meta = {"collection": lambda c: c.__name__.lower()}
 
         class OldMixinNamingConvention(Document, BaseMixin):
@@ -305,7 +299,7 @@ class TestClassMethods(unittest.TestCase):
             == OldMixinNamingConvention._get_collection_name()
         )
 
-        class BaseMixin(object):
+        class BaseMixin:
             meta = {"collection": lambda c: c.__name__.lower()}
 
         class BaseDocument(Document, BaseMixin):
@@ -317,8 +311,7 @@ class TestClassMethods(unittest.TestCase):
         assert "basedocument" == MyDocument._get_collection_name()
 
     def test_custom_collection_name_operations(self):
-        """Ensure that a collection with a specified name is used as expected.
+        """Ensure that a collection with a specified name is used as expected."""
-        """
         collection_name = "personCollTest"
 
         class Person(Document):
@@ -338,8 +331,7 @@ class TestClassMethods(unittest.TestCase):
         assert collection_name not in list_collection_names(self.db)
 
     def test_collection_name_and_primary(self):
-        """Ensure that a collection with a specified name may be used.
+        """Ensure that a collection with a specified name may be used."""
-        """
 
         class Person(Document):
             name = StringField(primary_key=True)

tests/document/test_delta.py

@@ -1,7 +1,7 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import SON
+
 from mongoengine import *
 from mongoengine.pymongo_support import list_collection_names
 from tests.utils import MongoDBTestCase
@@ -9,7 +9,7 @@ from tests.utils import MongoDBTestCase
 
 class TestDelta(MongoDBTestCase):
     def setUp(self):
-        super(TestDelta, self).setUp()
+        super().setUp()
 
         class Person(Document):
             name = StringField()
@@ -29,7 +29,8 @@ class TestDelta(MongoDBTestCase):
         self.delta(Document)
         self.delta(DynamicDocument)
 
-    def delta(self, DocClass):
+    @staticmethod
+    def delta(DocClass):
         class Doc(DocClass):
             string_field = StringField()
             int_field = IntField()
@@ -428,13 +429,20 @@ class TestDelta(MongoDBTestCase):
         assert doc.dict_field == {"hello": "world"}
         assert doc.list_field == ["1", 2, {"hello": "world"}]
 
-    def test_delta_recursive_db_field(self):
+    def test_delta_recursive_db_field_on_doc_and_embeddeddoc(self):
         self.delta_recursive_db_field(Document, EmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_doc_and_dynamicembeddeddoc(self):
         self.delta_recursive_db_field(Document, DynamicEmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_dynamicdoc_and_embeddeddoc(self):
         self.delta_recursive_db_field(DynamicDocument, EmbeddedDocument)
+
+    def test_delta_recursive_db_field_on_dynamicdoc_and_dynamicembeddeddoc(self):
         self.delta_recursive_db_field(DynamicDocument, DynamicEmbeddedDocument)
 
-    def delta_recursive_db_field(self, DocClass, EmbeddedClass):
+    @staticmethod
+    def delta_recursive_db_field(DocClass, EmbeddedClass):
         class Embedded(EmbeddedClass):
             string_field = StringField(db_field="db_string_field")
             int_field = IntField(db_field="db_int_field")
@@ -487,6 +495,7 @@ class TestDelta(MongoDBTestCase):
         doc = doc.reload(10)
         assert doc.embedded_field.dict_field == {}
 
+        assert doc._get_changed_fields() == []
         doc.embedded_field.list_field = []
         assert doc._get_changed_fields() == ["db_embedded_field.db_list_field"]
         assert doc.embedded_field._delta() == ({}, {"db_list_field": 1})
@@ -537,6 +546,7 @@ class TestDelta(MongoDBTestCase):
             {},
         )
         doc.save()
+        assert doc._get_changed_fields() == []
         doc = doc.reload(10)
 
         assert doc.embedded_field.list_field[0] == "1"
@@ -634,6 +644,10 @@ class TestDelta(MongoDBTestCase):
         doc.save()
         doc = doc.reload(10)
 
+        assert doc._delta() == (
+            {},
+            {},
+        )
         del doc.embedded_field.list_field[2].list_field
         assert doc._delta() == (
             {},
@@ -732,12 +746,12 @@ class TestDelta(MongoDBTestCase):
         assert organization._get_changed_fields() == []
 
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
-        assert {} == updates
+        assert updates == {}
 
         organization.employees.append(person)
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
         assert "employees" in updates
 
     def test_delta_with_dbref_false(self):
@@ -749,12 +763,12 @@ class TestDelta(MongoDBTestCase):
         assert organization._get_changed_fields() == []
 
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
-        assert {} == updates
+        assert updates == {}
 
         organization.employees.append(person)
         updates, removals = organization._delta()
-        assert {} == removals
+        assert removals == {}
         assert "employees" in updates
 
     def test_nested_nested_fields_mark_as_changed(self):
@@ -767,19 +781,46 @@ class TestDelta(MongoDBTestCase):
 
         MyDoc.drop_collection()
 
-        mydoc = MyDoc(
+        MyDoc(name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}).save()
-            name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}
-        ).save()
 
         mydoc = MyDoc.objects.first()
         subdoc = mydoc.subs["a"]["b"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
-        assert ["subs.a.b.name"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a.b.name"]
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
+
+    def test_nested_nested_fields_db_field_set__gets_mark_as_changed_and_cleaned(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            name = StringField(db_field="db_name")
+
+        class MyDoc(Document):
+            embed = EmbeddedDocumentField(EmbeddedDoc, db_field="db_embed")
+            name = StringField(db_field="db_name")
+
+        MyDoc.drop_collection()
+
+        MyDoc(name="testcase1", embed=EmbeddedDoc(name="foo")).save()
+
+        mydoc = MyDoc.objects.first()
+        mydoc.embed.name = "foo1"
+
+        assert mydoc.embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed.db_name"]
+
+        mydoc = MyDoc.objects.first()
+        embed = EmbeddedDoc(name="foo2")
+        embed.name = "bar"
+        mydoc.embed = embed
+
+        assert embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed"]
+
+        mydoc._clear_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_lower_level_mark_as_changed(self):
         class EmbeddedDoc(EmbeddedDocument):
@@ -794,17 +835,17 @@ class TestDelta(MongoDBTestCase):
 
         mydoc = MyDoc.objects.first()
         mydoc.subs["a"] = EmbeddedDoc()
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a"]
 
         subdoc = mydoc.subs["a"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a"]
         mydoc.save()
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_upper_level_mark_as_changed(self):
         class EmbeddedDoc(EmbeddedDocument):
@@ -821,15 +862,15 @@ class TestDelta(MongoDBTestCase):
         subdoc = mydoc.subs["a"]
         subdoc.name = "bar"
 
-        assert ["name"] == subdoc._get_changed_fields()
+        assert subdoc._get_changed_fields() == ["name"]
-        assert ["subs.a.name"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a.name"]
 
         mydoc.subs["a"] = EmbeddedDoc()
-        assert ["subs.a"] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == ["subs.a"]
         mydoc.save()
 
         mydoc._clear_changed_fields()
-        assert [] == mydoc._get_changed_fields()
+        assert mydoc._get_changed_fields() == []
 
     def test_referenced_object_changed_attributes(self):
         """Ensures that when you save a new reference to a field, the referenced object isn't altered"""

tests/document/test_dynamic.py

@@ -10,7 +10,7 @@ __all__ = ("TestDynamicDocument",)
 
 class TestDynamicDocument(MongoDBTestCase):
     def setUp(self):
-        super(TestDynamicDocument, self).setUp()
+        super().setUp()
 
         class Person(DynamicDocument):
             name = StringField()
@@ -37,6 +37,19 @@ class TestDynamicDocument(MongoDBTestCase):
         # Confirm no changes to self.Person
         assert not hasattr(self.Person, "age")
 
+    def test_dynamic_document_parse_values_in_constructor_like_document_do(self):
+        class ProductDynamicDocument(DynamicDocument):
+            title = StringField()
+            price = FloatField()
+
+        class ProductDocument(Document):
+            title = StringField()
+            price = FloatField()
+
+        product = ProductDocument(title="Blabla", price="12.5")
+        dyn_product = ProductDynamicDocument(title="Blabla", price="12.5")
+        assert product.price == dyn_product.price == 12.5
+
     def test_change_scope_of_variable(self):
         """Test changing the scope of a dynamic field has no adverse effects"""
         p = self.Person()
@@ -105,17 +118,17 @@ class TestDynamicDocument(MongoDBTestCase):
         p.save()
 
         raw_p = Person.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_cls": u"Person", "_id": p.id, "name": u"Dean"}
+        assert raw_p == {"_cls": "Person", "_id": p.id, "name": "Dean"}
 
         p.name = "OldDean"
         p.newattr = "garbage"
         p.save()
         raw_p = Person.objects.as_pymongo().get(id=p.id)
         assert raw_p == {
-            "_cls": u"Person",
+            "_cls": "Person",
             "_id": p.id,
             "name": "OldDean",
-            "newattr": u"garbage",
+            "newattr": "garbage",
         }
 
     def test_fields_containing_underscore(self):
@@ -131,14 +144,14 @@ class TestDynamicDocument(MongoDBTestCase):
         p.save()
 
         raw_p = WeirdPerson.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_id": p.id, "_name": u"Dean", "name": u"Dean"}
+        assert raw_p == {"_id": p.id, "_name": "Dean", "name": "Dean"}
 
         p.name = "OldDean"
         p._name = "NewDean"
         p._newattr1 = "garbage"  # Unknown fields won't be added
         p.save()
         raw_p = WeirdPerson.objects.as_pymongo().get(id=p.id)
-        assert raw_p == {"_id": p.id, "_name": u"NewDean", "name": u"OldDean"}
+        assert raw_p == {"_id": p.id, "_name": "NewDean", "name": "OldDean"}
 
     def test_dynamic_document_queries(self):
         """Ensure we can query dynamic fields"""

tests/document/test_indexes.py

@@ -1,13 +1,16 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 
+import pytest
 from pymongo.collation import Collation
 from pymongo.errors import OperationFailure
-import pytest
 
 from mongoengine import *
 from mongoengine.connection import get_db
+from mongoengine.mongodb_support import (
+    MONGODB_42,
+    get_mongodb_version,
+)
 
 
 class TestIndexes(unittest.TestCase):
@@ -172,8 +175,7 @@ class TestIndexes(unittest.TestCase):
         assert MyDoc._meta["index_specs"] == [{"fields": [("keywords", 1)]}]
 
     def test_embedded_document_index_meta(self):
-        """Ensure that embedded document indexes are created explicitly
+        """Ensure that embedded document indexes are created explicitly"""
-        """
 
         class Rank(EmbeddedDocument):
             title = StringField(required=True)
@@ -195,8 +197,7 @@ class TestIndexes(unittest.TestCase):
         assert [("rank.title", 1)] in info
 
     def test_explicit_geo2d_index(self):
-        """Ensure that geo2d indexes work when created via meta[indexes]
+        """Ensure that geo2d indexes work when created via meta[indexes]"""
-        """
 
         class Place(Document):
             location = DictField()
@@ -210,8 +211,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "2d")] in info
 
     def test_explicit_geo2d_index_embedded(self):
-        """Ensure that geo2d indexes work when created via meta[indexes]
+        """Ensure that geo2d indexes work when created via meta[indexes]"""
-        """
 
         class EmbeddedLocation(EmbeddedDocument):
             location = DictField()
@@ -230,8 +230,7 @@ class TestIndexes(unittest.TestCase):
         assert [("current.location.point", "2d")] in info
 
     def test_explicit_geosphere_index(self):
-        """Ensure that geosphere indexes work when created via meta[indexes]
+        """Ensure that geosphere indexes work when created via meta[indexes]"""
-        """
 
         class Place(Document):
             location = DictField()
@@ -247,8 +246,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "2dsphere")] in info
 
     def test_explicit_geohaystack_index(self):
-        """Ensure that geohaystack indexes work when created via meta[indexes]
+        """Ensure that geohaystack indexes work when created via meta[indexes]"""
-        """
         pytest.skip(
             "GeoHaystack index creation is not supported for now"
             "from meta, as it requires a bucketSize parameter."
@@ -269,8 +267,7 @@ class TestIndexes(unittest.TestCase):
         assert [("location.point", "geoHaystack")] in info
 
     def test_create_geohaystack_index(self):
-        """Ensure that geohaystack indexes can be created
+        """Ensure that geohaystack indexes can be created"""
-        """
 
         class Place(Document):
             location = DictField()
@@ -365,8 +362,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_cls_1_user_guid_1", "_id_"]
 
     def test_embedded_document_index(self):
-        """Tests settings an index on an embedded document
+        """Tests settings an index on an embedded document"""
-        """
 
         class Date(EmbeddedDocument):
             year = IntField(db_field="yr")
@@ -383,8 +379,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_id_", "date.yr_-1"]
 
     def test_list_embedded_document_index(self):
-        """Ensure list embedded documents can be indexed
+        """Ensure list embedded documents can be indexed"""
-        """
 
         class Tag(EmbeddedDocument):
             name = StringField(db_field="tag")
@@ -420,8 +415,7 @@ class TestIndexes(unittest.TestCase):
         assert sorted(info.keys()) == ["_cls_1", "_id_"]
 
     def test_covered_index(self):
-        """Ensure that covered indexes can be used
+        """Ensure that covered indexes can be used"""
-        """
 
         class Test(Document):
             a = IntField()
@@ -462,9 +456,11 @@ class TestIndexes(unittest.TestCase):
             .get("stage")
             == "IXSCAN"
         )
+        mongo_db = get_mongodb_version()
+        PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED"
         assert (
             query_plan.get("queryPlanner").get("winningPlan").get("stage")
-            == "PROJECTION"
+            == PROJECTION_STR
         )
 
         query_plan = Test.objects(a=1).explain()
@@ -551,15 +547,15 @@ class TestIndexes(unittest.TestCase):
         assert 5 == query_result.count()
 
         incorrect_collation = {"arndom": "wrdo"}
-        with pytest.raises(OperationFailure):
+        with pytest.raises(OperationFailure) as exc_info:
             BlogPost.objects.collation(incorrect_collation).count()
+        assert "Missing expected field" in str(exc_info.value)
 
         query_result = BlogPost.objects.collation({}).order_by("name")
         assert [x.name for x in query_result] == sorted(names)
 
     def test_unique(self):
-        """Ensure that uniqueness constraints are applied to fields.
+        """Ensure that uniqueness constraints are applied to fields."""
-        """
 
         class BlogPost(Document):
             title = StringField()
@@ -607,8 +603,7 @@ class TestIndexes(unittest.TestCase):
         )
 
     def test_unique_with(self):
-        """Ensure that unique_with constraints are applied to fields.
+        """Ensure that unique_with constraints are applied to fields."""
-        """
 
         class Date(EmbeddedDocument):
             year = IntField(db_field="yr")
@@ -633,8 +628,7 @@ class TestIndexes(unittest.TestCase):
             post3.save()
 
     def test_unique_embedded_document(self):
-        """Ensure that uniqueness constraints are applied to fields on embedded documents.
+        """Ensure that uniqueness constraints are applied to fields on embedded documents."""
-        """
 
         class SubDocument(EmbeddedDocument):
             year = IntField(db_field="yr")

tests/document/test_inheritance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 import warnings
 
@@ -46,8 +45,7 @@ class TestInheritance(MongoDBTestCase):
         test_doc.delete()
 
     def test_superclasses(self):
-        """Ensure that the correct list of superclasses is assembled.
+        """Ensure that the correct list of superclasses is assembled."""
-        """
 
         class Animal(Document):
             meta = {"allow_inheritance": True}
@@ -217,8 +215,7 @@ class TestInheritance(MongoDBTestCase):
         assert Pike._subclasses == ("Animal.Fish.Pike",)
 
     def test_inheritance_meta_data(self):
-        """Ensure that document may inherit fields from a superclass document.
+        """Ensure that document may inherit fields from a superclass document."""
-        """
 
         class Person(Document):
             name = StringField()
@@ -235,8 +232,7 @@ class TestInheritance(MongoDBTestCase):
         assert Employee._get_collection_name() == Person._get_collection_name()
 
     def test_inheritance_to_mongo_keys(self):
-        """Ensure that document may inherit fields from a superclass document.
+        """Ensure that document may inherit fields from a superclass document."""
-        """
 
         class Person(Document):
             name = StringField()
@@ -260,7 +256,7 @@ class TestInheritance(MongoDBTestCase):
         assert Employee._get_collection_name() == Person._get_collection_name()
 
     def test_indexes_and_multiple_inheritance(self):
-        """ Ensure that all of the indexes are created for a document with
+        """Ensure that all of the indexes are created for a document with
         multiple inheritance.
         """
 
@@ -284,14 +280,11 @@ class TestInheritance(MongoDBTestCase):
         C.ensure_indexes()
 
         assert sorted(
-            [idx["key"] for idx in C._get_collection().index_information().values()]
+            idx["key"] for idx in C._get_collection().index_information().values()
-        ) == sorted(
+        ) == sorted([[("_cls", 1), ("b", 1)], [("_id", 1)], [("_cls", 1), ("a", 1)]])
-            [[(u"_cls", 1), (u"b", 1)], [(u"_id", 1)], [(u"_cls", 1), (u"a", 1)]]
-        )
 
     def test_polymorphic_queries(self):
-        """Ensure that the correct subclasses are returned from a query
+        """Ensure that the correct subclasses are returned from a query"""
-        """
 
         class Animal(Document):
             meta = {"allow_inheritance": True}
@@ -348,8 +341,7 @@ class TestInheritance(MongoDBTestCase):
         assert "_cls" not in obj
 
     def test_cant_turn_off_inheritance_on_subclass(self):
-        """Ensure if inheritance is on in a subclass you cant turn it off.
+        """Ensure if inheritance is on in a subclass you cant turn it off."""
-        """
 
         class Animal(Document):
             name = StringField()
@@ -475,7 +467,7 @@ class TestInheritance(MongoDBTestCase):
         assert city.pk is None
         # TODO: expected error? Shouldn't we create a new error type?
         with pytest.raises(KeyError):
-            setattr(city, "pk", 1)
+            city.pk = 1
 
     def test_allow_inheritance_embedded_document(self):
         """Ensure embedded documents respect inheritance."""
@@ -499,8 +491,7 @@ class TestInheritance(MongoDBTestCase):
         assert "_cls" in doc.to_mongo()
 
     def test_document_inheritance(self):
-        """Ensure mutliple inheritance of abstract documents
+        """Ensure mutliple inheritance of abstract documents"""
-        """
 
         class DateCreatedDocument(Document):
             meta = {"allow_inheritance": True, "abstract": True}
@@ -508,14 +499,9 @@ class TestInheritance(MongoDBTestCase):
         class DateUpdatedDocument(Document):
             meta = {"allow_inheritance": True, "abstract": True}
 
-        try:
-
         class MyDocument(DateCreatedDocument, DateUpdatedDocument):
             pass
 
-        except Exception:
-            assert False, "Couldn't create MyDocument class"
-
     def test_abstract_documents(self):
         """Ensure that a document superclass can be marked as abstract
         thereby not using it as the name for the collection."""

tests/document/test_instance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import pickle
 import unittest
@@ -7,9 +6,9 @@ import weakref
 from datetime import datetime
 
 import bson
+import pytest
 from bson import DBRef, ObjectId
 from pymongo.errors import DuplicateKeyError
-import pytest
 
 from mongoengine import *
 from mongoengine import signals
@@ -24,7 +23,11 @@ from mongoengine.errors import (
     NotUniqueError,
     SaveConditionError,
 )
-from mongoengine.mongodb_support import MONGODB_34, MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_34,
+    MONGODB_36,
+    get_mongodb_version,
+)
 from mongoengine.pymongo_support import list_collection_names
 from mongoengine.queryset import NULLIFY, Q
 from tests import fixtures
@@ -62,12 +65,12 @@ class TestDocumentInstance(MongoDBTestCase):
         for collection in list_collection_names(self.db):
             self.db.drop_collection(collection)
 
-    def assertDbEqual(self, docs):
+    def _assert_db_equal(self, docs):
         assert list(self.Person._get_collection().find().sort("id")) == sorted(
             docs, key=lambda doc: doc["_id"]
         )
 
-    def assertHasInstance(self, field, instance):
+    def _assert_has_instance(self, field, instance):
         assert hasattr(field, "_instance")
         assert field._instance is not None
         if isinstance(field._instance, weakref.ProxyType):
@@ -161,8 +164,7 @@ class TestDocumentInstance(MongoDBTestCase):
         Log.objects
 
     def test_repr(self):
-        """Ensure that unicode representation works
+        """Ensure that unicode representation works"""
-        """
 
         class Article(Document):
             title = StringField()
@@ -170,7 +172,7 @@ class TestDocumentInstance(MongoDBTestCase):
             def __unicode__(self):
                 return self.title
 
-        doc = Article(title=u"привет мир")
+        doc = Article(title="привет мир")
 
         assert "<Article: привет мир>" == repr(doc)
 
@@ -183,12 +185,12 @@ class TestDocumentInstance(MongoDBTestCase):
             def __str__(self):
                 return None
 
-        doc = Article(title=u"привет мир")
+        doc = Article(title="привет мир")
         assert "<Article: None>" == repr(doc)
 
     def test_queryset_resurrects_dropped_collection(self):
         self.Person.drop_collection()
-        assert [] == list(self.Person.objects())
+        assert list(self.Person.objects()) == []
 
         # Ensure works correctly with inhertited classes
         class Actor(self.Person):
@@ -196,7 +198,7 @@ class TestDocumentInstance(MongoDBTestCase):
 
         Actor.objects()
         self.Person.drop_collection()
-        assert [] == list(Actor.objects())
+        assert list(Actor.objects()) == []
 
     def test_polymorphic_references(self):
         """Ensure that the correct subclasses are returned from a query
@@ -405,6 +407,16 @@ class TestDocumentInstance(MongoDBTestCase):
         assert person.name == "Test User"
         assert person.age == 30
 
+    def test__qs_property_does_not_raise(self):
+        # ensures no regression of #2500
+        class MyDocument(Document):
+            pass
+
+        MyDocument.drop_collection()
+        object = MyDocument()
+        object._qs().insert([MyDocument()])
+        assert MyDocument.objects.count() == 1
+
     def test_to_dbref(self):
         """Ensure that you can get a dbref of a document."""
         person = self.Person(name="Test User", age=30)
@@ -501,7 +513,7 @@ class TestDocumentInstance(MongoDBTestCase):
         doc.reload()
         Animal.drop_collection()
 
-    def test_update_shard_key_routing(self):
+    def test_save_update_shard_key_routing(self):
         """Ensures updating a doc with a specified shard_key includes it in
         the query.
         """
@@ -523,9 +535,32 @@ class TestDocumentInstance(MongoDBTestCase):
             query_op = q.db.system.profile.find({"ns": "mongoenginetest.animal"})[0]
             assert query_op["op"] == "update"
             if mongo_db <= MONGODB_34:
-                assert set(query_op["query"].keys()) == set(["_id", "is_mammal"])
+                assert set(query_op["query"].keys()) == {"_id", "is_mammal"}
             else:
-                assert set(query_op["command"]["q"].keys()) == set(["_id", "is_mammal"])
+                assert set(query_op["command"]["q"].keys()) == {"_id", "is_mammal"}
+
+        Animal.drop_collection()
+
+    def test_save_create_shard_key_routing(self):
+        """Ensures inserting a doc with a specified shard_key includes it in
+        the query.
+        """
+
+        class Animal(Document):
+            _id = UUIDField(binary=False, primary_key=True, default=uuid.uuid4)
+            is_mammal = BooleanField()
+            name = StringField()
+            meta = {"shard_key": ("is_mammal",)}
+
+        Animal.drop_collection()
+        doc = Animal(is_mammal=True, name="Dog")
+
+        with query_counter() as q:
+            doc.save()
+            query_op = q.db.system.profile.find({"ns": "mongoenginetest.animal"})[0]
+            assert query_op["op"] == "command"
+            assert query_op["command"]["findAndModify"] == "animal"
+            assert set(query_op["command"]["query"].keys()) == {"_id", "is_mammal"}
 
         Animal.drop_collection()
 
@@ -578,7 +613,8 @@ class TestDocumentInstance(MongoDBTestCase):
         doc.embedded_field.list_field.append(1)
         doc.embedded_field.dict_field["woot"] = "woot"
 
-        assert doc._get_changed_fields() == [
+        changed = doc._get_changed_fields()
+        assert changed == [
             "list_field",
             "dict_field.woot",
             "embedded_field.list_field",
@@ -704,11 +740,11 @@ class TestDocumentInstance(MongoDBTestCase):
         Doc.drop_collection()
 
         doc = Doc(embedded_field=Embedded(string="Hi"))
-        self.assertHasInstance(doc.embedded_field, doc)
+        self._assert_has_instance(doc.embedded_field, doc)
 
         doc.save()
         doc = Doc.objects.get()
-        self.assertHasInstance(doc.embedded_field, doc)
+        self._assert_has_instance(doc.embedded_field, doc)
 
     def test_embedded_document_complex_instance(self):
         """Ensure that embedded documents in complex fields can reference
@@ -723,11 +759,11 @@ class TestDocumentInstance(MongoDBTestCase):
 
         Doc.drop_collection()
         doc = Doc(embedded_field=[Embedded(string="Hi")])
-        self.assertHasInstance(doc.embedded_field[0], doc)
+        self._assert_has_instance(doc.embedded_field[0], doc)
 
         doc.save()
         doc = Doc.objects.get()
-        self.assertHasInstance(doc.embedded_field[0], doc)
+        self._assert_has_instance(doc.embedded_field[0], doc)
 
     def test_embedded_document_complex_instance_no_use_db_field(self):
         """Ensure that use_db_field is propagated to list of Emb Docs."""
@@ -756,11 +792,11 @@ class TestDocumentInstance(MongoDBTestCase):
 
         acc = Account()
         acc.email = Email(email="test@example.com")
-        self.assertHasInstance(acc._data["email"], acc)
+        self._assert_has_instance(acc._data["email"], acc)
         acc.save()
 
         acc1 = Account.objects.first()
-        self.assertHasInstance(acc1._data["email"], acc1)
+        self._assert_has_instance(acc1._data["email"], acc1)
 
     def test_instance_is_set_on_setattr_on_embedded_document_list(self):
         class Email(EmbeddedDocument):
@@ -772,11 +808,11 @@ class TestDocumentInstance(MongoDBTestCase):
         Account.drop_collection()
         acc = Account()
         acc.emails = [Email(email="test@example.com")]
-        self.assertHasInstance(acc._data["emails"][0], acc)
+        self._assert_has_instance(acc._data["emails"][0], acc)
         acc.save()
 
         acc1 = Account.objects.first()
-        self.assertHasInstance(acc1._data["emails"][0], acc1)
+        self._assert_has_instance(acc1._data["emails"][0], acc1)
 
     def test_save_checks_that_clean_is_called(self):
         class CustomError(Exception):
@@ -885,7 +921,7 @@ class TestDocumentInstance(MongoDBTestCase):
         with pytest.raises(InvalidDocumentError):
             self.Person().modify(set__age=10)
 
-        self.assertDbEqual([dict(doc.to_mongo())])
+        self._assert_db_equal([dict(doc.to_mongo())])
 
     def test_modify_invalid_query(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -895,7 +931,7 @@ class TestDocumentInstance(MongoDBTestCase):
         with pytest.raises(InvalidQueryError):
             doc1.modify({"id": doc2.id}, set__value=20)
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_match_another_document(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -905,7 +941,7 @@ class TestDocumentInstance(MongoDBTestCase):
         n_modified = doc1.modify({"name": doc2.name}, set__age=100)
         assert n_modified == 0
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_not_exists(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -915,7 +951,7 @@ class TestDocumentInstance(MongoDBTestCase):
         n_modified = doc2.modify({"name": doc2.name}, set__age=100)
         assert n_modified == 0
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_update(self):
         other_doc = self.Person(name="bob", age=10).save()
@@ -941,7 +977,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert doc.to_json() == doc_copy.to_json()
         assert doc._get_changed_fields() == []
 
-        self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
+        self._assert_db_equal([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
     def test_modify_with_positional_push(self):
         class Content(EmbeddedDocument):
@@ -1406,11 +1442,11 @@ class TestDocumentInstance(MongoDBTestCase):
         coll = self.Person._get_collection()
         doc = self.Person(name="John").save()
         raw_doc = coll.find_one({"_id": doc.pk})
-        assert set(raw_doc.keys()) == set(["_id", "_cls", "name"])
+        assert set(raw_doc.keys()) == {"_id", "_cls", "name"}
 
         doc.update(rename__name="first_name")
         raw_doc = coll.find_one({"_id": doc.pk})
-        assert set(raw_doc.keys()) == set(["_id", "_cls", "first_name"])
+        assert set(raw_doc.keys()) == {"_id", "_cls", "first_name"}
         assert raw_doc["first_name"] == "John"
 
     def test_inserts_if_you_set_the_pk(self):
@@ -1530,8 +1566,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert site.page.log_message == "Error: Dummy message"
 
     def test_update_list_field(self):
-        """Test update on `ListField` with $pull + $in.
+        """Test update on `ListField` with $pull + $in."""
-        """
 
         class Doc(Document):
             foo = ListField(StringField())
@@ -2020,7 +2055,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert promoted_employee.details is None
 
     def test_object_mixins(self):
-        class NameMixin(object):
+        class NameMixin:
             name = StringField()
 
         class Foo(EmbeddedDocument, NameMixin):
@@ -2034,7 +2069,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert ["id", "name", "widgets"] == sorted(Bar._fields.keys())
 
     def test_mixin_inheritance(self):
-        class BaseMixIn(object):
+        class BaseMixIn:
             count = IntField()
             data = StringField()
 
@@ -2793,15 +2828,13 @@ class TestDocumentInstance(MongoDBTestCase):
         register_connection("testdb-2", "mongoenginetest2")
 
         class A(Document):
-            """Uses default db_alias
+            """Uses default db_alias"""
-            """
 
             name = StringField()
             meta = {"allow_inheritance": True}
 
         class B(A):
-            """Uses testdb-2 db_alias
+            """Uses testdb-2 db_alias"""
-            """
 
             meta = {"db_alias": "testdb-2"}
 
@@ -2881,39 +2914,23 @@ class TestDocumentInstance(MongoDBTestCase):
         # Checks
         assert ",".join([str(b) for b in Book.objects.all()]) == "1,2,3,4,5,6,7,8,9"
         # bob related books
-        assert (
+        bob_books_qs = Book.objects.filter(
-            ",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
             Q(extra__a=bob) | Q(author=bob) | Q(extra__b=bob)
         )
-                ]
+        assert [str(b) for b in bob_books_qs] == ["1", "2", "3", "4"]
-            )
+        assert bob_books_qs.count() == 4
-            == "1,2,3,4"
-        )
 
         # Susan & Karl related books
-        assert (
+        susan_karl_books_qs = Book.objects.filter(
-            ",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
             Q(extra__a__all=[karl, susan])
             | Q(author__all=[karl, susan])
             | Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()])
         )
-                ]
+        assert [str(b) for b in susan_karl_books_qs] == ["1"]
-            )
+        assert susan_karl_books_qs.count() == 1
-            == "1"
-        )
 
         # $Where
-        assert (
+        custom_qs = Book.objects.filter(
-            u",".join(
-                [
-                    str(b)
-                    for b in Book.objects.filter(
             __raw__={
                 "$where": """
                                             function(){
@@ -2921,10 +2938,8 @@ class TestDocumentInstance(MongoDBTestCase):
                                                        this.name == '2';}"""
             }
         )
-                ]
+        assert [str(b) for b in custom_qs] == ["1", "2"]
-            )
+        assert custom_qs.count() == 2
-            == "1,2"
-        )
 
     def test_switch_db_instance(self):
         register_connection("testdb-1", "mongoenginetest2")
@@ -3284,7 +3299,7 @@ class TestDocumentInstance(MongoDBTestCase):
                 for node_name, node in self.nodes.items():
                     node.expand()
                     node.save(*args, **kwargs)
-                super(NodesSystem, self).save(*args, **kwargs)
+                super().save(*args, **kwargs)
 
         NodesSystem.drop_collection()
         Node.drop_collection()
@@ -3411,7 +3426,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert obj3 != dbref2
         assert dbref2 != obj3
 
-    def test_default_values(self):
+    def test_default_values_dont_get_override_upon_save_when_only_is_used(self):
         class Person(Document):
             created_on = DateTimeField(default=lambda: datetime.utcnow())
             name = StringField()
@@ -3589,8 +3604,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert u_from_db.height is None
 
     def test_not_saved_eq(self):
-        """Ensure we can compare documents not saved.
+        """Ensure we can compare documents not saved."""
-        """
 
         class Person(Document):
             pass
@@ -3734,7 +3748,7 @@ class TestDocumentInstance(MongoDBTestCase):
         _ = list(Jedi.objects)  # Ensure a proper document loads without errors
 
         # Forces a document with a wrong shape (may occur in case of migration)
-        value = u"I_should_be_a_dict"
+        value = "I_should_be_a_dict"
         coll.insert_one({"light_saber": value})
 
         with pytest.raises(InvalidDocumentError) as exc_info:
@@ -3799,5 +3813,95 @@ class ObjectKeyTestCase(MongoDBTestCase):
         assert book._object_key == {"pk": book.pk, "author__name": "Author"}
 
 
+class DBFieldMappingTest(MongoDBTestCase):
+    def setUp(self):
+        class Fields:
+            w1 = BooleanField(db_field="w2")
+
+            x1 = BooleanField(db_field="x2")
+            x2 = BooleanField(db_field="x3")
+
+            y1 = BooleanField(db_field="y0")
+            y2 = BooleanField(db_field="y1")
+
+            z1 = BooleanField(db_field="z2")
+            z2 = BooleanField(db_field="z1")
+
+        class Doc(Fields, Document):
+            pass
+
+        class DynDoc(Fields, DynamicDocument):
+            pass
+
+        self.Doc = Doc
+        self.DynDoc = DynDoc
+
+    def tearDown(self):
+        for collection in list_collection_names(self.db):
+            self.db.drop_collection(collection)
+
+    def test_setting_fields_in_constructor_of_strict_doc_uses_model_names(self):
+        doc = self.Doc(z1=True, z2=False)
+        assert doc.z1 is True
+        assert doc.z2 is False
+
+    def test_setting_fields_in_constructor_of_dyn_doc_uses_model_names(self):
+        doc = self.DynDoc(z1=True, z2=False)
+        assert doc.z1 is True
+        assert doc.z2 is False
+
+    def test_setting_unknown_field_in_constructor_of_dyn_doc_does_not_overwrite_model_fields(
+        self,
+    ):
+        doc = self.DynDoc(w2=True)
+        assert doc.w1 is None
+        assert doc.w2 is True
+
+    def test_unknown_fields_of_strict_doc_do_not_overwrite_dbfields_1(self):
+        doc = self.Doc()
+        doc.w2 = True
+        doc.x3 = True
+        doc.y0 = True
+        doc.save()
+        reloaded = self.Doc.objects.get(id=doc.id)
+        assert reloaded.w1 is None
+        assert reloaded.x1 is None
+        assert reloaded.x2 is None
+        assert reloaded.y1 is None
+        assert reloaded.y2 is None
+
+    def test_dbfields_are_loaded_to_the_right_modelfield_for_strict_doc_2(self):
+        doc = self.Doc()
+        doc.x2 = True
+        doc.y2 = True
+        doc.z2 = True
+        doc.save()
+        reloaded = self.Doc.objects.get(id=doc.id)
+        assert (
+            reloaded.x1,
+            reloaded.x2,
+            reloaded.y1,
+            reloaded.y2,
+            reloaded.z1,
+            reloaded.z2,
+        ) == (doc.x1, doc.x2, doc.y1, doc.y2, doc.z1, doc.z2)
+
+    def test_dbfields_are_loaded_to_the_right_modelfield_for_dyn_doc_2(self):
+        doc = self.DynDoc()
+        doc.x2 = True
+        doc.y2 = True
+        doc.z2 = True
+        doc.save()
+        reloaded = self.DynDoc.objects.get(id=doc.id)
+        assert (
+            reloaded.x1,
+            reloaded.x2,
+            reloaded.y1,
+            reloaded.y2,
+            reloaded.z1,
+            reloaded.z2,
+        ) == (doc.x1, doc.x2, doc.y1, doc.y2, doc.z1, doc.z2)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/document/test_json_serialisation.py

@@ -1,7 +1,7 @@
 import unittest
 import uuid
-
 from datetime import datetime
+
 from bson import ObjectId
 
 from mongoengine import *

tests/document/test_validation.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 
@@ -10,8 +9,7 @@ from tests.utils import MongoDBTestCase
 
 class TestValidatorError(MongoDBTestCase):
     def test_to_dict(self):
-        """Ensure a ValidationError handles error to_dict correctly.
+        """Ensure a ValidationError handles error to_dict correctly."""
-        """
         error = ValidationError("root")
         assert error.to_dict() == {}
 
@@ -91,8 +89,7 @@ class TestValidatorError(MongoDBTestCase):
             p.validate()
 
     def test_embedded_document_validation(self):
-        """Ensure that embedded documents may be validated.
+        """Ensure that embedded documents may be validated."""
-        """
 
         class Comment(EmbeddedDocument):
             date = DateTimeField()
@@ -213,10 +210,7 @@ class TestValidatorError(MongoDBTestCase):
         child.reference = parent
 
         # Saving the child should not raise a ValidationError
-        try:
         child.save()
-        except ValidationError as e:
-            self.fail("ValidationError raised: %s" % e.message)
 
 
 if __name__ == "__main__":

tests/fields/test_binary_field.py

@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
 import uuid
 
-from bson import Binary
 import pytest
+from bson import Binary
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase
@@ -14,14 +13,13 @@ BIN_VALUE = "\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde
 
 class TestBinaryField(MongoDBTestCase):
     def test_binary_fields(self):
-        """Ensure that binary fields can be stored and retrieved.
+        """Ensure that binary fields can be stored and retrieved."""
-        """
 
         class Attachment(Document):
             content_type = StringField()
             blob = BinaryField()
 
-        BLOB = "\xe6\x00\xc4\xff\x07".encode("latin-1")
+        BLOB = b"\xe6\x00\xc4\xff\x07"
         MIME_TYPE = "application/octet-stream"
 
         Attachment.drop_collection()
@@ -34,8 +32,7 @@ class TestBinaryField(MongoDBTestCase):
         assert BLOB == bytes(attachment_1.blob)
 
     def test_validation_succeeds(self):
-        """Ensure that valid values can be assigned to binary fields.
+        """Ensure that valid values can be assigned to binary fields."""
-        """
 
         class AttachmentRequired(Document):
             blob = BinaryField(required=True)
@@ -46,11 +43,11 @@ class TestBinaryField(MongoDBTestCase):
         attachment_required = AttachmentRequired()
         with pytest.raises(ValidationError):
             attachment_required.validate()
-        attachment_required.blob = Binary("\xe6\x00\xc4\xff\x07".encode("latin-1"))
+        attachment_required.blob = Binary(b"\xe6\x00\xc4\xff\x07")
         attachment_required.validate()
 
-        _5_BYTES = "\xe6\x00\xc4\xff\x07".encode("latin-1")
+        _5_BYTES = b"\xe6\x00\xc4\xff\x07"
-        _4_BYTES = "\xe6\x00\xc4\xff".encode("latin-1")
+        _4_BYTES = b"\xe6\x00\xc4\xff"
         with pytest.raises(ValidationError):
             AttachmentSizeLimit(blob=_5_BYTES).validate()
         AttachmentSizeLimit(blob=_4_BYTES).validate()
@@ -61,7 +58,7 @@ class TestBinaryField(MongoDBTestCase):
         class Attachment(Document):
             blob = BinaryField()
 
-        for invalid_data in (2, u"Im_a_unicode", ["some_str"]):
+        for invalid_data in (2, "Im_a_unicode", ["some_str"]):
             with pytest.raises(ValidationError):
                 Attachment(blob=invalid_data).validate()
 
@@ -132,7 +129,7 @@ class TestBinaryField(MongoDBTestCase):
 
         MyDocument.drop_collection()
 
-        bin_data = "\xe6\x00\xc4\xff\x07".encode("latin-1")
+        bin_data = b"\xe6\x00\xc4\xff\x07"
         doc = MyDocument(bin_field=bin_data).save()
 
         n_updated = MyDocument.objects(bin_field=bin_data).update_one(

tests/fields/test_boolean_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
@@ -14,6 +13,17 @@ class TestBooleanField(MongoDBTestCase):
         person.save()
         assert get_as_pymongo(person) == {"_id": person.id, "admin": True}
 
+    def test_construction_does_not_fail_uncastable_value(self):
+        class BoolFail:
+            def __bool__(self):
+                return "bogus"
+
+        class Person(Document):
+            admin = BooleanField()
+
+        person = Person(admin=BoolFail())
+        person.admin == "bogus"
+
     def test_validation(self):
         """Ensure that invalid values cannot be assigned to boolean
         fields.

tests/fields/test_cached_reference_field.py

@@ -1,13 +1,29 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest
 
-from mongoengine import *
+from mongoengine import (
+    CachedReferenceField,
+    DecimalField,
+    Document,
+    EmbeddedDocument,
+    EmbeddedDocumentField,
+    InvalidDocumentError,
+    ListField,
+    ReferenceField,
+    StringField,
+    ValidationError,
+)
 from tests.utils import MongoDBTestCase
 
 
 class TestCachedReferenceField(MongoDBTestCase):
+    def test_constructor_fail_bad_document_type(self):
+        with pytest.raises(
+            ValidationError, match="must be a document class or a string"
+        ):
+            CachedReferenceField(document_type=0)
+
     def test_get_and_save(self):
         """
         Tests #1047: CachedReferenceField creates DBRefs on to_python,
@@ -191,9 +207,9 @@ class TestCachedReferenceField(MongoDBTestCase):
 
         assert dict(a2.to_mongo()) == {
             "_id": a2.pk,
-            "name": u"Wilson Junior",
+            "name": "Wilson Junior",
-            "tp": u"pf",
+            "tp": "pf",
-            "father": {"_id": a1.pk, "tp": u"pj"},
+            "father": {"_id": a1.pk, "tp": "pj"},
         }
 
         assert Person.objects(father=a1)._query == {"father._id": a1.pk}
@@ -205,9 +221,9 @@ class TestCachedReferenceField(MongoDBTestCase):
         a2.reload()
         assert dict(a2.to_mongo()) == {
             "_id": a2.pk,
-            "name": u"Wilson Junior",
+            "name": "Wilson Junior",
-            "tp": u"pf",
+            "tp": "pf",
-            "father": {"_id": a1.pk, "tp": u"pf"},
+            "father": {"_id": a1.pk, "tp": "pf"},
         }
 
     def test_cached_reference_fields_on_embedded_documents(self):

tests/fields/test_complex_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import itertools
 import math
@@ -7,7 +6,6 @@ import re
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
@@ -61,7 +59,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase):
             assert log == log1
 
         # Test string padding
-        microsecond = map(int, [math.pow(10, x) for x in range(6)])
+        microsecond = map(int, (math.pow(10, x) for x in range(6)))
         mm = dd = hh = ii = ss = [1, 10]
 
         for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond):

tests/fields/test_date_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest

tests/fields/test_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime as dt
 
 import pytest
@@ -10,7 +9,6 @@ except ImportError:
 
 from mongoengine import *
 from mongoengine import connection
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_decimal_field.py

@@ -1,62 +1,12 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest
 
-from mongoengine import *
+from mongoengine import DecimalField, Document, ValidationError
 from tests.utils import MongoDBTestCase
 
 
 class TestDecimalField(MongoDBTestCase):
-    def test_validation(self):
-        """Ensure that invalid values cannot be assigned to decimal fields.
-        """
-
-        class Person(Document):
-            height = DecimalField(min_value=Decimal("0.1"), max_value=Decimal("3.5"))
-
-        Person.drop_collection()
-
-        Person(height=Decimal("1.89")).save()
-        person = Person.objects.first()
-        assert person.height == Decimal("1.89")
-
-        person.height = "2.0"
-        person.save()
-        person.height = 0.01
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = Decimal("0.01")
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = Decimal("4.0")
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = "something invalid"
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person_2 = Person(height="something invalid")
-        with pytest.raises(ValidationError):
-            person_2.validate()
-
-    def test_comparison(self):
-        class Person(Document):
-            money = DecimalField()
-
-        Person.drop_collection()
-
-        Person(money=6).save()
-        Person(money=7).save()
-        Person(money=8).save()
-        Person(money=10).save()
-
-        assert 2 == Person.objects(money__gt=Decimal("7")).count()
-        assert 2 == Person.objects(money__gt=7).count()
-        assert 2 == Person.objects(money__gt="7").count()
-
-        assert 3 == Person.objects(money__gte="7").count()
-
     def test_storage(self):
         class Person(Document):
             float_value = DecimalField(precision=4)
@@ -108,3 +58,83 @@ class TestDecimalField(MongoDBTestCase):
         for field_name in ["float_value", "string_value"]:
             actual = list(Person.objects().scalar(field_name))
             assert expected == actual
+
+    def test_save_none(self):
+        class Person(Document):
+            value = DecimalField()
+
+        Person.drop_collection()
+
+        person = Person(value=None)
+        assert person.value is None
+        person.save()
+        fetched_person = Person.objects.first()
+        fetched_person.value is None
+
+    def test_validation(self):
+        """Ensure that invalid values cannot be assigned to decimal fields."""
+
+        class Person(Document):
+            height = DecimalField(min_value=Decimal("0.1"), max_value=Decimal("3.5"))
+
+        Person.drop_collection()
+
+        Person(height=Decimal("1.89")).save()
+        person = Person.objects.first()
+        assert person.height == Decimal("1.89")
+
+        person.height = "2.0"
+        person.save()
+        person.height = 0.01
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = Decimal("0.01")
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = Decimal("4.0")
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = "something invalid"
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person_2 = Person(height="something invalid")
+        with pytest.raises(ValidationError):
+            person_2.validate()
+
+    def test_comparison(self):
+        class Person(Document):
+            money = DecimalField()
+
+        Person.drop_collection()
+
+        Person(money=6).save()
+        Person(money=7).save()
+        Person(money=8).save()
+        Person(money=10).save()
+
+        assert 2 == Person.objects(money__gt=Decimal("7")).count()
+        assert 2 == Person.objects(money__gt=7).count()
+        assert 2 == Person.objects(money__gt="7").count()
+
+        assert 3 == Person.objects(money__gte="7").count()
+
+    def test_precision_0(self):
+        """prevent regression of a bug that was raising an exception when using precision=0"""
+
+        class TestDoc(Document):
+            d = DecimalField(precision=0)
+
+        TestDoc.drop_collection()
+
+        td = TestDoc(d=Decimal("12.00032678131263"))
+        assert td.d == Decimal("12")
+
+    def test_precision_negative_raise(self):
+        """prevent regression of a bug that was raising an exception when using precision=0"""
+        with pytest.raises(
+            ValidationError, match="precision must be a positive integer"
+        ):
+
+            class TestDoc(Document):
+                dneg = DecimalField(precision=-1)

tests/fields/test_dict_field.py

@@ -1,11 +1,12 @@
-# -*- coding: utf-8 -*-
-from bson import InvalidDocument
 import pytest
+from bson import InvalidDocument
 
 from mongoengine import *
 from mongoengine.base import BaseDict
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
-
+    MONGODB_36,
+    get_mongodb_version,
+)
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
 
@@ -113,7 +114,7 @@ class TestDictField(MongoDBTestCase):
         post.info.setdefault("authors", [])
         post.save()
         post.reload()
-        assert [] == post.info["authors"]
+        assert post.info["authors"] == []
 
     def test_dictfield_dump_document(self):
         """Ensure a DictField can handle another document's dump."""

tests/fields/test_email_field.py

@@ -1,9 +1,6 @@
-# -*- coding: utf-8 -*-
-import sys
-
 import pytest
 
-from mongoengine import *
+from mongoengine import Document, EmailField, ValidationError
 from tests.utils import MongoDBTestCase
 
 
@@ -31,11 +28,11 @@ class TestEmailField(MongoDBTestCase):
             user.validate()
 
         # unicode domain
-        user = User(email=u"user@пример.рф")
+        user = User(email="user@пример.рф")
         user.validate()
 
         # invalid unicode domain
-        user = User(email=u"user@пример")
+        user = User(email="user@пример")
         with pytest.raises(ValidationError):
             user.validate()
 
@@ -49,7 +46,7 @@ class TestEmailField(MongoDBTestCase):
             email = EmailField()
 
         # unicode user shouldn't validate by default...
-        user = User(email=u"Dörte@Sörensen.example.com")
+        user = User(email="Dörte@Sörensen.example.com")
         with pytest.raises(ValidationError):
             user.validate()
 
@@ -57,7 +54,7 @@ class TestEmailField(MongoDBTestCase):
         class User(Document):
             email = EmailField(allow_utf8_user=True)
 
-        user = User(email=u"Dörte@Sörensen.example.com")
+        user = User(email="Dörte@Sörensen.example.com")
         user.validate()
 
     def test_email_field_domain_whitelist(self):

tests/fields/test_embedded_document_field.py

@@ -1,5 +1,7 @@
-# -*- coding: utf-8 -*-
+from copy import deepcopy
+
 import pytest
+from bson import ObjectId
 
 from mongoengine import (
     Document,
@@ -10,10 +12,10 @@ from mongoengine import (
     InvalidQueryError,
     ListField,
     LookUpError,
+    MapField,
     StringField,
     ValidationError,
 )
-
 from tests.utils import MongoDBTestCase
 
 
@@ -75,7 +77,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             Person.objects(settings__notexist="bar").first()
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         with pytest.raises(LookUpError):
             Person.objects.only("settings.notexist")
@@ -111,7 +113,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             assert Person.objects(settings__notexist="bar").first().id == p.id
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         # Test existing attribute
         assert Person.objects(settings__base_foo="basefoo").first().id == p.id
@@ -319,7 +321,7 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             Person.objects(settings__notexist="bar").first()
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         with pytest.raises(LookUpError):
             Person.objects.only("settings.notexist")
@@ -347,8 +349,35 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase):
         # Test non exiting attribute
         with pytest.raises(InvalidQueryError) as exc_info:
             assert Person.objects(settings__notexist="bar").first().id == p.id
-        assert str(exc_info.value) == u'Cannot resolve field "notexist"'
+        assert str(exc_info.value) == 'Cannot resolve field "notexist"'
 
         # Test existing attribute
         assert Person.objects(settings__base_foo="basefoo").first().id == p.id
         assert Person.objects(settings__sub_foo="subfoo").first().id == p.id
+
+    def test_deepcopy_set__instance(self):
+        """Ensure that the _instance attribute on EmbeddedDocument exists after a deepcopy"""
+
+        class Wallet(EmbeddedDocument):
+            money = IntField()
+
+        class Person(Document):
+            wallet = EmbeddedDocumentField(Wallet)
+            wallet_map = MapField(EmbeddedDocumentField(Wallet))
+
+        # Test on fresh EmbeddedDoc
+        emb_doc = Wallet(money=1)
+        assert emb_doc._instance is None
+        copied_emb_doc = deepcopy(emb_doc)
+        assert copied_emb_doc._instance is None
+
+        # Test on attached EmbeddedDoc
+        doc = Person(
+            id=ObjectId(), wallet=Wallet(money=2), wallet_map={"test": Wallet(money=2)}
+        )
+        assert doc.wallet._instance == doc
+        copied_emb_doc = deepcopy(doc.wallet)
+        assert copied_emb_doc._instance is None
+
+        copied_map_emb_doc = deepcopy(doc.wallet_map)
+        assert copied_map_emb_doc["test"]._instance is None

tests/fields/test_enum_field.py

@@ -0,0 +1,145 @@
+from enum import Enum
+
+import pytest
+from bson import InvalidDocument
+
+from mongoengine import Document, EnumField, ValidationError
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class Status(Enum):
+    NEW = "new"
+    DONE = "done"
+
+
+class Color(Enum):
+    RED = 1
+    BLUE = 2
+
+
+class ModelWithEnum(Document):
+    status = EnumField(Status)
+
+
+class TestStringEnumField(MongoDBTestCase):
+    def test_storage(self):
+        model = ModelWithEnum(status=Status.NEW).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "status": "new"}
+
+    def test_set_enum(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=Status.NEW).save()
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_set_by_value(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_filter(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects(status="new").count() == 1
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects(status=Status.DONE).count() == 0
+
+    def test_change_value(self):
+        m = ModelWithEnum(status="new")
+        m.status = Status.DONE
+        m.save()
+        assert m.status == Status.DONE
+
+        m.status = "wrong"
+        assert m.status == "wrong"
+        with pytest.raises(ValidationError):
+            m.validate()
+
+    def test_set_default(self):
+        class ModelWithDefault(Document):
+            status = EnumField(Status, default=Status.DONE)
+
+        m = ModelWithDefault().save()
+        assert m.status == Status.DONE
+
+    def test_enum_field_can_be_empty(self):
+        ModelWithEnum.drop_collection()
+        m = ModelWithEnum().save()
+        assert m.status is None
+        assert ModelWithEnum.objects()[0].status is None
+        assert ModelWithEnum.objects(status=None).count() == 1
+
+    def test_set_none_explicitly(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=None).save()
+        assert ModelWithEnum.objects.first().status is None
+
+    def test_cannot_create_model_with_wrong_enum_value(self):
+        m = ModelWithEnum(status="wrong_one")
+        with pytest.raises(ValidationError):
+            m.validate()
+
+    def test_partial_choices(self):
+        partial = [Status.DONE]
+        enum_field = EnumField(Status, choices=partial)
+        assert enum_field.choices == partial
+
+        class FancyDoc(Document):
+            z = enum_field
+
+        FancyDoc(z=Status.DONE).validate()
+        with pytest.raises(
+            ValidationError, match=r"Value must be one of .*Status.DONE"
+        ):
+            FancyDoc(z=Status.NEW).validate()
+
+    def test_wrong_choices(self):
+        with pytest.raises(ValueError, match="Invalid choices"):
+            EnumField(Status, choices=["my", "custom", "options"])
+        with pytest.raises(ValueError, match="Invalid choices"):
+            EnumField(Status, choices=[Color.RED])
+        with pytest.raises(ValueError, match="Invalid choices"):
+            EnumField(Status, choices=[Status.DONE, Color.RED])
+
+
+class ModelWithColor(Document):
+    color = EnumField(Color, default=Color.RED)
+
+
+class TestIntEnumField(MongoDBTestCase):
+    def test_enum_with_int(self):
+        ModelWithColor.drop_collection()
+        m = ModelWithColor().save()
+        assert m.color == Color.RED
+        assert ModelWithColor.objects(color=Color.RED).count() == 1
+        assert ModelWithColor.objects(color=1).count() == 1
+        assert ModelWithColor.objects(color=2).count() == 0
+
+    def test_create_int_enum_by_value(self):
+        model = ModelWithColor(color=2).save()
+        assert model.color == Color.BLUE
+
+    def test_storage_enum_with_int(self):
+        model = ModelWithColor(color=Color.BLUE).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "color": 2}
+
+    def test_validate_model(self):
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color=3).validate()
+
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color="wrong_type").validate()
+
+
+class TestFunkyEnumField(MongoDBTestCase):
+    def test_enum_incompatible_bson_type_fails_during_save(self):
+        class FunkyColor(Enum):
+            YELLOW = object()
+
+        class ModelWithFunkyColor(Document):
+            color = EnumField(FunkyColor)
+
+        m = ModelWithFunkyColor(color=FunkyColor.YELLOW)
+
+        with pytest.raises(InvalidDocument, match="[cC]annot encode object"):
+            m.save()

tests/fields/test_fields.py

@@ -1,9 +1,8 @@
-# -*- coding: utf-8 -*-
 import datetime
 import unittest
 
-from bson import DBRef, ObjectId, SON
 import pytest
+from bson import SON, DBRef, ObjectId
 
 from mongoengine import (
     BooleanField,
@@ -35,9 +34,12 @@ from mongoengine import (
     StringField,
     ValidationError,
 )
-from mongoengine.base import BaseField, EmbeddedDocumentList, _document_registry
+from mongoengine.base import (
+    BaseField,
+    EmbeddedDocumentList,
+    _document_registry,
+)
 from mongoengine.errors import DeprecatedError
-
 from tests.utils import MongoDBTestCase
 
 
@@ -293,7 +295,7 @@ class TestField(MongoDBTestCase):
         HandleNoneFields.drop_collection()
 
         doc = HandleNoneFields()
-        doc.str_fld = u"spam ham egg"
+        doc.str_fld = "spam ham egg"
         doc.int_fld = 42
         doc.flt_fld = 4.2
         doc.com_dt_fld = datetime.datetime.utcnow()
@@ -307,7 +309,7 @@ class TestField(MongoDBTestCase):
         )
         assert res == 1
 
-        # Retrive data from db and verify it.
+        # Retrieve data from db and verify it.
         ret = HandleNoneFields.objects.all()[0]
         assert ret.str_fld is None
         assert ret.int_fld is None
@@ -329,19 +331,19 @@ class TestField(MongoDBTestCase):
         HandleNoneFields.drop_collection()
 
         doc = HandleNoneFields()
-        doc.str_fld = u"spam ham egg"
+        doc.str_fld = "spam ham egg"
         doc.int_fld = 42
         doc.flt_fld = 4.2
         doc.comp_dt_fld = datetime.datetime.utcnow()
         doc.save()
 
         # Unset all the fields
-        HandleNoneFields._get_collection().update(
+        HandleNoneFields._get_collection().update_one(
             {"_id": doc.id},
             {"$unset": {"str_fld": 1, "int_fld": 1, "flt_fld": 1, "comp_dt_fld": 1}},
         )
 
-        # Retrive data from db and verify it.
+        # Retrieve data from db and verify it.
         ret = HandleNoneFields.objects.first()
         assert ret.str_fld is None
         assert ret.int_fld is None
@@ -375,34 +377,6 @@ class TestField(MongoDBTestCase):
         person.id = str(ObjectId())
         person.validate()
 
-    def test_string_validation(self):
-        """Ensure that invalid values cannot be assigned to string fields."""
-
-        class Person(Document):
-            name = StringField(max_length=20)
-            userid = StringField(r"[0-9a-z_]+$")
-
-        person = Person(name=34)
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        # Test regex validation on userid
-        person = Person(userid="test.User")
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person.userid = "test_user"
-        assert person.userid == "test_user"
-        person.validate()
-
-        # Test max length validation on name
-        person = Person(name="Name that is more than twenty characters")
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person.name = "Shorter name"
-        person.validate()
-
     def test_db_field_validation(self):
         """Ensure that db_field doesn't accept invalid values."""
 
@@ -427,9 +401,9 @@ class TestField(MongoDBTestCase):
     def test_list_validation(self):
         """Ensure that a list field only accepts lists with valid elements."""
         access_level_choices = (
-            ("a", u"Administration"),
+            ("a", "Administration"),
-            ("b", u"Manager"),
+            ("b", "Manager"),
-            ("c", u"Staff"),
+            ("c", "Staff"),
         )
 
         class User(Document):
@@ -477,7 +451,7 @@ class TestField(MongoDBTestCase):
         post.access_list = ["a", "b"]
         post.validate()
 
-        assert post.get_access_list_display() == u"Administration, Manager"
+        assert post.get_access_list_display() == "Administration, Manager"
 
         post.comments = ["a"]
         with pytest.raises(ValidationError):
@@ -545,8 +519,7 @@ class TestField(MongoDBTestCase):
         post.validate()
 
     def test_sorted_list_sorting(self):
-        """Ensure that a sorted list field properly sorts values.
+        """Ensure that a sorted list field properly sorts values."""
-        """
 
         class Comment(EmbeddedDocument):
             order = IntField()
@@ -662,8 +635,7 @@ class TestField(MongoDBTestCase):
         )
 
     def test_list_field_manipulative_operators(self):
-        """Ensure that ListField works with standard list operators that manipulate the list.
+        """Ensure that ListField works with standard list operators that manipulate the list."""
-        """
 
         class BlogPost(Document):
             ref = StringField()
@@ -1084,7 +1056,7 @@ class TestField(MongoDBTestCase):
 
         e = Simple().save()
         e.mapping = []
-        assert [] == e._changed_fields
+        assert e._changed_fields == []
 
         class Simple(Document):
             mapping = DictField()
@@ -1093,7 +1065,7 @@ class TestField(MongoDBTestCase):
 
         e = Simple().save()
         e.mapping = {}
-        assert [] == e._changed_fields
+        assert e._changed_fields == []
 
     def test_slice_marks_field_as_changed(self):
         class Simple(Document):
@@ -1360,9 +1332,9 @@ class TestField(MongoDBTestCase):
         foo.delete()
         bar = Bar.objects.get()
         with pytest.raises(DoesNotExist):
-            getattr(bar, "ref")
+            bar.ref
         with pytest.raises(DoesNotExist):
-            getattr(bar, "generic_ref")
+            bar.generic_ref
 
         # When auto_dereference is disabled, there is no trouble returning DBRef
         bar = Bar.objects.get()
@@ -1373,8 +1345,7 @@ class TestField(MongoDBTestCase):
         assert bar.generic_ref == {"_ref": expected, "_cls": "Foo"}
 
     def test_list_item_dereference(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -1399,8 +1370,7 @@ class TestField(MongoDBTestCase):
         assert group_obj.members[1].name == user2.name
 
     def test_recursive_reference(self):
-        """Ensure that ReferenceFields can reference their own documents.
+        """Ensure that ReferenceFields can reference their own documents."""
-        """
 
         class Employee(Document):
             name = StringField()
@@ -1427,8 +1397,7 @@ class TestField(MongoDBTestCase):
         assert peter.friends == friends
 
     def test_recursive_embedding(self):
-        """Ensure that EmbeddedDocumentFields can contain their own documents.
+        """Ensure that EmbeddedDocumentFields can contain their own documents."""
-        """
 
         class TreeNode(EmbeddedDocument):
             name = StringField()
@@ -1504,8 +1473,7 @@ class TestField(MongoDBTestCase):
             AbstractDoc.drop_collection()
 
     def test_reference_class_with_abstract_parent(self):
-        """Ensure that a class with an abstract parent can be referenced.
+        """Ensure that a class with an abstract parent can be referenced."""
-        """
 
         class Sibling(Document):
             name = StringField()
@@ -1575,8 +1543,7 @@ class TestField(MongoDBTestCase):
             brother.save()
 
     def test_generic_reference(self):
-        """Ensure that a GenericReferenceField properly dereferences items.
+        """Ensure that a GenericReferenceField properly dereferences items."""
-        """
 
         class Link(Document):
             title = StringField()
@@ -1615,8 +1582,7 @@ class TestField(MongoDBTestCase):
         assert isinstance(bm.bookmark_object, Link)
 
     def test_generic_reference_list(self):
-        """Ensure that a ListField properly dereferences generic references.
+        """Ensure that a ListField properly dereferences generic references."""
-        """
 
         class Link(Document):
             title = StringField()
@@ -1719,8 +1685,7 @@ class TestField(MongoDBTestCase):
         assert bm.bookmark_object == post_1
 
     def test_generic_reference_string_choices(self):
-        """Ensure that a GenericReferenceField can handle choices as strings
+        """Ensure that a GenericReferenceField can handle choices as strings"""
-        """
 
         class Link(Document):
             title = StringField()
@@ -1812,8 +1777,7 @@ class TestField(MongoDBTestCase):
         assert user.bookmarks == [post_1]
 
     def test_generic_reference_list_item_modification(self):
-        """Ensure that modifications of related documents (through generic reference) don't influence on querying
+        """Ensure that modifications of related documents (through generic reference) don't influence on querying"""
-        """
 
         class Post(Document):
             title = StringField()
@@ -1901,8 +1865,7 @@ class TestField(MongoDBTestCase):
         assert doc == doc2
 
     def test_choices_allow_using_sets_as_choices(self):
-        """Ensure that sets can be used when setting choices
+        """Ensure that sets can be used when setting choices"""
-        """
 
         class Shirt(Document):
             size = StringField(choices={"M", "L"})
@@ -1921,8 +1884,7 @@ class TestField(MongoDBTestCase):
         shirt.validate()
 
     def test_choices_validation_accept_possible_value(self):
-        """Ensure that value is in a container of allowed values.
+        """Ensure that value is in a container of allowed values."""
-        """
 
         class Shirt(Document):
             size = StringField(choices=("S", "M"))
@@ -1931,8 +1893,7 @@ class TestField(MongoDBTestCase):
         shirt.validate()
 
     def test_choices_validation_reject_unknown_value(self):
-        """Ensure that unallowed value are rejected upon validation
+        """Ensure that unallowed value are rejected upon validation"""
-        """
 
         class Shirt(Document):
             size = StringField(choices=("S", "M"))
@@ -1990,8 +1951,7 @@ class TestField(MongoDBTestCase):
             shirt1.validate()
 
     def test_simple_choices_validation(self):
-        """Ensure that value is in a container of allowed values.
+        """Ensure that value is in a container of allowed values."""
-        """
 
         class Shirt(Document):
             size = StringField(max_length=3, choices=("S", "M", "L", "XL", "XXL"))
@@ -2040,12 +2000,11 @@ class TestField(MongoDBTestCase):
             shirt.validate()
 
     def test_simple_choices_validation_invalid_value(self):
-        """Ensure that error messages are correct.
+        """Ensure that error messages are correct."""
-        """
         SIZES = ("S", "M", "L", "XL", "XXL")
         COLORS = (("R", "Red"), ("B", "Blue"))
-        SIZE_MESSAGE = u"Value must be one of ('S', 'M', 'L', 'XL', 'XXL')"
+        SIZE_MESSAGE = "Value must be one of ('S', 'M', 'L', 'XL', 'XXL')"
-        COLOR_MESSAGE = u"Value must be one of ['R', 'B']"
+        COLOR_MESSAGE = "Value must be one of ['R', 'B']"
 
         class Shirt(Document):
             size = StringField(max_length=3, choices=SIZES)
@@ -2108,7 +2067,7 @@ class TestField(MongoDBTestCase):
             assert "comments" in error_dict
             assert 1 in error_dict["comments"]
             assert "content" in error_dict["comments"][1]
-            assert error_dict["comments"][1]["content"] == u"Field is required"
+            assert error_dict["comments"][1]["content"] == "Field is required"
 
         post.comments[1].content = "here we go"
         post.validate()
@@ -2118,9 +2077,9 @@ class TestField(MongoDBTestCase):
         a ComplexBaseField.
         """
 
-        class EnumField(BaseField):
+        class SomeField(BaseField):
             def __init__(self, **kwargs):
-                super(EnumField, self).__init__(**kwargs)
+                super().__init__(**kwargs)
 
             def to_mongo(self, value):
                 return value
@@ -2129,7 +2088,7 @@ class TestField(MongoDBTestCase):
                 return tuple(value)
 
         class TestDoc(Document):
-            items = ListField(EnumField())
+            items = ListField(SomeField())
 
         TestDoc.drop_collection()
 
@@ -2273,6 +2232,13 @@ class TestField(MongoDBTestCase):
         with pytest.raises(FieldDoesNotExist):
             Doc(bar="test")
 
+    def test_undefined_field_works_no_confusion_with_db_field(self):
+        class Doc(Document):
+            foo = StringField(db_field="bar")
+
+        with pytest.raises(FieldDoesNotExist):
+            Doc(bar="test")
+
 
 class TestEmbeddedDocumentListField(MongoDBTestCase):
     def setUp(self):
@@ -2615,11 +2581,11 @@ class TestEmbeddedDocumentListField(MongoDBTestCase):
         """
         post = self.BlogPost(
             comments=[
-                self.Comments(author="user1", message=u"сообщение"),
+                self.Comments(author="user1", message="сообщение"),
-                self.Comments(author="user2", message=u"хабарлама"),
+                self.Comments(author="user2", message="хабарлама"),
             ]
         ).save()
-        assert post.comments.get(message=u"сообщение").author == "user1"
+        assert post.comments.get(message="сообщение").author == "user1"
 
     def test_save(self):
         """

tests/fields/test_file_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import copy
 import os
 import tempfile
@@ -12,7 +11,7 @@ from mongoengine import *
 from mongoengine.connection import get_db
 
 try:
-    from PIL import Image
+    from PIL import Image  # noqa: F401
 
     HAS_PIL = True
 except ImportError:
@@ -49,15 +48,14 @@ class TestFileField(MongoDBTestCase):
         DemoFile.objects.create()
 
     def test_file_fields(self):
-        """Ensure that file fields can be written to and their data retrieved
+        """Ensure that file fields can be written to and their data retrieved"""
-        """
 
         class PutFile(Document):
             the_file = FileField()
 
         PutFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         putfile = PutFile()
@@ -92,16 +90,15 @@ class TestFileField(MongoDBTestCase):
         result.the_file.delete()
 
     def test_file_fields_stream(self):
-        """Ensure that file fields can be written to and their data retrieved
+        """Ensure that file fields can be written to and their data retrieved"""
-        """
 
         class StreamFile(Document):
             the_file = FileField()
 
         StreamFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
-        more_text = "Foo Bar".encode("latin-1")
+        more_text = b"Foo Bar"
         content_type = "text/plain"
 
         streamfile = StreamFile()
@@ -136,8 +133,8 @@ class TestFileField(MongoDBTestCase):
 
         StreamFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
-        more_text = "Foo Bar".encode("latin-1")
+        more_text = b"Foo Bar"
 
         streamfile = StreamFile()
         streamfile.save()
@@ -166,8 +163,8 @@ class TestFileField(MongoDBTestCase):
         class SetFile(Document):
             the_file = FileField()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
-        more_text = "Foo Bar".encode("latin-1")
+        more_text = b"Foo Bar"
 
         SetFile.drop_collection()
 
@@ -195,7 +192,7 @@ class TestFileField(MongoDBTestCase):
         GridDocument.drop_collection()
 
         with tempfile.TemporaryFile() as f:
-            f.write("Hello World!".encode("latin-1"))
+            f.write(b"Hello World!")
             f.flush()
 
             # Test without default
@@ -212,7 +209,7 @@ class TestFileField(MongoDBTestCase):
             assert doc_b.the_file.grid_id == doc_c.the_file.grid_id
 
             # Test with default
-            doc_d = GridDocument(the_file="".encode("latin-1"))
+            doc_d = GridDocument(the_file=b"")
             doc_d.save()
 
             doc_e = GridDocument.objects.with_id(doc_d.id)
@@ -229,8 +226,7 @@ class TestFileField(MongoDBTestCase):
         assert ["doc_b", "doc_e"] == grid_fs.list()
 
     def test_file_uniqueness(self):
-        """Ensure that each instance of a FileField is unique
+        """Ensure that each instance of a FileField is unique"""
-        """
 
         class TestFile(Document):
             name = StringField()
@@ -239,7 +235,7 @@ class TestFileField(MongoDBTestCase):
         # First instance
         test_file = TestFile()
         test_file.name = "Hello, World!"
-        test_file.the_file.put("Hello, World!".encode("latin-1"))
+        test_file.the_file.put(b"Hello, World!")
         test_file.save()
 
         # Second instance
@@ -286,8 +282,7 @@ class TestFileField(MongoDBTestCase):
         assert test_file.the_file.get().length == 4971
 
     def test_file_boolean(self):
-        """Ensure that a boolean test of a FileField indicates its presence
+        """Ensure that a boolean test of a FileField indicates its presence"""
-        """
 
         class TestFile(Document):
             the_file = FileField()
@@ -296,9 +291,7 @@ class TestFileField(MongoDBTestCase):
 
         test_file = TestFile()
         assert not bool(test_file.the_file)
-        test_file.the_file.put(
+        test_file.the_file.put(b"Hello, World!", content_type="text/plain")
-            "Hello, World!".encode("latin-1"), content_type="text/plain"
-        )
         test_file.save()
         assert bool(test_file.the_file)
 
@@ -315,12 +308,12 @@ class TestFileField(MongoDBTestCase):
         assert test_file.the_file not in [{"test": 1}]
 
     def test_file_disk_space(self):
-        """ Test disk space usage when we delete/replace a file """
+        """Test disk space usage when we delete/replace a file"""
 
         class TestFile(Document):
             the_file = FileField()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         testfile = TestFile()
@@ -364,7 +357,7 @@ class TestFileField(MongoDBTestCase):
         testfile.the_file.put(text, content_type=content_type, filename="hello")
         testfile.save()
 
-        text = "Bonjour, World!".encode("latin-1")
+        text = b"Bonjour, World!"
         testfile.the_file.replace(text, content_type=content_type, filename="hello")
         testfile.save()
 
@@ -388,7 +381,7 @@ class TestFileField(MongoDBTestCase):
         TestImage.drop_collection()
 
         with tempfile.TemporaryFile() as f:
-            f.write("Hello World!".encode("latin-1"))
+            f.write(b"Hello World!")
             f.flush()
 
             t = TestImage()
@@ -430,7 +423,7 @@ class TestFileField(MongoDBTestCase):
     @require_pil
     def test_image_field_resize(self):
         class TestImage(Document):
-            image = ImageField(size=(185, 37))
+            image = ImageField(size=(185, 37, True))
 
         TestImage.drop_collection()
 
@@ -472,7 +465,7 @@ class TestFileField(MongoDBTestCase):
     @require_pil
     def test_image_field_thumbnail(self):
         class TestImage(Document):
-            image = ImageField(thumbnail_size=(92, 18))
+            image = ImageField(thumbnail_size=(92, 18, True))
 
         TestImage.drop_collection()
 
@@ -504,21 +497,21 @@ class TestFileField(MongoDBTestCase):
         # First instance
         test_file = TestFile()
         test_file.name = "Hello, World!"
-        test_file.the_file.put("Hello, World!".encode("latin-1"), name="hello.txt")
+        test_file.the_file.put(b"Hello, World!", name="hello.txt")
         test_file.save()
 
         data = get_db("test_files").macumba.files.find_one()
         assert data.get("name") == "hello.txt"
 
         test_file = TestFile.objects.first()
-        assert test_file.the_file.read() == "Hello, World!".encode("latin-1")
+        assert test_file.the_file.read() == b"Hello, World!"
 
         test_file = TestFile.objects.first()
-        test_file.the_file = "Hello, World!".encode("latin-1")
+        test_file.the_file = b"Hello, World!"
         test_file.save()
 
         test_file = TestFile.objects.first()
-        assert test_file.the_file.read() == "Hello, World!".encode("latin-1")
+        assert test_file.the_file.read() == b"Hello, World!"
 
     def test_copyable(self):
         class PutFile(Document):
@@ -526,7 +519,7 @@ class TestFileField(MongoDBTestCase):
 
         PutFile.drop_collection()
 
-        text = "Hello, World!".encode("latin-1")
+        text = b"Hello, World!"
         content_type = "text/plain"
 
         putfile = PutFile()

tests/fields/test_float_field.py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
@@ -20,8 +18,7 @@ class TestFloatField(MongoDBTestCase):
         assert 1 == TestDocument.objects(float_fld__ne=1).count()
 
     def test_validation(self):
-        """Ensure that invalid values cannot be assigned to float fields.
+        """Ensure that invalid values cannot be assigned to float fields."""
-        """
 
         class Person(Document):
             height = FloatField(min_value=0.1, max_value=3.5)

tests/fields/test_geo_fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *
@@ -9,7 +8,7 @@ class TestGeoField(MongoDBTestCase):
     def _test_for_expected_error(self, Cls, loc, expected):
         try:
             Cls(loc=loc).validate()
-            self.fail("Should not validate the location {0}".format(loc))
+            self.fail(f"Should not validate the location {loc}")
         except ValidationError as e:
             assert expected == e.to_dict()["loc"]
 
@@ -291,8 +290,7 @@ class TestGeoField(MongoDBTestCase):
         Location(loc=[[[[1, 2], [3, 4], [5, 6], [1, 2]]]]).validate()
 
     def test_indexes_geopoint(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
+        """Ensure that indexes are created automatically for GeoPointFields."""
-        """
 
         class Event(Document):
             title = StringField()
@@ -318,8 +316,7 @@ class TestGeoField(MongoDBTestCase):
         assert geo_indicies == [{"fields": [("venue.location", "2d")]}]
 
     def test_indexes_2dsphere(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
+        """Ensure that indexes are created automatically for GeoPointFields."""
-        """
 
         class Event(Document):
             title = StringField()
@@ -333,8 +330,7 @@ class TestGeoField(MongoDBTestCase):
         assert {"fields": [("point", "2dsphere")]} in geo_indicies
 
     def test_indexes_2dsphere_embedded(self):
-        """Ensure that indexes are created automatically for GeoPointFields.
+        """Ensure that indexes are created automatically for GeoPointFields."""
-        """
 
         class Venue(EmbeddedDocument):
             name = StringField()
@@ -381,7 +377,7 @@ class TestGeoField(MongoDBTestCase):
 
             meta = {"indexes": [[("location", "2dsphere"), ("datetime", 1)]]}
 
-        assert [] == Log._geo_indices()
+        assert Log._geo_indices() == []
 
         Log.drop_collection()
         Log.ensure_indexes()
@@ -401,7 +397,7 @@ class TestGeoField(MongoDBTestCase):
                 "indexes": [{"fields": [("location", "2dsphere"), ("datetime", 1)]}]
             }
 
-        assert [] == Log._geo_indices()
+        assert Log._geo_indices() == []
 
         Log.drop_collection()
         Log.ensure_indexes()

tests/fields/test_int_field.py

@@ -1,15 +1,12 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
 class TestIntField(MongoDBTestCase):
     def test_int_validation(self):
-        """Ensure that invalid values cannot be assigned to int fields.
+        """Ensure that invalid values cannot be assigned to int fields."""
-        """
 
         class Person(Document):
             age = IntField(min_value=0, max_value=110)

tests/fields/test_lazy_reference_field.py

@@ -1,10 +1,9 @@
-# -*- coding: utf-8 -*-
-from bson import DBRef, ObjectId
 import pytest
+from bson import DBRef, ObjectId
 
 from mongoengine import *
 from mongoengine.base import LazyReference
-
+from mongoengine.context_managers import query_counter
 from tests.utils import MongoDBTestCase
 
 
@@ -331,6 +330,70 @@ class TestLazyReferenceField(MongoDBTestCase):
         occ.in_embedded.in_list = [animal1.id, animal2.id]
         check_fields_type(occ)
 
+    def test_lazy_reference_embedded_dereferencing(self):
+        # Test case for #2375
+
+        # -- Test documents
+
+        class Author(Document):
+            name = StringField()
+
+        class AuthorReference(EmbeddedDocument):
+            author = LazyReferenceField(Author)
+
+        class Book(Document):
+            authors = EmbeddedDocumentListField(AuthorReference)
+
+        # -- Cleanup
+
+        Author.drop_collection()
+        Book.drop_collection()
+
+        # -- Create test data
+
+        author_1 = Author(name="A1").save()
+        author_2 = Author(name="A2").save()
+        author_3 = Author(name="A3").save()
+        book = Book(
+            authors=[
+                AuthorReference(author=author_1),
+                AuthorReference(author=author_2),
+                AuthorReference(author=author_3),
+            ]
+        ).save()
+
+        with query_counter() as qc:
+            book = Book.objects.first()
+            # Accessing the list must not trigger dereferencing.
+            book.authors
+            assert qc == 1
+
+        for ref in book.authors:
+            with pytest.raises(AttributeError):
+                ref["author"].name
+            assert isinstance(ref.author, LazyReference)
+            assert isinstance(ref.author.id, ObjectId)
+
+    def test_lazy_reference_in_list_with_changed_element(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            in_list = ListField(LazyReferenceField(Animal))
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal(name="doggo").save()
+
+        animal1.tag = "blue"
+
+        occ = Ocurrence(in_list=[animal1]).save()
+        animal1.save()
+        assert isinstance(occ.in_list[0], LazyReference)
+        assert occ.in_list[0].pk == animal1.pk
+
 
 class TestGenericLazyReferenceField(MongoDBTestCase):
     def test_generic_lazy_reference_simple(self):

tests/fields/test_long_field.py

@@ -1,13 +1,28 @@
-from bson.int64 import Int64
 import pytest
+from bson.int64 import Int64
 
 from mongoengine import *
 from mongoengine.connection import get_db
-
+from tests.utils import MongoDBTestCase, get_as_pymongo
-from tests.utils import MongoDBTestCase
 
 
 class TestLongField(MongoDBTestCase):
+    def test_storage(self):
+        class Person(Document):
+            value = LongField()
+
+        Person.drop_collection()
+        person = Person(value=5000)
+        person.save()
+        assert get_as_pymongo(person) == {"_id": person.id, "value": 5000}
+
+    def test_construction_does_not_fail_with_invalid_value(self):
+        class Person(Document):
+            value = LongField()
+
+        person = Person(value="not_an_int")
+        assert person.value == "not_an_int"
+
     def test_long_field_is_considered_as_int64(self):
         """
         Tests that long fields are stored as long in mongo, even if long
@@ -25,25 +40,21 @@ class TestLongField(MongoDBTestCase):
         assert isinstance(doc.some_long, int)
 
     def test_long_validation(self):
-        """Ensure that invalid values cannot be assigned to long fields.
+        """Ensure that invalid values cannot be assigned to long fields."""
-        """
 
         class TestDocument(Document):
             value = LongField(min_value=0, max_value=110)
 
-        doc = TestDocument()
+        TestDocument(value=50).validate()
-        doc.value = 50
-        doc.validate()
 
-        doc.value = -1
         with pytest.raises(ValidationError):
-            doc.validate()
+            TestDocument(value=-1).validate()
-        doc.value = 120
+
         with pytest.raises(ValidationError):
-            doc.validate()
+            TestDocument(value=120).validate()
-        doc.value = "ten"
+
         with pytest.raises(ValidationError):
-            doc.validate()
+            TestDocument(value="ten").validate()
 
     def test_long_ne_operator(self):
         class TestDocument(Document):
@@ -54,4 +65,5 @@ class TestLongField(MongoDBTestCase):
         TestDocument(long_fld=None).save()
         TestDocument(long_fld=1).save()
 
-        assert 1 == TestDocument.objects(long_fld__ne=None).count()
+        assert TestDocument.objects(long_fld__ne=None).count() == 1
+        assert TestDocument.objects(long_fld__ne=1).count() == 1

tests/fields/test_map_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest
@@ -136,11 +135,11 @@ class TestMapField(MongoDBTestCase):
 
         BlogPost.drop_collection()
 
-        tree = BlogPost(info_dict={u"éééé": {"description": u"VALUE: éééé"}})
+        tree = BlogPost(info_dict={"éééé": {"description": "VALUE: éééé"}})
 
         tree.save()
 
         assert (
-            BlogPost.objects.get(id=tree.id).info_dict[u"éééé"].description
+            BlogPost.objects.get(id=tree.id).info_dict["éééé"].description
-            == u"VALUE: éééé"
+            == "VALUE: éééé"
         )

tests/fields/test_reference_field.py

@@ -1,6 +1,5 @@
-# -*- coding: utf-8 -*-
-from bson import DBRef, SON
 import pytest
+from bson import SON, DBRef
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase
@@ -88,7 +87,7 @@ class TestReferenceField(MongoDBTestCase):
             parent = ReferenceField("self", dbref=False)
 
         p = Person(name="Steve", parent=DBRef("person", "abcdefghijklmnop"))
-        assert p.to_mongo() == SON([("name", u"Steve"), ("parent", "abcdefghijklmnop")])
+        assert p.to_mongo() == SON([("name", "Steve"), ("parent", "abcdefghijklmnop")])
 
     def test_objectid_reference_fields(self):
         class Person(Document):
@@ -108,8 +107,7 @@ class TestReferenceField(MongoDBTestCase):
         assert p.parent == p1
 
     def test_undefined_reference(self):
-        """Ensure that ReferenceFields may reference undefined Documents.
+        """Ensure that ReferenceFields may reference undefined Documents."""
-        """
 
         class Product(Document):
             name = StringField()

tests/fields/test_sequence_field.py

@@ -1,7 +1,4 @@
-# -*- coding: utf-8 -*-
-
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
@@ -168,8 +165,8 @@ class TestSequenceField(MongoDBTestCase):
         ids = [i.id for i in Person.objects]
         assert ids == list(range(1, 11))
 
-        id = [i.id for i in Animal.objects]
+        _id = [i.id for i in Animal.objects]
-        assert id == list(range(1, 11))
+        assert _id == list(range(1, 11))
 
         c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
         assert c["next"] == 10
@@ -276,3 +273,25 @@ class TestSequenceField(MongoDBTestCase):
         assert foo.counter == bar.counter
         assert foo._fields["counter"].owner_document == Foo
         assert bar._fields["counter"].owner_document == Bar
+
+    def test_sequence_setattr_not_incrementing_counter(self):
+        class Person(DynamicDocument):
+            id = SequenceField(primary_key=True)
+            name = StringField()
+
+        self.db["mongoengine.counters"].drop()
+        Person.drop_collection()
+
+        for x in range(10):
+            Person(name="Person %s" % x).save()
+
+        c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
+        assert c["next"] == 10
+
+        # Setting SequenceField field value should not increment counter:
+        new_person = Person()
+        new_person.id = 1100
+
+        # Counter should still be at 10
+        c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
+        assert c["next"] == 10

tests/fields/test_string_field.py

@@ -0,0 +1,43 @@
+import pytest
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class TestStringField(MongoDBTestCase):
+    def test_storage(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        person = Person(name="test123")
+        person.save()
+        assert get_as_pymongo(person) == {"_id": person.id, "name": "test123"}
+
+    def test_validation(self):
+        class Person(Document):
+            name = StringField(max_length=20, min_length=2)
+            userid = StringField(r"[0-9a-z_]+$")
+
+        with pytest.raises(ValidationError, match="only accepts string values"):
+            Person(name=34).validate()
+
+        with pytest.raises(ValidationError, match="value is too short"):
+            Person(name="s").validate()
+
+        # Test regex validation on userid
+        person = Person(userid="test.User")
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person.userid = "test_user"
+        assert person.userid == "test_user"
+        person.validate()
+
+        # Test max length validation on name
+        person = Person(name="Name that is more than twenty characters")
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person = Person(name="a friendl name", userid="7a757668sqjdkqlsdkq")
+        person.validate()

tests/fields/test_url_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *
@@ -27,7 +26,7 @@ class TestURLField(MongoDBTestCase):
             url = URLField()
 
         link = Link()
-        link.url = u"http://привет.com"
+        link.url = "http://привет.com"
 
         # TODO fix URL validation - this *IS* a valid URL
         # For now we just want to make sure that the error message is correct
@@ -35,12 +34,11 @@ class TestURLField(MongoDBTestCase):
             link.validate()
         assert (
             str(exc_info.value)
-            == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])"
+            == "ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])"
         )
 
     def test_url_scheme_validation(self):
-        """Ensure that URLFields validate urls with specific schemes properly.
+        """Ensure that URLFields validate urls with specific schemes properly."""
-        """
 
         class Link(Document):
             url = URLField()

tests/fields/test_uuid_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 import pytest
@@ -18,8 +17,7 @@ class TestUUIDField(MongoDBTestCase):
         assert get_as_pymongo(person) == {"_id": person.id, "api_key": str(uid)}
 
     def test_field_string(self):
-        """Test UUID fields storing as String
+        """Test UUID fields storing as String"""
-        """
         Person.drop_collection()
 
         uu = uuid.uuid4()

tests/fixtures.py

@@ -53,7 +53,7 @@ signals.post_save.connect(PickleSignalsTest.post_save, sender=PickleSignalsTest)
 signals.post_delete.connect(PickleSignalsTest.post_delete, sender=PickleSignalsTest)
 
 
-class Mixin(object):
+class Mixin:
     name = StringField()
 
 

tests/queryset/test_field_list.py

@@ -148,8 +148,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         assert qs._loaded_fields.as_dict() == {"c": {"$slice": 2}, "a": 1}
 
     def test_only(self):
-        """Ensure that QuerySet.only only returns the requested fields.
+        """Ensure that QuerySet.only only returns the requested fields."""
-        """
         person = self.Person(name="test", age=25)
         person.save()
 
@@ -365,8 +364,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         Email.drop_collection()
 
     def test_slicing_fields(self):
-        """Ensure that query slicing an array works.
+        """Ensure that query slicing an array works."""
-        """
 
         class Numbers(Document):
             n = ListField(IntField())
@@ -401,8 +399,7 @@ class TestOnlyExcludeAll(unittest.TestCase):
         assert numbers.n == [-5, -4, -3, -2, -1]
 
     def test_slicing_nested_fields(self):
-        """Ensure that query slicing an embedded array works.
+        """Ensure that query slicing an embedded array works."""
-        """
 
         class EmbeddedNumber(EmbeddedDocument):
             n = ListField(IntField())

tests/queryset/test_geo.py

@@ -2,7 +2,6 @@ import datetime
 import unittest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
@@ -496,8 +495,8 @@ class TestGeoQueries(MongoDBTestCase):
         p.save()
         qs = Place.objects().only("location")
         assert qs.as_pymongo()[0]["location"] == {
-            u"type": u"Point",
+            "type": "Point",
-            u"coordinates": [24.946861267089844, 60.16311983618494],
+            "coordinates": [24.946861267089844, 60.16311983618494],
         }
 
     def test_2dsphere_point_sets_correctly(self):

tests/queryset/test_modify.py

@@ -1,6 +1,12 @@
 import unittest
 
-from mongoengine import Document, IntField, ListField, StringField, connect
+from mongoengine import (
+    Document,
+    IntField,
+    ListField,
+    StringField,
+    connect,
+)
 
 
 class Doc(Document):
@@ -13,7 +19,7 @@ class TestFindAndModify(unittest.TestCase):
         connect(db="mongoenginetest")
         Doc.drop_collection()
 
-    def assertDbEqual(self, docs):
+    def _assert_db_equal(self, docs):
         assert list(Doc._collection.find().sort("id")) == docs
 
     def test_modify(self):
@@ -22,7 +28,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(set__value=-1)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_new(self):
         Doc(id=0, value=0).save()
@@ -31,18 +37,18 @@ class TestFindAndModify(unittest.TestCase):
         new_doc = Doc.objects(id=1).modify(set__value=-1, new=True)
         doc.value = -1
         assert new_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_not_existing(self):
         Doc(id=0, value=0).save()
         assert Doc.objects(id=1).modify(set__value=-1) is None
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_modify_with_upsert(self):
         Doc(id=0, value=0).save()
         old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True)
         assert old_doc is None
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
 
     def test_modify_with_upsert_existing(self):
         Doc(id=0, value=0).save()
@@ -50,13 +56,13 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_upsert_with_new(self):
         Doc(id=0, value=0).save()
         new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1)
         assert new_doc.to_mongo() == {"_id": 1, "value": 1}
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
 
     def test_modify_with_remove(self):
         Doc(id=0, value=0).save()
@@ -64,12 +70,12 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(remove=True)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_find_and_modify_with_remove_not_existing(self):
         Doc(id=0, value=0).save()
         assert Doc.objects(id=1).modify(remove=True) is None
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_modify_with_order_by(self):
         Doc(id=0, value=3).save()
@@ -79,7 +85,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects().order_by("-id").modify(set__value=-1)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual(
+        self._assert_db_equal(
             [
                 {"_id": 0, "value": 3},
                 {"_id": 1, "value": 2},
@@ -94,7 +100,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).only("id").modify(set__value=-1)
         assert old_doc.to_mongo() == {"_id": 1}
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_push(self):
         class BlogPost(Document):

tests/queryset/test_pickable.py

@@ -1,8 +1,6 @@
 import pickle
-import unittest
 
 from mongoengine import Document, IntField, StringField
-from mongoengine.connection import connect
 from tests.utils import MongoDBTestCase
 
 
@@ -18,18 +16,15 @@ class TestQuerysetPickable(MongoDBTestCase):
     """
 
     def setUp(self):
-        super(TestQuerysetPickable, self).setUp()
+        super().setUp()
         self.john = Person.objects.create(name="John", age=21)
 
     def test_picke_simple_qs(self):
-
         qs = Person.objects.all()
-
         pickle.dumps(qs)
 
     def _get_loaded(self, qs):
         s = pickle.dumps(qs)
-
         return pickle.loads(s)
 
     def test_unpickle(self):

tests/queryset/test_queryset_aggregation.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 import warnings
 
@@ -66,7 +64,7 @@ class TestQuerysetAggregate(MongoDBTestCase):
 
         pipeline = [{"$match": {"name": "Isabella Luanna"}}]
         data = Person.objects().aggregate(pipeline)
-        assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}]
+        assert list(data) == [{"_id": p1.pk, "age": 16, "name": "Isabella Luanna"}]
 
     def test_queryset_aggregation_with_skip(self):
         class Person(Document):
@@ -250,6 +248,34 @@ class TestQuerysetAggregate(MongoDBTestCase):
 
         assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
 
+    def test_queryset_aggregation_geonear_aggregation_on_pointfield(self):
+        """test ensures that $geonear can be used as a 1-stage pipeline and that
+        MongoEngine does not interfer with such pipeline (#2473)
+        """
+
+        class Aggr(Document):
+            name = StringField()
+            c = PointField()
+
+        Aggr.drop_collection()
+
+        agg1 = Aggr(name="X", c=[10.634584, 35.8245029]).save()
+        agg2 = Aggr(name="Y", c=[10.634584, 35.8245029]).save()
+
+        pipeline = [
+            {
+                "$geoNear": {
+                    "near": {"type": "Point", "coordinates": [10.634584, 35.8245029]},
+                    "distanceField": "c",
+                    "spherical": True,
+                }
+            }
+        ]
+        assert list(Aggr.objects.aggregate(*pipeline)) == [
+            {"_id": agg1.id, "c": 0.0, "name": "X"},
+            {"_id": agg2.id, "c": 0.0, "name": "Y"},
+        ]
+
 
 if __name__ == "__main__":
     unittest.main()

tests/queryset/test_transform.py

@@ -1,7 +1,7 @@
 import unittest
 
-from bson.son import SON
 import pytest
+from bson.son import SON
 
 from mongoengine import *
 from mongoengine.queryset import Q, transform
@@ -12,8 +12,7 @@ class TestTransform(unittest.TestCase):
         connect(db="mongoenginetest")
 
     def test_transform_query(self):
-        """Ensure that the _transform_query function operates correctly.
+        """Ensure that the _transform_query function operates correctly."""
-        """
         assert transform.query(name="test", age=30) == {"name": "test", "age": 30}
         assert transform.query(age__lt=30) == {"age": {"$lt": 30}}
         assert transform.query(age__gt=20, age__lt=50) == {
@@ -88,8 +87,7 @@ class TestTransform(unittest.TestCase):
         assert update == {"$set": {"tags": ["mongo", "db"]}}
 
     def test_query_field_name(self):
-        """Ensure that the correct field name is used when querying.
+        """Ensure that the correct field name is used when querying."""
-        """
 
         class Comment(EmbeddedDocument):
             content = StringField(db_field="commentContent")
@@ -106,18 +104,17 @@ class TestTransform(unittest.TestCase):
         post = BlogPost(**data)
         post.save()
 
-        assert "postTitle" in BlogPost.objects(title=data["title"])._query
+        qs = BlogPost.objects(title=data["title"])
-        assert not ("title" in BlogPost.objects(title=data["title"])._query)
+        assert qs._query == {"postTitle": data["title"]}
-        assert BlogPost.objects(title=data["title"]).count() == 1
+        assert qs.count() == 1
 
-        assert "_id" in BlogPost.objects(pk=post.id)._query
+        qs = BlogPost.objects(pk=post.id)
-        assert BlogPost.objects(pk=post.id).count() == 1
+        assert qs._query == {"_id": post.id}
+        assert qs.count() == 1
 
-        assert (
+        qs = BlogPost.objects(comments__content="test")
-            "postComments.commentContent"
+        assert qs._query == {"postComments.commentContent": "test"}
-            in BlogPost.objects(comments__content="test")._query
+        assert qs.count() == 1
-        )
-        assert BlogPost.objects(comments__content="test").count() == 1
 
         BlogPost.drop_collection()
 
@@ -330,7 +327,7 @@ class TestTransform(unittest.TestCase):
         word = Word(word="abc", index=1)
         update = transform.update(MainDoc, pull__content__text=word)
         assert update == {
-            "$pull": {"content.text": SON([("word", u"abc"), ("index", 1)])}
+            "$pull": {"content.text": SON([("word", "abc"), ("index", 1)])}
         }
 
         update = transform.update(MainDoc, pull__content__heading="xyz")
@@ -344,6 +341,31 @@ class TestTransform(unittest.TestCase):
         )
         assert update == {"$pull": {"content.text": {"word": {"$nin": ["foo", "bar"]}}}}
 
+    def test_transform_embedded_document_list_fields(self):
+        """
+        Test added to check filtering
+        EmbeddedDocumentListField which is inside a EmbeddedDocumentField
+        """
+
+        class Drink(EmbeddedDocument):
+            id = StringField()
+            meta = {"strict": False}
+
+        class Shop(Document):
+            drinks = EmbeddedDocumentListField(Drink)
+
+        Shop.drop_collection()
+        drinks = [Drink(id="drink_1"), Drink(id="drink_2")]
+        Shop.objects.create(drinks=drinks)
+        q_obj = transform.query(
+            Shop, drinks__all=[{"$elemMatch": {"_id": x.id}} for x in drinks]
+        )
+        assert q_obj == {
+            "drinks": {"$all": [{"$elemMatch": {"_id": x.id}} for x in drinks]}
+        }
+
+        Shop.drop_collection()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/queryset/test_visitor.py

@@ -2,8 +2,8 @@ import datetime
 import re
 import unittest
 
-from bson import ObjectId
 import pytest
+from bson import ObjectId
 
 from mongoengine import *
 from mongoengine.errors import InvalidQueryError
@@ -23,8 +23,7 @@ class TestQ(unittest.TestCase):
         self.Person = Person
 
     def test_empty_q(self):
-        """Ensure that empty Q objects won't hurt.
+        """Ensure that empty Q objects won't hurt."""
-        """
         q1 = Q()
         q2 = Q(age__gte=18)
         q3 = Q()
@@ -58,8 +57,7 @@ class TestQ(unittest.TestCase):
         assert Post.objects.filter(Q(created_user=user)).count() == 1
 
     def test_and_combination(self):
-        """Ensure that Q-objects correctly AND together.
+        """Ensure that Q-objects correctly AND together."""
-        """
 
         class TestDoc(Document):
             x = IntField()
@@ -89,8 +87,7 @@ class TestQ(unittest.TestCase):
         assert query.to_query(TestDoc) == mongo_query
 
     def test_or_combination(self):
-        """Ensure that Q-objects correctly OR together.
+        """Ensure that Q-objects correctly OR together."""
-        """
 
         class TestDoc(Document):
             x = IntField()
@@ -101,8 +98,7 @@ class TestQ(unittest.TestCase):
         assert query == {"$or": [{"x": {"$lt": 3}}, {"x": {"$gt": 7}}]}
 
     def test_and_or_combination(self):
-        """Ensure that Q-objects handle ANDing ORed components.
+        """Ensure that Q-objects handle ANDing ORed components."""
-        """
 
         class TestDoc(Document):
             x = IntField()
@@ -136,8 +132,7 @@ class TestQ(unittest.TestCase):
         assert 2 == TestDoc.objects(q1 & q2).count()
 
     def test_or_and_or_combination(self):
-        """Ensure that Q-objects handle ORing ANDed ORed components. :)
+        """Ensure that Q-objects handle ORing ANDed ORed components. :)"""
-        """
 
         class TestDoc(Document):
             x = IntField()
@@ -208,8 +203,7 @@ class TestQ(unittest.TestCase):
         assert test.count() == 3
 
     def test_q(self):
-        """Ensure that Q objects may be used to query for documents.
+        """Ensure that Q objects may be used to query for documents."""
-        """
 
         class BlogPost(Document):
             title = StringField()
@@ -286,8 +280,7 @@ class TestQ(unittest.TestCase):
             self.Person.objects.filter("user1")
 
     def test_q_regex(self):
-        """Ensure that Q objects can be queried using regexes.
+        """Ensure that Q objects can be queried using regexes."""
-        """
         person = self.Person(name="Guido van Rossum")
         person.save()
 
@@ -320,8 +313,7 @@ class TestQ(unittest.TestCase):
         )
 
     def test_q_lists(self):
-        """Ensure that Q objects query ListFields correctly.
+        """Ensure that Q objects query ListFields correctly."""
-        """
 
         class BlogPost(Document):
             tags = ListField(StringField())

tests/test_common.py

@@ -1,5 +1,3 @@
-import unittest
-
 import pytest
 
 from mongoengine import Document

tests/test_connection.py

@@ -1,16 +1,11 @@
 import datetime
+import unittest
 
-from bson.tz_util import utc
 import pymongo
-
+import pytest
+from bson.tz_util import utc
 from pymongo import MongoClient, ReadPreference
 from pymongo.errors import InvalidName, OperationFailure
-import pytest
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
 
 import mongoengine.connection
 from mongoengine import (
@@ -22,8 +17,8 @@ from mongoengine import (
     register_connection,
 )
 from mongoengine.connection import (
-    ConnectionFailure,
     DEFAULT_DATABASE_NAME,
+    ConnectionFailure,
     disconnect,
     get_connection,
     get_db,
@@ -34,18 +29,6 @@ def get_tz_awareness(connection):
     return connection.codec_options.tz_aware
 
 
-try:
-    import mongomock
-
-    MONGOMOCK_INSTALLED = True
-except ImportError:
-    MONGOMOCK_INSTALLED = False
-
-require_mongomock = pytest.mark.skipif(
-    not MONGOMOCK_INSTALLED, reason="you need mongomock installed to run this testcase"
-)
-
-
 class ConnectionTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -194,14 +177,12 @@ class ConnectionTest(unittest.TestCase):
         assert len(mongoengine.connection._connections) == 3
 
     def test_connect_with_invalid_db_name(self):
-        """Ensure that connect() method fails fast if db name is invalid
+        """Ensure that connect() method fails fast if db name is invalid"""
-        """
         with pytest.raises(InvalidName):
-            connect("mongomock://localhost")
+            connect("mongodb://localhost")
 
     def test_connect_with_db_name_external(self):
-        """Ensure that connect() works if db name is $external
+        """Ensure that connect() works if db name is $external"""
-        """
         """Ensure that the connect() method works properly."""
         connect("$external")
 
@@ -217,112 +198,11 @@ class ConnectionTest(unittest.TestCase):
         assert isinstance(conn, pymongo.mongo_client.MongoClient)
 
     def test_connect_with_invalid_db_name_type(self):
-        """Ensure that connect() method fails fast if db name has invalid type
+        """Ensure that connect() method fails fast if db name has invalid type"""
-        """
         with pytest.raises(TypeError):
             non_string_db_name = ["e. g. list instead of a string"]
             connect(non_string_db_name)
 
-    @require_mongomock
-    def test_connect_in_mocking(self):
-        """Ensure that the connect() method works properly in mocking.
-        """
-        connect("mongoenginetest", host="mongomock://localhost")
-        conn = get_connection()
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect("mongoenginetest2", host="mongomock://localhost", alias="testdb2")
-        conn = get_connection("testdb2")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            "mongoenginetest3",
-            host="mongodb://localhost",
-            is_mock=True,
-            alias="testdb3",
-        )
-        conn = get_connection("testdb3")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect("mongoenginetest4", is_mock=True, alias="testdb4")
-        conn = get_connection("testdb4")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host="mongodb://localhost:27017/mongoenginetest5",
-            is_mock=True,
-            alias="testdb5",
-        )
-        conn = get_connection("testdb5")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host="mongomock://localhost:27017/mongoenginetest6", alias="testdb6")
-        conn = get_connection("testdb6")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host="mongomock://localhost:27017/mongoenginetest7",
-            is_mock=True,
-            alias="testdb7",
-        )
-        conn = get_connection("testdb7")
-        assert isinstance(conn, mongomock.MongoClient)
-
-    @require_mongomock
-    def test_default_database_with_mocking(self):
-        """Ensure that the default database is correctly set when using mongomock.
-        """
-        disconnect_all()
-
-        class SomeDocument(Document):
-            pass
-
-        conn = connect(host="mongomock://localhost:27017/mongoenginetest")
-        some_document = SomeDocument()
-        # database won't exist until we save a document
-        some_document.save()
-        assert conn.get_default_database().name == "mongoenginetest"
-        assert conn.database_names()[0] == "mongoenginetest"
-
-    @require_mongomock
-    def test_connect_with_host_list(self):
-        """Ensure that the connect() method works when host is a list
-
-        Uses mongomock to test w/o needing multiple mongod/mongos processes
-        """
-        connect(host=["mongomock://localhost"])
-        conn = get_connection()
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host=["mongodb://localhost"], is_mock=True, alias="testdb2")
-        conn = get_connection("testdb2")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(host=["localhost"], is_mock=True, alias="testdb3")
-        conn = get_connection("testdb3")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["mongomock://localhost:27017", "mongomock://localhost:27018"],
-            alias="testdb4",
-        )
-        conn = get_connection("testdb4")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["mongodb://localhost:27017", "mongodb://localhost:27018"],
-            is_mock=True,
-            alias="testdb5",
-        )
-        conn = get_connection("testdb5")
-        assert isinstance(conn, mongomock.MongoClient)
-
-        connect(
-            host=["localhost:27017", "localhost:27018"], is_mock=True, alias="testdb6"
-        )
-        conn = get_connection("testdb6")
-        assert isinstance(conn, mongomock.MongoClient)
-
     def test_disconnect_cleans_globals(self):
         """Ensure that the disconnect() method cleans the globals objects"""
         connections = mongoengine.connection._connections
@@ -452,8 +332,7 @@ class ConnectionTest(unittest.TestCase):
         disconnect_all()
 
     def test_sharing_connections(self):
-        """Ensure that connections are shared when the connection settings are exactly the same
+        """Ensure that connections are shared when the connection settings are exactly the same"""
-        """
         connect("mongoenginetests", alias="testdb1")
         expected_connection = get_connection("testdb1")
 
@@ -564,8 +443,7 @@ class ConnectionTest(unittest.TestCase):
         authd_conn.admin.system.users.delete_many({})
 
     def test_register_connection(self):
-        """Ensure that connections with different aliases may be registered.
+        """Ensure that connections with different aliases may be registered."""
-        """
         register_connection("testdb", "mongoenginetest2")
 
         with pytest.raises(ConnectionFailure):
@@ -578,8 +456,7 @@ class ConnectionTest(unittest.TestCase):
         assert db.name == "mongoenginetest2"
 
     def test_register_connection_defaults(self):
-        """Ensure that defaults are used when the host and port are None.
+        """Ensure that defaults are used when the host and port are None."""
-        """
         register_connection("testdb", "mongoenginetest", host=None, port=None)
 
         conn = get_connection("testdb")

tests/test_connection_mongomock.py

@@ -0,0 +1,167 @@
+import unittest
+
+import pytest
+
+import mongoengine.connection
+from mongoengine import Document, StringField, connect, disconnect_all
+from mongoengine.connection import get_connection
+
+try:
+    import mongomock
+
+    MONGOMOCK_INSTALLED = True
+except ImportError:
+    MONGOMOCK_INSTALLED = False
+
+require_mongomock = pytest.mark.skipif(
+    not MONGOMOCK_INSTALLED, reason="you need mongomock installed to run this testcase"
+)
+
+
+class MongoMockConnectionTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        disconnect_all()
+
+    @classmethod
+    def tearDownClass(cls):
+        disconnect_all()
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    @require_mongomock
+    def test_connect_in_mocking(self):
+        """Ensure that the connect() method works properly in mocking."""
+        connect("mongoenginetest", host="mongomock://localhost")
+        conn = get_connection()
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect("mongoenginetest2", host="mongomock://localhost", alias="testdb2")
+        conn = get_connection("testdb2")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            "mongoenginetest3",
+            host="mongodb://localhost",
+            is_mock=True,
+            alias="testdb3",
+        )
+        conn = get_connection("testdb3")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect("mongoenginetest4", is_mock=True, alias="testdb4")
+        conn = get_connection("testdb4")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host="mongodb://localhost:27017/mongoenginetest5",
+            is_mock=True,
+            alias="testdb5",
+        )
+        conn = get_connection("testdb5")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host="mongomock://localhost:27017/mongoenginetest6", alias="testdb6")
+        conn = get_connection("testdb6")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host="mongomock://localhost:27017/mongoenginetest7",
+            is_mock=True,
+            alias="testdb7",
+        )
+        conn = get_connection("testdb7")
+        assert isinstance(conn, mongomock.MongoClient)
+
+    @require_mongomock
+    def test_default_database_with_mocking(self):
+        """Ensure that the default database is correctly set when using mongomock."""
+        disconnect_all()
+
+        class SomeDocument(Document):
+            pass
+
+        conn = connect(host="mongomock://localhost:27017/mongoenginetest")
+        some_document = SomeDocument()
+        # database won't exist until we save a document
+        some_document.save()
+        assert SomeDocument.objects.count() == 1
+        assert conn.get_default_database().name == "mongoenginetest"
+        assert conn.list_database_names()[0] == "mongoenginetest"
+
+    @require_mongomock
+    def test_basic_queries_against_mongomock(self):
+        disconnect_all()
+
+        connect(host="mongomock://localhost:27017/mongoenginetest")
+
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        assert Person.objects.count() == 0
+
+        bob = Person(name="Bob").save()
+        john = Person(name="John").save()
+        assert Person.objects.count() == 2
+
+        qs = Person.objects(name="Bob")
+        assert qs.count() == 1
+        assert qs.first() == bob
+        assert list(qs.as_pymongo()) == [{"_id": bob.id, "name": "Bob"}]
+
+        pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}]
+        data = Person.objects.order_by("name").aggregate(pipeline)
+        assert list(data) == [
+            {"_id": bob.id, "name": "BOB"},
+            {"_id": john.id, "name": "JOHN"},
+        ]
+
+        Person.drop_collection()
+        assert Person.objects.count() == 0
+
+    @require_mongomock
+    def test_connect_with_host_list(self):
+        """Ensure that the connect() method works when host is a list
+
+        Uses mongomock to test w/o needing multiple mongod/mongos processes
+        """
+        connect(host=["mongomock://localhost"])
+        conn = get_connection()
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host=["mongodb://localhost"], is_mock=True, alias="testdb2")
+        conn = get_connection("testdb2")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(host=["localhost"], is_mock=True, alias="testdb3")
+        conn = get_connection("testdb3")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["mongomock://localhost:27017", "mongomock://localhost:27018"],
+            alias="testdb4",
+        )
+        conn = get_connection("testdb4")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["mongodb://localhost:27017", "mongodb://localhost:27018"],
+            is_mock=True,
+            alias="testdb5",
+        )
+        conn = get_connection("testdb5")
+        assert isinstance(conn, mongomock.MongoClient)
+
+        connect(
+            host=["localhost:27017", "localhost:27018"], is_mock=True, alias="testdb6"
+        )
+        conn = get_connection("testdb6")
+        assert isinstance(conn, mongomock.MongoClient)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_context_managers.py

@@ -117,8 +117,7 @@ class TestContextManagers:
         assert 1 == Group.objects.count()
 
     def test_no_dereference_context_manager_object_id(self):
-        """Ensure that DBRef items in ListFields aren't dereferenced.
+        """Ensure that DBRef items in ListFields aren't dereferenced."""
-        """
         connect("mongoenginetest")
 
         class User(Document):
@@ -155,8 +154,7 @@ class TestContextManagers:
         assert isinstance(group.generic, User)
 
     def test_no_dereference_context_manager_dbref(self):
-        """Ensure that DBRef items in ListFields aren't dereferenced.
+        """Ensure that DBRef items in ListFields aren't dereferenced."""
-        """
         connect("mongoenginetest")
 
         class User(Document):
@@ -182,11 +180,11 @@ class TestContextManagers:
 
         with no_dereference(Group) as Group:
             group = Group.objects.first()
-            assert all([not isinstance(m, User) for m in group.members])
+            assert all(not isinstance(m, User) for m in group.members)
             assert not isinstance(group.ref, User)
             assert not isinstance(group.generic, User)
 
-        assert all([isinstance(m, User) for m in group.members])
+        assert all(isinstance(m, User) for m in group.members)
         assert isinstance(group.ref, User)
         assert isinstance(group.generic, User)
 

tests/test_datastructures.py

@@ -3,16 +3,24 @@ import unittest
 import pytest
 
 from mongoengine import Document
-from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    StrictDict,
+)
 
 
-class DocumentStub(object):
+class DocumentStub:
     def __init__(self):
         self._changed_fields = []
+        self._unset_fields = []
 
     def _mark_as_changed(self, key):
         self._changed_fields.append(key)
 
+    def _mark_as_unset(self, key):
+        self._unset_fields.append(key)
+
 
 class TestBaseDict:
     @staticmethod
@@ -314,7 +322,7 @@ class TestBaseList:
     def test___setitem___item_0_calls_mark_as_changed(self):
         base_list = self._get_baselist([True])
         base_list[0] = False
-        assert base_list._instance._changed_fields == ["my_name"]
+        assert base_list._instance._changed_fields == ["my_name.0"]
         assert base_list == [False]
 
     def test___setitem___item_1_calls_mark_as_changed(self):
@@ -417,7 +425,7 @@ class TestStrictDict(unittest.TestCase):
         d.a = 1
         assert d.a == 1
         with pytest.raises(AttributeError):
-            getattr(d, "b")
+            d.b
 
     def test_setattr_raises_on_nonexisting_attr(self):
         d = self.dtype()

tests/test_dereference.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import DBRef, ObjectId
@@ -17,8 +16,7 @@ class FieldTest(unittest.TestCase):
         cls.db.drop_database("mongoenginetest")
 
     def test_list_item_dereference(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -51,7 +49,7 @@ class FieldTest(unittest.TestCase):
             len(group_obj.members)
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Document select_related
@@ -60,7 +58,7 @@ class FieldTest(unittest.TestCase):
 
             group_obj = Group.objects.first().select_related()
             assert q == 2
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Queryset select_related
@@ -69,15 +67,14 @@ class FieldTest(unittest.TestCase):
             group_objs = Group.objects.select_related()
             assert q == 2
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
         User.drop_collection()
         Group.drop_collection()
 
     def test_list_item_dereference_dref_false(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -102,14 +99,14 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
             # verifies that no additional queries gets executed
             # if we re-iterate over the ListField once it is
             # dereferenced
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
@@ -120,7 +117,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
 
             assert q == 2
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
         # Queryset select_related
@@ -129,12 +126,11 @@ class FieldTest(unittest.TestCase):
             group_objs = Group.objects.select_related()
             assert q == 2
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
     def test_list_item_dereference_orphan_dbref(self):
-        """Ensure that orphan DBRef items in ListFields are dereferenced.
+        """Ensure that orphan DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -162,14 +158,14 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
             # verifies that no additional queries gets executed
             # if we re-iterate over the ListField once it is
             # dereferenced
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
             assert group_obj._data["members"]._dereferenced
 
@@ -177,8 +173,7 @@ class FieldTest(unittest.TestCase):
         Group.drop_collection()
 
     def test_list_item_dereference_dref_false_stores_as_type(self):
-        """Ensure that DBRef items are stored as their type
+        """Ensure that DBRef items are stored as their type"""
-        """
 
         class User(Document):
             my_id = IntField(primary_key=True)
@@ -199,8 +194,7 @@ class FieldTest(unittest.TestCase):
         assert group.members == [user]
 
     def test_handle_old_style_references(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -233,8 +227,7 @@ class FieldTest(unittest.TestCase):
         assert group.members[-1].name == "String!"
 
     def test_migrate_references(self):
-        """Example of migrating ReferenceField storage
+        """Example of migrating ReferenceField storage"""
-        """
 
         # Create some sample data
         class User(Document):
@@ -279,8 +272,7 @@ class FieldTest(unittest.TestCase):
         assert isinstance(raw_data["members"][0], ObjectId)
 
     def test_recursive_reference(self):
-        """Ensure that ReferenceFields can reference their own documents.
+        """Ensure that ReferenceFields can reference their own documents."""
-        """
 
         class Employee(Document):
             name = StringField()
@@ -370,8 +362,7 @@ class FieldTest(unittest.TestCase):
         assert Post.objects.all()[0].user_lists == [[u1, u2], [u3]]
 
     def test_circular_reference(self):
-        """Ensure you can handle circular references
+        """Ensure you can handle circular references"""
-        """
 
         class Relation(EmbeddedDocument):
             name = StringField()
@@ -404,8 +395,7 @@ class FieldTest(unittest.TestCase):
         assert "[<Person: Mother>, <Person: Daughter>]" == "%s" % Person.objects()
 
     def test_circular_reference_on_self(self):
-        """Ensure you can handle circular references
+        """Ensure you can handle circular references"""
-        """
 
         class Person(Document):
             name = StringField()
@@ -426,13 +416,13 @@ class FieldTest(unittest.TestCase):
 
         daughter.relations.append(mother)
         daughter.relations.append(daughter)
+        assert daughter._get_changed_fields() == ["relations"]
         daughter.save()
 
         assert "[<Person: Mother>, <Person: Daughter>]" == "%s" % Person.objects()
 
     def test_circular_tree_reference(self):
-        """Ensure you can handle circular references with more than one level
+        """Ensure you can handle circular references with more than one level"""
-        """
 
         class Other(EmbeddedDocument):
             name = StringField()
@@ -515,10 +505,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -531,10 +521,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -548,18 +538,17 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for m in group_obj.members:
                     assert "User" in m.__class__.__name__
 
     def test_generic_reference_orphan_dbref(self):
-        """Ensure that generic orphan DBRef items in ListFields are dereferenced.
+        """Ensure that generic orphan DBRef items in ListFields are dereferenced."""
-        """
 
         class UserA(Document):
             name = StringField()
@@ -603,11 +592,11 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
             assert group_obj._data["members"]._dereferenced
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
             assert group_obj._data["members"]._dereferenced
 
@@ -659,10 +648,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -675,10 +664,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for m in group_obj.members:
@@ -692,10 +681,10 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for m in group_obj.members:
@@ -734,10 +723,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert isinstance(m, User)
 
         # Document select_related
@@ -747,7 +736,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -761,7 +750,7 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
                 for k, m in group_obj.members.items():
@@ -812,10 +801,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for k, m in group_obj.members.items():
@@ -828,10 +817,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
             for k, m in group_obj.members.items():
@@ -845,10 +834,10 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
                 for k, m in group_obj.members.items():
@@ -863,7 +852,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 1
             assert group_obj.members == {}
 
@@ -902,10 +891,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -918,10 +907,10 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 2
 
             for k, m in group_obj.members.items():
@@ -935,13 +924,13 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 2
 
-                for k, m in group_obj.members.items():
+                for _, m in group_obj.members.items():
                     assert isinstance(m, UserA)
 
         UserA.drop_collection()
@@ -989,13 +978,13 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert "User" in m.__class__.__name__
 
         # Document select_related
@@ -1005,13 +994,13 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first().select_related()
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 4
 
-            for k, m in group_obj.members.items():
+            for _, m in group_obj.members.items():
                 assert "User" in m.__class__.__name__
 
         # Queryset select_related
@@ -1022,13 +1011,13 @@ class FieldTest(unittest.TestCase):
             assert q == 4
 
             for group_obj in group_objs:
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                [m for m in group_obj.members]
+                _ = [m for m in group_obj.members]
                 assert q == 4
 
-                for k, m in group_obj.members.items():
+                for _, m in group_obj.members.items():
                     assert "User" in m.__class__.__name__
 
         Group.objects.delete()
@@ -1040,7 +1029,7 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             assert q == 1
 
-            [m for m in group_obj.members]
+            _ = [m for m in group_obj.members]
             assert q == 1
 
         UserA.drop_collection()
@@ -1169,8 +1158,7 @@ class FieldTest(unittest.TestCase):
         assert msg.author.name == "new-name"
 
     def test_list_lookup_not_checked_in_map(self):
-        """Ensure we dereference list data correctly
+        """Ensure we dereference list data correctly"""
-        """
 
         class Comment(Document):
             id = IntField(primary_key=True)
@@ -1192,8 +1180,7 @@ class FieldTest(unittest.TestCase):
         assert 1 == msg.comments[1].id
 
     def test_list_item_dereference_dref_false_save_doesnt_cause_extra_queries(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -1222,8 +1209,7 @@ class FieldTest(unittest.TestCase):
             assert q == 2
 
     def test_list_item_dereference_dref_true_save_doesnt_cause_extra_queries(self):
-        """Ensure that DBRef items in ListFields are dereferenced.
+        """Ensure that DBRef items in ListFields are dereferenced."""
-        """
 
         class User(Document):
             name = StringField()
@@ -1335,7 +1321,7 @@ class FieldTest(unittest.TestCase):
         BrandGroup.drop_collection()
 
         brand1 = Brand(title="Moschino").save()
-        brand2 = Brand(title=u"Денис Симачёв").save()
+        brand2 = Brand(title="Денис Симачёв").save()
 
         BrandGroup(title="top_brands", brands=[brand1, brand2]).save()
         brand_groups = BrandGroup.objects().all()

tests/test_replicaset_connection.py

@@ -5,7 +5,6 @@ from pymongo import MongoClient, ReadPreference
 import mongoengine
 from mongoengine.connection import ConnectionFailure
 
-
 CONN_CLASS = MongoClient
 READ_PREF = ReadPreference.SECONDARY
 
@@ -22,8 +21,7 @@ class ConnectionTest(unittest.TestCase):
         mongoengine.connection._dbs = {}
 
     def test_replicaset_uri_passes_read_preference(self):
-        """Requires a replica set called "rs" on port 27017
+        """Requires a replica set called "rs" on port 27017"""
-        """
         try:
             conn = mongoengine.connect(
                 db="mongoenginetest",

tests/test_signals.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *
@@ -250,7 +249,7 @@ class TestSignal(unittest.TestCase):
         assert self.pre_signals == post_signals
 
     def test_model_signals(self):
-        """ Model saves should throw some signals. """
+        """Model saves should throw some signals."""
 
         def create_author():
             self.Author(name="Bill Shakespeare")
@@ -341,7 +340,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_signal_kwargs(self):
-        """ Make sure signal_kwargs is passed to signals calls. """
+        """Make sure signal_kwargs is passed to signals calls."""
 
         def live_and_let_die():
             a = self.Author(name="Bill Shakespeare")
@@ -386,7 +385,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_queryset_delete_signals(self):
-        """ Queryset delete should throw some signals. """
+        """Queryset delete should throw some signals."""
 
         self.Another(name="Bill Shakespeare").save()
         assert self.get_signal_output(self.Another.objects.delete) == [
@@ -397,7 +396,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_signals_with_explicit_doc_ids(self):
-        """ Model saves must have a created flag the first time."""
+        """Model saves must have a created flag the first time."""
         ei = self.ExplicitId(id=123)
         # post save must received the created flag, even if there's already
         # an object id present

tests/test_utils.py

@@ -1,5 +1,4 @@
 import re
-import unittest
 
 import pytest
 
@@ -10,7 +9,7 @@ signal_output = []
 
 class TestLazyRegexCompiler:
     def test_lazy_regex_compiler_verify_laziness_of_descriptor(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@", flags=32)
 
         descriptor = UserEmail.__dict__["EMAIL_REGEX"]
@@ -24,7 +23,7 @@ class TestLazyRegexCompiler:
         assert user_email.EMAIL_REGEX is UserEmail.EMAIL_REGEX
 
     def test_lazy_regex_compiler_verify_cannot_set_descriptor_on_instance(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@")
 
         user_email = UserEmail()
@@ -32,7 +31,7 @@ class TestLazyRegexCompiler:
             user_email.EMAIL_REGEX = re.compile("@")
 
     def test_lazy_regex_compiler_verify_can_override_class_attr(self):
-        class UserEmail(object):
+        class UserEmail:
             EMAIL_REGEX = LazyRegexCompiler("@")
 
         UserEmail.EMAIL_REGEX = re.compile("cookies")

tests/utils.py

@@ -1,3 +1,4 @@
+import operator
 import unittest
 
 import pytest
@@ -6,7 +7,6 @@ from mongoengine import connect
 from mongoengine.connection import disconnect_all, get_db
 from mongoengine.mongodb_support import get_mongodb_version
 
-
 MONGO_TEST_DB = "mongoenginetest"  # standard name for the test database
 
 
@@ -33,6 +33,14 @@ def get_as_pymongo(doc):
     return doc.__class__.objects.as_pymongo().get(id=doc.id)
 
 
+def requires_mongodb_lt_42(func):
+    return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt)
+
+
+def requires_mongodb_gte_44(func):
+    return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge)
+
+
 def _decorated_with_ver_requirement(func, mongo_version_req, oper):
     """Return a MongoDB version requirement decorator.
 
@@ -50,7 +58,7 @@ def _decorated_with_ver_requirement(func, mongo_version_req, oper):
     ran against MongoDB < v3.6.
 
     :param mongo_version_req: The mongodb version requirement (tuple(int, int))
-    :param oper: The operator to apply (e.g: operator.ge)
+    :param oper: The operator to apply (e.g. operator.ge)
     """
 
     def _inner(*args, **kwargs):
@@ -59,7 +67,7 @@ def _decorated_with_ver_requirement(func, mongo_version_req, oper):
             return func(*args, **kwargs)
 
         pretty_version = ".".join(str(n) for n in mongo_version_req)
-        pytest.skip("Needs MongoDB v{}+".format(pretty_version))
+        pytest.skip(f"Needs MongoDB v{pretty_version}+")
 
     _inner.__name__ = func.__name__
     _inner.__doc__ = func.__doc__

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py35,pypy3}-{mg34,mg36,mg39,mg310}
+envlist = pypy3-{mg34,mg36,mg39,mg311}
 
 [testenv]
 commands =
@@ -8,6 +8,6 @@ deps =
     mg34: pymongo>=3.4,<3.5
     mg36: pymongo>=3.6,<3.7
     mg39: pymongo>=3.9,<3.10
-    mg310: pymongo>=3.10,<3.11
+    mg311: pymongo>=3.11,<3.12
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs