Merge pull request #2513 from bagerard/prepare_release

Prepare release

.github/workflows/github-actions.yml

@@ -11,9 +11,10 @@ on:
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+*'
 env:
-  MONGODB_3_4: 3.4.19
-  MONGODB_3_6: 3.6.13
-  MONGODB_4_0: 4.0.13
+  MONGODB_3_6: 3.6.14
+  MONGODB_4_0: 4.0.23
+  MONGODB_4_2: 4.2
+  MONGODB_4_4: 4.4
 
   PYMONGO_3_4: 3.4
   PYMONGO_3_6: 3.6
@@ -47,14 +48,14 @@ jobs:
         MONGODB: [$MONGODB_4_0]
         PYMONGO: [$PYMONGO_3_11]
         include:
-          - python-version: 3.7
-            MONGODB: $MONGODB_3_4
-            PYMONGO: $PYMONGO_3_6
           - python-version: 3.7
             MONGODB: $MONGODB_3_6
             PYMONGO: $PYMONGO_3_9
           - python-version: 3.7
-            MONGODB: $MONGODB_3_6
+            MONGODB: $MONGODB_4_2
+            PYMONGO: $PYMONGO_3_6
+          - python-version: 3.7
+            MONGODB: $MONGODB_4_4
             PYMONGO: $PYMONGO_3_11
     steps:
     - uses: actions/checkout@v2
@@ -74,6 +75,7 @@ jobs:
     - name: Send coverage to Coveralls
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COVERALLS_SERVICE_NAME: github
       if: ${{ matrix.python-version == env.MAIN_PYTHON_VERSION }}
       run: coveralls
 
@@ -138,4 +140,4 @@ jobs:
     - name: publish pypi
       uses: pypa/gh-action-pypi-publish@master
       with:
-        password: ${{ secrets.pypi_token }}
+        password: ${{ secrets.pypi_token }}

.github/workflows/install_mongo.sh

@@ -2,7 +2,17 @@
 
 MONGODB=$1
 
+# Mongo > 4.0 follows different name convention for download links
 mongo_build=mongodb-linux-x86_64-${MONGODB}
+
+if [[ "$MONGODB" == *"4.2"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+elif [[ "$MONGODB" == *"4.4"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+fi
+
 wget http://fastdl.mongodb.org/linux/$mongo_build.tgz
 tar xzf $mongo_build.tgz
-${PWD}/$mongo_build/bin/mongod --version
+
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+$mongodb_dir/bin/mongod --version

.github/workflows/start_mongo.sh

@@ -2,7 +2,8 @@
 
 MONGODB=$1
 
-mongodb_dir=${PWD}/mongodb-linux-x86_64-${MONGODB}
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+
 mkdir $mongodb_dir/data
 $mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork
 mongo --eval 'db.version();'    # Make sure mongo is awake

.pre-commit-config.yaml

@@ -1,17 +1,26 @@
 fail_fast: false
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
   - repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 21.4b2
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.1
     hooks:
       - id: flake8
-        additional_dependencies:
-          - flake8-import-order
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.4
+    rev: v2.14.0
     hooks:
       - id: pyupgrade
         args: [--py36-plus]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.8.0
+    hooks:
+      - id: isort

AUTHORS

@@ -259,3 +259,4 @@ that much better:
  * Agustin Barto (https://github.com/abarto)
  * Stankiewicz Mateusz (https://github.com/mas15)
  * Felix Schultheiß (https://github.com/felix-smashdocs)
+ * Jan Stein (https://github.com/janste63)

CONTRIBUTING.rst

@@ -35,8 +35,8 @@ Travis runs the tests against the main Python 3.x versions.
 Style Guide
 -----------
 
-MongoEngine's codebase is formatted with `black <https://github.com/python/black>`_, other tools like
-flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
+MongoEngine's codebase is auto-formatted with `black <https://github.com/python/black>`_, imports are ordered with `isort <https://pycqa.github.io/isort/>`_
+and other tools like flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
 
 To install all development tools, simply run the following commands:
 
@@ -58,6 +58,10 @@ To enable ``pre-commit`` simply run:
 See the ``.pre-commit-config.yaml`` configuration file for more information
 on how it works.
 
+pre-commit will now run upon every commit and will reject anything that doesn't comply.
+
+You can also run all the checks with ``pre-commit run -a``, this is what is used in the CI.
+
 Testing
 -------
 

benchmarks/test_inserts.py

@@ -31,7 +31,7 @@ myNoddys = noddy.find()
     print("-" * 100)
     print("PyMongo: Creating 10000 dictionaries.")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 from pymongo import MongoClient, WriteConcern
@@ -54,7 +54,7 @@ myNoddys = noddy.find()
     print("-" * 100)
     print('PyMongo: Creating 10000 dictionaries (write_concern={"w": 0}).')
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     setup = """
 from pymongo import MongoClient
@@ -84,7 +84,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print("MongoEngine: Creating 10000 dictionaries.")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -102,7 +102,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print("MongoEngine: Creating 10000 dictionaries (using a single field assignment).")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -118,7 +118,7 @@ myNoddys = Noddy.objects()
     print("-" * 100)
     print('MongoEngine: Creating 10000 dictionaries (write_concern={"w": 0}).')
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -136,7 +136,7 @@ myNoddys = Noddy.objects()
         'MongoEngine: Creating 10000 dictionaries (write_concern={"w": 0}, validate=False).'
     )
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
     stmt = """
 for i in range(10000):
@@ -154,7 +154,7 @@ myNoddys = Noddy.objects()
         'MongoEngine: Creating 10000 dictionaries (force_insert=True, write_concern={"w": 0}, validate=False).'
     )
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print("{}s".format(t.timeit(1)))
+    print(f"{t.timeit(1)}s")
 
 
 if __name__ == "__main__":

docs/changelog.rst

@@ -1,4 +1,5 @@
 
+
 =========
 Changelog
 =========
@@ -7,6 +8,17 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 
+Changes in 0.23.1
+===========
+- Bug fix: ignore LazyReferenceFields when clearing _changed_fields #2484
+- Improve connection doc #2481
+
+Changes in 0.23.0
+=================
+- Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471
+- Add MongoDB 4.2 and 4.4 to CI
+- Add support for allowDiskUse on querysets #2468
+
 Changes in 0.22.1
 =================
 - Declare that Py3.5 is not supported in package metadata #2449

docs/guide/connecting.rst

@@ -5,7 +5,7 @@ Connecting to MongoDB
 =====================
 
 Connections in MongoEngine are registered globally and are identified with aliases.
-If no `alias` is provided during the connection, it will use "default" as alias.
+If no ``alias`` is provided during the connection, it will use "default" as alias.
 
 To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect`
 function. The first argument is the name of the database to connect to::
@@ -14,27 +14,66 @@ function. The first argument is the name of the database to connect to::
     connect('project1')
 
 By default, MongoEngine assumes that the :program:`mongod` instance is running
-on **localhost** on port **27017**. If MongoDB is running elsewhere, you should
-provide the :attr:`host` and :attr:`port` arguments to
-:func:`~mongoengine.connect`::
+on **localhost** on port **27017**.
 
-    connect('project1', host='192.168.1.35', port=12345)
+If MongoDB is running elsewhere, you need to provide details on how to connect. There are two ways of
+doing this. Using a connection string in URI format (**this is the preferred method**) or individual attributes
+provided as keyword arguments.
+
+Connect with URI string
+=======================
+
+When using a connection string in URI format you should specify the connection details
+as the :attr:`host` to :func:`~mongoengine.connect`. In a web application context for instance, the URI
+is typically read from the config file::
+
+        connect(host="mongodb://127.0.0.1:27017/my_db")
+
+If the database requires authentication, you can specify it in the
+URI. As each database can have its own users configured, you need to tell MongoDB
+where to look for the user you are working with, that's what the ``?authSource=admin`` bit
+of the MongoDB connection string is for::
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against the 'admin' database (by default as authSource isn't provided)
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db")
+
+    # Equivalent to previous connection but explicitly states that
+    # it should use admin as the authentication source database
+    connect(host="mongodb://my_user:my_password@hostname:port/my_db?authSource=admin")
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against that same database
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=my_db")
+
+The URI string can also be used to configure advanced parameters like ssl, replicaSet, etc. For more
+information or example about URI string, you can refer to the `official doc <https://docs.mongodb.com/manual/reference/connection-string/>`_::
+
+    connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=admin&ssl=true&replicaSet=globaldb")
+
+.. note:: URI containing SRV records (e.g "mongodb+srv://server.example.com/") can be used as well
+
+Connect with keyword attributes
+===============================
+
+The second option for specifying the connection details is to provide the information as keyword
+attributes to :func:`~mongoengine.connect`::
+
+    connect('my_db', host='127.0.0.1', port=27017)
 
 If the database requires authentication, :attr:`username`, :attr:`password`
 and :attr:`authentication_source` arguments should be provided::
 
-    connect('project1', username='webapp', password='pwd123', authentication_source='admin')
+    connect('my_db', username='my_user', password='my_password', authentication_source='admin')
 
-URI style connections are also supported -- just supply the URI as
-the :attr:`host` to
-:func:`~mongoengine.connect`::
-
-    connect('project1', host='mongodb://localhost/database_name')
-
-.. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host`
+The set of attributes that :func:`~mongoengine.connect` recognizes includes but is not limited to:
+:attr:`host`, :attr:`port`, :attr:`read_preference`, :attr:`username`, :attr:`password`, :attr:`authentication_source`, :attr:`authentication_mechanism`,
+:attr:`replicaset`, :attr:`tls`, etc. Most of the parameters accepted by `pymongo.MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_
+can be used with :func:`~mongoengine.connect` and will simply be forwarded when instantiating the `pymongo.MongoClient`.
 
 .. note:: Database, username and password from URI string overrides
-    corresponding parameters in :func:`~mongoengine.connect`: ::
+    corresponding parameters in :func:`~mongoengine.connect`, this should
+    obviously be avoided: ::
 
         connect(
             db='test',
@@ -43,28 +82,19 @@ the :attr:`host` to
             host='mongodb://admin:qwerty@localhost/production'
         )
 
-    will establish connection to ``production`` database using
-    ``admin`` username and ``qwerty`` password.
+    will establish connection to ``production`` database using ``admin`` username and ``qwerty`` password.
 
 .. note:: Calling :func:`~mongoengine.connect` without argument will establish
     a connection to the "test" database by default
 
-Replica Sets
-============
+Read Preferences
+================
 
-MongoEngine supports connecting to replica sets::
-
-    from mongoengine import connect
-
-    # Regular connect
-    connect('dbname', replicaset='rs-name')
-
-    # MongoDB URI-style connect
-    connect(host='mongodb://localhost/dbname?replicaSet=rs-name')
-
-Read preferences are supported through the connection or via individual
+As stated above, Read preferences are supported through the connection but also via individual
 queries by passing the read_preference ::
 
+    from pymongo import ReadPreference
+
     Bar.objects().read_preference(ReadPreference.PRIMARY)
     Bar.objects(read_preference=ReadPreference.PRIMARY)
 

docs/guide/defining-documents.rst

@@ -290,12 +290,12 @@ as the constructor's argument::
         content = StringField()
 
 
-.. _one-to-many-with-listfields:
+.. _many-to-many-with-listfields:
 
-One to Many with ListFields
+Many to Many with ListFields
 '''''''''''''''''''''''''''
 
-If you are implementing a one to many relationship via a list of references,
+If you are implementing a many to many relationship via a list of references,
 then the references are stored as DBRefs and to query you need to pass an
 instance of the object to the query::
 
@@ -432,10 +432,10 @@ Document collections
 ====================
 Document classes that inherit **directly** from :class:`~mongoengine.Document`
 will have their own **collection** in the database. The name of the collection
-is by default the name of the class, converted to lowercase (so in the example
-above, the collection would be called `page`). If you need to change the name
-of the collection (e.g. to use MongoEngine with an existing database), then
-create a class dictionary attribute called :attr:`meta` on your document, and
+is by default the name of the class converted to snake_case (e.g if your Document class
+is named `CompanyUser`, the corresponding collection would be `company_user`). If you need
+to change the name of the collection (e.g. to use MongoEngine with an existing database),
+then create a class dictionary attribute called :attr:`meta` on your document, and
 set :attr:`collection` to the name of the collection that you want your
 document class to use::
 

docs/guide/migration.rst

@@ -35,7 +35,7 @@ existing ``User`` model with a `default=True`. Thus you simply update the ``User
 
     class User(Document):
         name = StringField(required=True)
-        enabled = BooleaField(default=True)
+        enabled = BooleanField(default=True)
 
 Without applying any migration, we now reload an object from the database into the ``User`` class
 and checks its `enabled` attribute:

docs/guide/validation.rst

@@ -120,4 +120,3 @@ the validation and cleaning of a document when you call :meth:`~mongoengine.docu
     Person(age=1000).save(validate=False)
     person = Person.objects.first()
     assert person.age == 1000
-

mongoengine/__init__.py

@@ -1,10 +1,12 @@
 # Import submodules so that we can expose their __all__
-from mongoengine import connection
-from mongoengine import document
-from mongoengine import errors
-from mongoengine import fields
-from mongoengine import queryset
-from mongoengine import signals
+from mongoengine import (
+    connection,
+    document,
+    errors,
+    fields,
+    queryset,
+    signals,
+)
 
 # Import everything from each submodule so that it can be accessed via
 # mongoengine, e.g. instead of `from mongoengine.connection import connect`,
@@ -17,7 +19,6 @@ from mongoengine.fields import *  # noqa: F401
 from mongoengine.queryset import *  # noqa: F401
 from mongoengine.signals import *  # noqa: F401
 
-
 __all__ = (
     list(document.__all__)
     + list(fields.__all__)
@@ -28,7 +29,7 @@ __all__ = (
 )
 
 
-VERSION = (0, 22, 1)
+VERSION = (0, 23, 1)
 
 
 def get_version():

mongoengine/base/document.py

@@ -1,10 +1,9 @@
 import copy
-
 import numbers
 from functools import partial
 
-from bson import DBRef, ObjectId, SON, json_util
 import pymongo
+from bson import SON, DBRef, ObjectId, json_util
 
 from mongoengine import signals
 from mongoengine.base.common import get_document
@@ -156,7 +155,7 @@ class BaseDocument:
         # Handle dynamic data only if an initialised dynamic document
         if self._dynamic and not self._dynamic_lock:
 
-            if not hasattr(self, name) and not name.startswith("_"):
+            if name not in self._fields_ordered and not name.startswith("_"):
                 DynamicField = _import_class("DynamicField")
                 field = DynamicField(db_field=name, null=True)
                 field.name = name
@@ -615,7 +614,9 @@ class BaseDocument:
     def _get_changed_fields(self):
         """Return a list of all fields that have explicitly been changed."""
         EmbeddedDocument = _import_class("EmbeddedDocument")
+        LazyReferenceField = _import_class("LazyReferenceField")
         ReferenceField = _import_class("ReferenceField")
+        GenericLazyReferenceField = _import_class("GenericLazyReferenceField")
         GenericReferenceField = _import_class("GenericReferenceField")
         SortedListField = _import_class("SortedListField")
 
@@ -641,7 +642,13 @@ class BaseDocument:
                 changed_fields += [f"{key}{k}" for k in changed if k]
             elif isinstance(data, (list, tuple, dict)):
                 if hasattr(field, "field") and isinstance(
-                    field.field, (ReferenceField, GenericReferenceField)
+                    field.field,
+                    (
+                        LazyReferenceField,
+                        ReferenceField,
+                        GenericLazyReferenceField,
+                        GenericReferenceField,
+                    ),
                 ):
                     continue
                 elif isinstance(field, SortedListField) and field._ordering:

mongoengine/base/fields.py

@@ -1,11 +1,15 @@
 import operator
 import weakref
 
-from bson import DBRef, ObjectId, SON
 import pymongo
+from bson import SON, DBRef, ObjectId
 
 from mongoengine.base.common import UPDATE_OPERATORS
-from mongoengine.base.datastructures import BaseDict, BaseList, EmbeddedDocumentList
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    EmbeddedDocumentList,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import DeprecatedError, ValidationError
 
@@ -267,6 +271,17 @@ class ComplexBaseField(BaseField):
         self.field = field
         super().__init__(**kwargs)
 
+    @staticmethod
+    def _lazy_load_refs(instance, name, ref_values, *, max_depth):
+        _dereference = _import_class("DeReference")()
+        documents = _dereference(
+            ref_values,
+            max_depth=max_depth,
+            instance=instance,
+            name=name,
+        )
+        return documents
+
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references."""
         if instance is None:
@@ -284,19 +299,15 @@ class ComplexBaseField(BaseField):
             or isinstance(self.field, (GenericReferenceField, ReferenceField))
         )
 
-        _dereference = _import_class("DeReference")()
-
         if (
             instance._initialised
             and dereference
             and instance._data.get(self.name)
             and not getattr(instance._data[self.name], "_dereferenced", False)
         ):
-            instance._data[self.name] = _dereference(
-                instance._data.get(self.name),
-                max_depth=1,
-                instance=instance,
-                name=self.name,
+            ref_values = instance._data.get(self.name)
+            instance._data[self.name] = self._lazy_load_refs(
+                ref_values=ref_values, instance=instance, name=self.name, max_depth=1
             )
             if hasattr(instance._data[self.name], "_dereferenced"):
                 instance._data[self.name]._dereferenced = True
@@ -322,7 +333,9 @@ class ComplexBaseField(BaseField):
             and isinstance(value, (BaseList, BaseDict))
             and not value._dereferenced
         ):
-            value = _dereference(value, max_depth=1, instance=instance, name=self.name)
+            value = self._lazy_load_refs(
+                ref_values=value, instance=instance, name=self.name, max_depth=1
+            )
             value._dereferenced = True
             instance._data[self.name] = value
 

mongoengine/base/metaclasses.py

@@ -2,7 +2,11 @@ import itertools
 import warnings
 
 from mongoengine.base.common import _document_registry
-from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
+from mongoengine.base.fields import (
+    BaseField,
+    ComplexBaseField,
+    ObjectIdField,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidDocumentError
 from mongoengine.queryset import (
@@ -12,7 +16,6 @@ from mongoengine.queryset import (
     QuerySetManager,
 )
 
-
 __all__ = ("DocumentMetaclass", "TopLevelDocumentMetaclass")
 
 

mongoengine/connection.py

@@ -217,8 +217,8 @@ def register_connection(
 
 def disconnect(alias=DEFAULT_CONNECTION_NAME):
     """Close the connection with a given alias."""
-    from mongoengine.base.common import _get_documents_by_db
     from mongoengine import Document
+    from mongoengine.base.common import _get_documents_by_db
 
     if alias in _connections:
         get_connection(alias=alias).close()

mongoengine/dereference.py

@@ -1,4 +1,4 @@
-from bson import DBRef, SON
+from bson import SON, DBRef
 
 from mongoengine.base import (
     BaseDict,
@@ -10,7 +10,12 @@ from mongoengine.base import (
 from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.fields import DictField, ListField, MapField, ReferenceField
+from mongoengine.fields import (
+    DictField,
+    ListField,
+    MapField,
+    ReferenceField,
+)
 from mongoengine.queryset import QuerySet
 
 

mongoengine/document.py

@@ -1,7 +1,7 @@
 import re
 
-from bson.dbref import DBRef
 import pymongo
+from bson.dbref import DBRef
 from pymongo.read_preferences import ReadPreference
 
 from mongoengine import signals
@@ -16,14 +16,23 @@ from mongoengine.base import (
 )
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from mongoengine.context_managers import set_write_concern, switch_collection, switch_db
+from mongoengine.context_managers import (
+    set_write_concern,
+    switch_collection,
+    switch_db,
+)
 from mongoengine.errors import (
     InvalidDocumentError,
     InvalidQueryError,
     SaveConditionError,
 )
 from mongoengine.pymongo_support import list_collection_names
-from mongoengine.queryset import NotUniqueError, OperationError, QuerySet, transform
+from mongoengine.queryset import (
+    NotUniqueError,
+    OperationError,
+    QuerySet,
+    transform,
+)
 
 __all__ = (
     "Document",
@@ -109,7 +118,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     By default, the MongoDB collection used to store documents created using a
     :class:`~mongoengine.Document` subclass will be the name of the subclass
-    converted to lowercase. A different collection may be specified by
+    converted to snake_case. A different collection may be specified by
     providing :attr:`collection` to the :attr:`meta` dictionary in the class
     definition.
 
@@ -323,7 +332,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
         _refs=None,
         save_condition=None,
         signal_kwargs=None,
-        **kwargs
+        **kwargs,
     ):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
@@ -554,7 +563,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
             if not getattr(ref, "_changed_fields", True):
                 continue
 
-            ref_id = "{},{}".format(ref.__class__.__name__, str(ref._data))
+            ref_id = f"{ref.__class__.__name__},{str(ref._data)}"
             if ref and ref_id not in _refs:
                 _refs.append(ref_id)
                 kwargs["_refs"] = _refs

mongoengine/errors.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 
-
 __all__ = (
     "NotRegistered",
     "InvalidDocumentError",

mongoengine/fields.py

@@ -1,5 +1,6 @@
 import datetime
 import decimal
+import inspect
 import itertools
 import re
 import socket
@@ -8,10 +9,10 @@ import uuid
 from io import BytesIO
 from operator import itemgetter
 
-from bson import Binary, DBRef, ObjectId, SON
-from bson.int64 import Int64
 import gridfs
 import pymongo
+from bson import SON, Binary, DBRef, ObjectId
+from bson.int64 import Int64
 from pymongo import ReturnDocument
 
 try:
@@ -21,7 +22,6 @@ except ImportError:
 else:
     import dateutil.parser
 
-
 from mongoengine.base import (
     BaseDocument,
     BaseField,
@@ -35,7 +35,11 @@ from mongoengine.base.utils import LazyRegexCompiler
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
+from mongoengine.errors import (
+    DoesNotExist,
+    InvalidQueryError,
+    ValidationError,
+)
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
 from mongoengine.queryset.transform import STRING_OPERATORS
@@ -514,7 +518,7 @@ class BooleanField(BaseField):
     def to_python(self, value):
         try:
             value = bool(value)
-        except ValueError:
+        except (ValueError, TypeError):
             pass
         return value
 
@@ -914,7 +918,7 @@ class ListField(ComplexBaseField):
     """A list field that wraps a standard field, allowing multiple instances
     of the field to be used as a list in the database.
 
-    If using with ReferenceFields see: :ref:`one-to-many-with-listfields`
+    If using with ReferenceFields see: :ref:`many-to-many-with-listfields`
 
     .. note::
         Required means it cannot be empty - as the default for ListFields is []
@@ -1028,17 +1032,6 @@ def key_not_string(d):
             return True
 
 
-def key_has_dot_or_dollar(d):
-    """Helper function to recursively determine if any key in a
-    dictionary contains a dot or a dollar sign.
-    """
-    for k, v in d.items():
-        if ("." in k or k.startswith("$")) or (
-            isinstance(v, dict) and key_has_dot_or_dollar(v)
-        ):
-            return True
-
-
 def key_starts_with_dollar(d):
     """Helper function to recursively determine if any key in a
     dictionary starts with a dollar
@@ -1172,7 +1165,7 @@ class ReferenceField(BaseField):
 
         :param document_type: The type of Document that will be referenced
         :param dbref:  Store the reference as :class:`~pymongo.dbref.DBRef`
-          or as the :class:`~pymongo.objectid.ObjectId`.id .
+          or as the :class:`~pymongo.objectid.ObjectId`.
         :param reverse_delete_rule: Determines what to do when the referring
           object is deleted
         :param kwargs: Keyword arguments passed into the parent :class:`~mongoengine.BaseField`
@@ -1204,6 +1197,14 @@ class ReferenceField(BaseField):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         """Descriptor to allow lazy dereferencing."""
         if instance is None:
@@ -1211,20 +1212,17 @@ class ReferenceField(BaseField):
             return self
 
         # Get value from document instance if available
-        value = instance._data.get(self.name)
+        ref_value = instance._data.get(self.name)
         auto_dereference = instance._fields[self.name]._auto_dereference
         # Dereference DBRefs
-        if auto_dereference and isinstance(value, DBRef):
-            if hasattr(value, "cls"):
+        if auto_dereference and isinstance(ref_value, DBRef):
+            if hasattr(ref_value, "cls"):
                 # Dereference using the class type specified in the reference
-                cls = get_document(value.cls)
+                cls = get_document(ref_value.cls)
             else:
                 cls = self.document_type
-            dereferenced = cls._get_db().dereference(value)
-            if dereferenced is None:
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = cls._from_son(dereferenced)
+
+            instance._data[self.name] = self._lazy_load_ref(cls, ref_value)
 
         return super().__get__(instance, owner)
 
@@ -1311,8 +1309,8 @@ class CachedReferenceField(BaseField):
             fields = []
 
         # XXX ValidationError raised outside of the "validate" method.
-        if not isinstance(document_type, str) and not issubclass(
-            document_type, Document
+        if not isinstance(document_type, str) and not (
+            inspect.isclass(document_type) and issubclass(document_type, Document)
         ):
             self.error(
                 "Argument to CachedReferenceField constructor must be a"
@@ -1363,6 +1361,14 @@ class CachedReferenceField(BaseField):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         if instance is None:
             # Document class being used rather than a document object
@@ -1374,11 +1380,7 @@ class CachedReferenceField(BaseField):
 
         # Dereference DBRefs
         if auto_dereference and isinstance(value, DBRef):
-            dereferenced = self.document_type._get_db().dereference(value)
-            if dereferenced is None:
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = self.document_type._from_son(dereferenced)
+            instance._data[self.name] = self._lazy_load_ref(self.document_type, value)
 
         return super().__get__(instance, owner)
 
@@ -1503,6 +1505,14 @@ class GenericReferenceField(BaseField):
             value = value._class_name
         super()._validate_choices(value)
 
+    @staticmethod
+    def _lazy_load_ref(ref_cls, dbref):
+        dereferenced_son = ref_cls._get_db().dereference(dbref)
+        if dereferenced_son is None:
+            raise DoesNotExist(f"Trying to dereference unknown document {dbref}")
+
+        return ref_cls._from_son(dereferenced_son)
+
     def __get__(self, instance, owner):
         if instance is None:
             return self
@@ -1510,12 +1520,9 @@ class GenericReferenceField(BaseField):
         value = instance._data.get(self.name)
 
         auto_dereference = instance._fields[self.name]._auto_dereference
-        if auto_dereference and isinstance(value, (dict, SON)):
-            dereferenced = self.dereference(value)
-            if dereferenced is None:
-                raise DoesNotExist("Trying to dereference unknown document %s" % value)
-            else:
-                instance._data[self.name] = dereferenced
+        if auto_dereference and isinstance(value, dict):
+            doc_cls = get_document(value["_cls"])
+            instance._data[self.name] = self._lazy_load_ref(doc_cls, value["_ref"])
 
         return super().__get__(instance, owner)
 
@@ -1534,14 +1541,6 @@ class GenericReferenceField(BaseField):
                 " saved to the database"
             )
 
-    def dereference(self, value):
-        doc_cls = get_document(value["_cls"])
-        reference = value["_ref"]
-        doc = doc_cls._get_db().dereference(reference)
-        if doc is not None:
-            doc = doc_cls._from_son(doc)
-        return doc
-
     def to_mongo(self, document):
         if document is None:
             return None
@@ -1642,7 +1641,7 @@ class EnumField(BaseField):
                 "'choices' can't be set on EnumField, "
                 "it is implicitly set as the enum class"
             )
-        kwargs["choices"] = list(self._enum_cls)
+        kwargs["choices"] = list(self._enum_cls)  # Implicit validator
         super().__init__(**kwargs)
 
     def __set__(self, instance, value):
@@ -1659,13 +1658,6 @@ class EnumField(BaseField):
             return value.value
         return value
 
-    def validate(self, value):
-        if value and not isinstance(value, self._enum_cls):
-            try:
-                self._enum_cls(value)
-            except Exception as e:
-                self.error(str(e))
-
     def prepare_query_value(self, op, value):
         if value is None:
             return value
@@ -2431,7 +2423,7 @@ class LazyReferenceField(BaseField):
           object is deleted
         :param passthrough: When trying to access unknown fields, the
           :class:`~mongoengine.base.datastructure.LazyReference` instance will
-          automatically call `fetch()` and try to retrive the field on the fetched
+          automatically call `fetch()` and try to retrieve the field on the fetched
           document. Note this only work getting field (not setting or deleting).
         """
         # XXX ValidationError raised outside of the "validate" method.

mongoengine/mongodb_support.py

@@ -3,11 +3,12 @@ Helper functions, constants, and types to aid with MongoDB version support
 """
 from mongoengine.connection import get_connection
 
-
 # Constant that can be used to compare the version retrieved with
 # get_mongodb_version()
 MONGODB_34 = (3, 4)
 MONGODB_36 = (3, 6)
+MONGODB_42 = (4, 2)
+MONGODB_44 = (4, 4)
 
 
 def get_mongodb_version():

mongoengine/queryset/base.py

@@ -2,13 +2,12 @@ import copy
 import itertools
 import re
 import warnings
-
 from collections.abc import Mapping
 
-from bson import SON, json_util
-from bson.code import Code
 import pymongo
 import pymongo.errors
+from bson import SON, json_util
+from bson.code import Code
 from pymongo.collection import ReturnDocument
 from pymongo.common import validate_read_preference
 from pymongo.read_concern import ReadConcern
@@ -34,7 +33,6 @@ from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
 
-
 __all__ = ("BaseQuerySet", "DO_NOTHING", "NULLIFY", "CASCADE", "DENY", "PULL")
 
 # Delete rules
@@ -64,6 +62,7 @@ class BaseQuerySet:
         self._ordering = None
         self._snapshot = False
         self._timeout = True
+        self._allow_disk_use = False
         self._read_preference = None
         self._read_concern = None
         self._iter = False
@@ -799,6 +798,7 @@ class BaseQuerySet:
             "_ordering",
             "_snapshot",
             "_timeout",
+            "_allow_disk_use",
             "_read_preference",
             "_read_concern",
             "_iter",
@@ -1165,6 +1165,16 @@ class BaseQuerySet:
         queryset._snapshot = enabled
         return queryset
 
+    def allow_disk_use(self, enabled):
+        """Enable or disable the use of temporary files on disk while processing a blocking sort operation.
+         (To store data exceeding the 100 megabyte system memory limit)
+
+        :param enabled: whether or not temporary files on disk are used
+        """
+        queryset = self.clone()
+        queryset._allow_disk_use = enabled
+        return queryset
+
     def timeout(self, enabled):
         """Enable or disable the default mongod timeout when querying. (no_cursor_timeout option)
 
@@ -1343,21 +1353,18 @@ class BaseQuerySet:
 
         MapReduceDocument = _import_class("MapReduceDocument")
 
-        if not hasattr(self._collection, "map_reduce"):
-            raise NotImplementedError("Requires MongoDB >= 1.7.1")
-
         map_f_scope = {}
         if isinstance(map_f, Code):
             map_f_scope = map_f.scope
             map_f = str(map_f)
-        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope)
+        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None)
 
         reduce_f_scope = {}
         if isinstance(reduce_f, Code):
             reduce_f_scope = reduce_f.scope
             reduce_f = str(reduce_f)
         reduce_f_code = queryset._sub_js_fields(reduce_f)
-        reduce_f = Code(reduce_f_code, reduce_f_scope)
+        reduce_f = Code(reduce_f_code, reduce_f_scope or None)
 
         mr_args = {"query": queryset._query}
 
@@ -1367,7 +1374,7 @@ class BaseQuerySet:
                 finalize_f_scope = finalize_f.scope
                 finalize_f = str(finalize_f)
             finalize_f_code = queryset._sub_js_fields(finalize_f)
-            finalize_f = Code(finalize_f_code, finalize_f_scope)
+            finalize_f = Code(finalize_f_code, finalize_f_scope or None)
             mr_args["finalize"] = finalize_f
 
         if scope:
@@ -1604,6 +1611,9 @@ class BaseQuerySet:
         if not self._timeout:
             cursor_args["no_cursor_timeout"] = True
 
+        if self._allow_disk_use:
+            cursor_args["allow_disk_use"] = True
+
         if self._loaded_fields:
             cursor_args[fields_name] = self._loaded_fields.as_dict()
 

mongoengine/queryset/manager.py

@@ -1,4 +1,5 @@
 from functools import partial
+
 from mongoengine.queryset.queryset import QuerySet
 
 __all__ = ("queryset_manager", "QuerySetManager")

mongoengine/queryset/queryset.py

@@ -1,11 +1,11 @@
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (
-    BaseQuerySet,
     CASCADE,
     DENY,
     DO_NOTHING,
     NULLIFY,
     PULL,
+    BaseQuerySet,
 )
 
 __all__ = (

mongoengine/queryset/transform.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
 
-from bson import ObjectId, SON
-from bson.dbref import DBRef
 import pymongo
+from bson import SON, ObjectId
+from bson.dbref import DBRef
 
 from mongoengine.base import UPDATE_OPERATORS
 from mongoengine.common import _import_class

requirements-dev.txt

@@ -1,6 +1,5 @@
 black
 flake8
-flake8-import-order
 pre-commit
 pytest
 ipdb

setup.cfg

@@ -2,9 +2,17 @@
 ignore=E501,F403,F405,I201,I202,W504,W605,W503,B007
 exclude=build,dist,docs,venv,venv3,.tox,.eggs,tests
 max-complexity=47
-application-import-names=mongoengine,tests
 
 [tool:pytest]
 # Limits the discovery to tests directory
 # avoids that it runs for instance the benchmark
 testpaths = tests
+
+[isort]
+known_first_party = mongoengine,tests
+default_section = THIRDPARTY
+multi_line_output = 3
+include_trailing_comma = True
+combine_as_imports = True
+line_length = 70
+ensure_newline_before_comments = 1

setup.py

@@ -53,8 +53,8 @@ class PyTest(TestCommand):
 
     def run_tests(self):
         # import here, cause outside the eggs aren't loaded
-        from pkg_resources import _namespace_packages
         import pytest
+        from pkg_resources import _namespace_packages
 
         # Purge modules under test from sys.modules. The test loader will
         # re-import them from the build location. Required when 2to3 is used

tests/document/test_class_methods.py

@@ -177,7 +177,7 @@ class TestClassMethods(unittest.TestCase):
         assert BlogPostWithCustomField.compare_indexes() == {"missing": [], "extra": []}
 
     def test_compare_indexes_for_text_indexes(self):
-        """ Ensure that compare_indexes behaves correctly for text indexes """
+        """Ensure that compare_indexes behaves correctly for text indexes"""
 
         class Doc(Document):
             a = StringField()

tests/document/test_delta.py

@@ -1,6 +1,7 @@
 import unittest
 
 from bson import SON
+
 from mongoengine import *
 from mongoengine.pymongo_support import list_collection_names
 from tests.utils import MongoDBTestCase

tests/document/test_indexes.py

@@ -1,12 +1,16 @@
 import unittest
 from datetime import datetime
 
+import pytest
 from pymongo.collation import Collation
 from pymongo.errors import OperationFailure
-import pytest
 
 from mongoengine import *
 from mongoengine.connection import get_db
+from mongoengine.mongodb_support import (
+    MONGODB_42,
+    get_mongodb_version,
+)
 
 
 class TestIndexes(unittest.TestCase):
@@ -452,9 +456,11 @@ class TestIndexes(unittest.TestCase):
             .get("stage")
             == "IXSCAN"
         )
+        mongo_db = get_mongodb_version()
+        PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED"
         assert (
             query_plan.get("queryPlanner").get("winningPlan").get("stage")
-            == "PROJECTION"
+            == PROJECTION_STR
         )
 
         query_plan = Test.objects(a=1).explain()

tests/document/test_instance.py

@@ -6,9 +6,9 @@ import weakref
 from datetime import datetime
 
 import bson
+import pytest
 from bson import DBRef, ObjectId
 from pymongo.errors import DuplicateKeyError
-import pytest
 
 from mongoengine import *
 from mongoengine import signals
@@ -23,7 +23,11 @@ from mongoengine.errors import (
     NotUniqueError,
     SaveConditionError,
 )
-from mongoengine.mongodb_support import MONGODB_34, MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_34,
+    MONGODB_36,
+    get_mongodb_version,
+)
 from mongoengine.pymongo_support import list_collection_names
 from mongoengine.queryset import NULLIFY, Q
 from tests import fixtures

tests/document/test_json_serialisation.py

@@ -1,7 +1,7 @@
 import unittest
 import uuid
-
 from datetime import datetime
+
 from bson import ObjectId
 
 from mongoengine import *

tests/fields/test_binary_field.py

@@ -1,7 +1,7 @@
 import uuid
 
-from bson import Binary
 import pytest
+from bson import Binary
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase

tests/fields/test_boolean_field.py

@@ -13,6 +13,17 @@ class TestBooleanField(MongoDBTestCase):
         person.save()
         assert get_as_pymongo(person) == {"_id": person.id, "admin": True}
 
+    def test_construction_does_not_fail_uncastable_value(self):
+        class BoolFail:
+            def __bool__(self):
+                return "bogus"
+
+        class Person(Document):
+            admin = BooleanField()
+
+        person = Person(admin=BoolFail())
+        person.admin == "bogus"
+
     def test_validation(self):
         """Ensure that invalid values cannot be assigned to boolean
         fields.

tests/fields/test_cached_reference_field.py

@@ -2,11 +2,28 @@ from decimal import Decimal
 
 import pytest
 
-from mongoengine import *
+from mongoengine import (
+    CachedReferenceField,
+    DecimalField,
+    Document,
+    EmbeddedDocument,
+    EmbeddedDocumentField,
+    InvalidDocumentError,
+    ListField,
+    ReferenceField,
+    StringField,
+    ValidationError,
+)
 from tests.utils import MongoDBTestCase
 
 
 class TestCachedReferenceField(MongoDBTestCase):
+    def test_constructor_fail_bad_document_type(self):
+        with pytest.raises(
+            ValidationError, match="must be a document class or a string"
+        ):
+            CachedReferenceField(document_type=0)
+
     def test_get_and_save(self):
         """
         Tests #1047: CachedReferenceField creates DBRefs on to_python,

tests/fields/test_complex_datetime_field.py

@@ -6,7 +6,6 @@ import re
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_datetime_field.py

@@ -9,7 +9,6 @@ except ImportError:
 
 from mongoengine import *
 from mongoengine import connection
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_decimal_field.py

@@ -2,59 +2,11 @@ from decimal import Decimal
 
 import pytest
 
-from mongoengine import *
+from mongoengine import DecimalField, Document, ValidationError
 from tests.utils import MongoDBTestCase
 
 
 class TestDecimalField(MongoDBTestCase):
-    def test_validation(self):
-        """Ensure that invalid values cannot be assigned to decimal fields."""
-
-        class Person(Document):
-            height = DecimalField(min_value=Decimal("0.1"), max_value=Decimal("3.5"))
-
-        Person.drop_collection()
-
-        Person(height=Decimal("1.89")).save()
-        person = Person.objects.first()
-        assert person.height == Decimal("1.89")
-
-        person.height = "2.0"
-        person.save()
-        person.height = 0.01
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = Decimal("0.01")
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = Decimal("4.0")
-        with pytest.raises(ValidationError):
-            person.validate()
-        person.height = "something invalid"
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person_2 = Person(height="something invalid")
-        with pytest.raises(ValidationError):
-            person_2.validate()
-
-    def test_comparison(self):
-        class Person(Document):
-            money = DecimalField()
-
-        Person.drop_collection()
-
-        Person(money=6).save()
-        Person(money=7).save()
-        Person(money=8).save()
-        Person(money=10).save()
-
-        assert 2 == Person.objects(money__gt=Decimal("7")).count()
-        assert 2 == Person.objects(money__gt=7).count()
-        assert 2 == Person.objects(money__gt="7").count()
-
-        assert 3 == Person.objects(money__gte="7").count()
-
     def test_storage(self):
         class Person(Document):
             float_value = DecimalField(precision=4)
@@ -106,3 +58,63 @@ class TestDecimalField(MongoDBTestCase):
         for field_name in ["float_value", "string_value"]:
             actual = list(Person.objects().scalar(field_name))
             assert expected == actual
+
+    def test_save_none(self):
+        class Person(Document):
+            value = DecimalField()
+
+        Person.drop_collection()
+
+        person = Person(value=None)
+        assert person.value is None
+        person.save()
+        fetched_person = Person.objects.first()
+        fetched_person.value is None
+
+    def test_validation(self):
+        """Ensure that invalid values cannot be assigned to decimal fields."""
+
+        class Person(Document):
+            height = DecimalField(min_value=Decimal("0.1"), max_value=Decimal("3.5"))
+
+        Person.drop_collection()
+
+        Person(height=Decimal("1.89")).save()
+        person = Person.objects.first()
+        assert person.height == Decimal("1.89")
+
+        person.height = "2.0"
+        person.save()
+        person.height = 0.01
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = Decimal("0.01")
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = Decimal("4.0")
+        with pytest.raises(ValidationError):
+            person.validate()
+        person.height = "something invalid"
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person_2 = Person(height="something invalid")
+        with pytest.raises(ValidationError):
+            person_2.validate()
+
+    def test_comparison(self):
+        class Person(Document):
+            money = DecimalField()
+
+        Person.drop_collection()
+
+        Person(money=6).save()
+        Person(money=7).save()
+        Person(money=8).save()
+        Person(money=10).save()
+
+        assert 2 == Person.objects(money__gt=Decimal("7")).count()
+        assert 2 == Person.objects(money__gt=7).count()
+        assert 2 == Person.objects(money__gt="7").count()
+
+        assert 3 == Person.objects(money__gte="7").count()

tests/fields/test_dict_field.py

@@ -1,10 +1,12 @@
-from bson import InvalidDocument
 import pytest
+from bson import InvalidDocument
 
 from mongoengine import *
 from mongoengine.base import BaseDict
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
-
+from mongoengine.mongodb_support import (
+    MONGODB_36,
+    get_mongodb_version,
+)
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
 

tests/fields/test_embedded_document_field.py

@@ -12,7 +12,6 @@ from mongoengine import (
     StringField,
     ValidationError,
 )
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_enum_field.py

@@ -1,9 +1,9 @@
 from enum import Enum
 
-from bson import InvalidDocument
 import pytest
+from bson import InvalidDocument
 
-from mongoengine import *
+from mongoengine import Document, EnumField, ValidationError
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
 
@@ -45,6 +45,11 @@ class TestStringEnumField(MongoDBTestCase):
         m.save()
         assert m.status == Status.DONE
 
+        m.status = "wrong"
+        assert m.status == "wrong"
+        with pytest.raises(ValidationError):
+            m.validate()
+
     def test_set_default(self):
         class ModelWithDefault(Document):
             status = EnumField(Status, default=Status.DONE)

tests/fields/test_fields.py

@@ -1,8 +1,8 @@
 import datetime
 import unittest
 
-from bson import DBRef, ObjectId, SON
 import pytest
+from bson import SON, DBRef, ObjectId
 
 from mongoengine import (
     BooleanField,
@@ -34,9 +34,12 @@ from mongoengine import (
     StringField,
     ValidationError,
 )
-from mongoengine.base import BaseField, EmbeddedDocumentList, _document_registry
+from mongoengine.base import (
+    BaseField,
+    EmbeddedDocumentList,
+    _document_registry,
+)
 from mongoengine.errors import DeprecatedError
-
 from tests.utils import MongoDBTestCase
 
 
@@ -306,7 +309,7 @@ class TestField(MongoDBTestCase):
         )
         assert res == 1
 
-        # Retrive data from db and verify it.
+        # Retrieve data from db and verify it.
         ret = HandleNoneFields.objects.all()[0]
         assert ret.str_fld is None
         assert ret.int_fld is None
@@ -340,7 +343,7 @@ class TestField(MongoDBTestCase):
             {"$unset": {"str_fld": 1, "int_fld": 1, "flt_fld": 1, "comp_dt_fld": 1}},
         )
 
-        # Retrive data from db and verify it.
+        # Retrieve data from db and verify it.
         ret = HandleNoneFields.objects.first()
         assert ret.str_fld is None
         assert ret.int_fld is None
@@ -374,34 +377,6 @@ class TestField(MongoDBTestCase):
         person.id = str(ObjectId())
         person.validate()
 
-    def test_string_validation(self):
-        """Ensure that invalid values cannot be assigned to string fields."""
-
-        class Person(Document):
-            name = StringField(max_length=20)
-            userid = StringField(r"[0-9a-z_]+$")
-
-        person = Person(name=34)
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        # Test regex validation on userid
-        person = Person(userid="test.User")
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person.userid = "test_user"
-        assert person.userid == "test_user"
-        person.validate()
-
-        # Test max length validation on name
-        person = Person(name="Name that is more than twenty characters")
-        with pytest.raises(ValidationError):
-            person.validate()
-
-        person.name = "Shorter name"
-        person.validate()
-
     def test_db_field_validation(self):
         """Ensure that db_field doesn't accept invalid values."""
 

tests/fields/test_file_field.py

@@ -308,7 +308,7 @@ class TestFileField(MongoDBTestCase):
         assert test_file.the_file not in [{"test": 1}]
 
     def test_file_disk_space(self):
-        """ Test disk space usage when we delete/replace a file """
+        """Test disk space usage when we delete/replace a file"""
 
         class TestFile(Document):
             the_file = FileField()

tests/fields/test_float_field.py

@@ -1,7 +1,6 @@
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_int_field.py

@@ -1,7 +1,6 @@
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_lazy_reference_field.py

@@ -1,10 +1,9 @@
-from bson import DBRef, ObjectId
 import pytest
+from bson import DBRef, ObjectId
 
 from mongoengine import *
 from mongoengine.base import LazyReference
 from mongoengine.context_managers import query_counter
-
 from tests.utils import MongoDBTestCase
 
 
@@ -375,6 +374,26 @@ class TestLazyReferenceField(MongoDBTestCase):
             assert isinstance(ref.author, LazyReference)
             assert isinstance(ref.author.id, ObjectId)
 
+    def test_lazy_reference_in_list_with_changed_element(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocurrence(Document):
+            in_list = ListField(LazyReferenceField(Animal))
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal(name="doggo").save()
+
+        animal1.tag = "blue"
+
+        occ = Ocurrence(in_list=[animal1]).save()
+        animal1.save()
+        assert isinstance(occ.in_list[0], LazyReference)
+        assert occ.in_list[0].pk == animal1.pk
+
 
 class TestGenericLazyReferenceField(MongoDBTestCase):
     def test_generic_lazy_reference_simple(self):

tests/fields/test_long_field.py

@@ -1,13 +1,28 @@
-from bson.int64 import Int64
 import pytest
+from bson.int64 import Int64
 
 from mongoengine import *
 from mongoengine.connection import get_db
-
-from tests.utils import MongoDBTestCase
+from tests.utils import MongoDBTestCase, get_as_pymongo
 
 
 class TestLongField(MongoDBTestCase):
+    def test_storage(self):
+        class Person(Document):
+            value = LongField()
+
+        Person.drop_collection()
+        person = Person(value=5000)
+        person.save()
+        assert get_as_pymongo(person) == {"_id": person.id, "value": 5000}
+
+    def test_construction_does_not_fail_with_invalid_value(self):
+        class Person(Document):
+            value = LongField()
+
+        person = Person(value="not_an_int")
+        assert person.value == "not_an_int"
+
     def test_long_field_is_considered_as_int64(self):
         """
         Tests that long fields are stored as long in mongo, even if long
@@ -30,19 +45,16 @@ class TestLongField(MongoDBTestCase):
         class TestDocument(Document):
             value = LongField(min_value=0, max_value=110)
 
-        doc = TestDocument()
-        doc.value = 50
-        doc.validate()
+        TestDocument(value=50).validate()
 
-        doc.value = -1
         with pytest.raises(ValidationError):
-            doc.validate()
-        doc.value = 120
+            TestDocument(value=-1).validate()
+
         with pytest.raises(ValidationError):
-            doc.validate()
-        doc.value = "ten"
+            TestDocument(value=120).validate()
+
         with pytest.raises(ValidationError):
-            doc.validate()
+            TestDocument(value="ten").validate()
 
     def test_long_ne_operator(self):
         class TestDocument(Document):
@@ -53,4 +65,5 @@ class TestLongField(MongoDBTestCase):
         TestDocument(long_fld=None).save()
         TestDocument(long_fld=1).save()
 
-        assert 1 == TestDocument.objects(long_fld__ne=None).count()
+        assert TestDocument.objects(long_fld__ne=None).count() == 1
+        assert TestDocument.objects(long_fld__ne=1).count() == 1

tests/fields/test_reference_field.py

@@ -1,5 +1,5 @@
-from bson import DBRef, SON
 import pytest
+from bson import SON, DBRef
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase

tests/fields/test_sequence_field.py

@@ -1,5 +1,4 @@
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 
@@ -274,3 +273,25 @@ class TestSequenceField(MongoDBTestCase):
         assert foo.counter == bar.counter
         assert foo._fields["counter"].owner_document == Foo
         assert bar._fields["counter"].owner_document == Bar
+
+    def test_sequence_setattr_not_incrementing_counter(self):
+        class Person(DynamicDocument):
+            id = SequenceField(primary_key=True)
+            name = StringField()
+
+        self.db["mongoengine.counters"].drop()
+        Person.drop_collection()
+
+        for x in range(10):
+            Person(name="Person %s" % x).save()
+
+        c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
+        assert c["next"] == 10
+
+        # Setting SequenceField field value should not increment counter:
+        new_person = Person()
+        new_person.id = 1100
+
+        # Counter should still be at 10
+        c = self.db["mongoengine.counters"].find_one({"_id": "person.id"})
+        assert c["next"] == 10

tests/fields/test_string_field.py

@@ -0,0 +1,43 @@
+import pytest
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class TestStringField(MongoDBTestCase):
+    def test_storage(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        person = Person(name="test123")
+        person.save()
+        assert get_as_pymongo(person) == {"_id": person.id, "name": "test123"}
+
+    def test_validation(self):
+        class Person(Document):
+            name = StringField(max_length=20, min_length=2)
+            userid = StringField(r"[0-9a-z_]+$")
+
+        with pytest.raises(ValidationError, match="only accepts string values"):
+            Person(name=34).validate()
+
+        with pytest.raises(ValidationError, match="value is too short"):
+            Person(name="s").validate()
+
+        # Test regex validation on userid
+        person = Person(userid="test.User")
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person.userid = "test_user"
+        assert person.userid == "test_user"
+        person.validate()
+
+        # Test max length validation on name
+        person = Person(name="Name that is more than twenty characters")
+        with pytest.raises(ValidationError):
+            person.validate()
+
+        person = Person(name="a friendl name", userid="7a757668sqjdkqlsdkq")
+        person.validate()

tests/queryset/test_geo.py

@@ -2,7 +2,6 @@ import datetime
 import unittest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/queryset/test_modify.py

@@ -1,6 +1,12 @@
 import unittest
 
-from mongoengine import Document, IntField, ListField, StringField, connect
+from mongoengine import (
+    Document,
+    IntField,
+    ListField,
+    StringField,
+    connect,
+)
 
 
 class Doc(Document):

tests/queryset/test_queryset.py

@@ -3,17 +3,20 @@ import unittest
 import uuid
 from decimal import Decimal
 
-from bson import DBRef, ObjectId
 import pymongo
+import pytest
+from bson import DBRef, ObjectId
 from pymongo.read_preferences import ReadPreference
 from pymongo.results import UpdateResult
-import pytest
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.errors import InvalidQueryError
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_36,
+    get_mongodb_version,
+)
 from mongoengine.queryset import (
     DoesNotExist,
     MultipleObjectsReturned,
@@ -21,6 +24,10 @@ from mongoengine.queryset import (
     QuerySetManager,
     queryset_manager,
 )
+from tests.utils import (
+    requires_mongodb_gte_44,
+    requires_mongodb_lt_42,
+)
 
 
 class db_ops_tracker(query_counter):
@@ -1489,6 +1496,7 @@ class TestQueryset(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_query(self):
         """Ensure that queries are properly formed for use in exec_js."""
 
@@ -1526,6 +1534,7 @@ class TestQueryset(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_field_sub(self):
         """Ensure that field substitutions occur properly in exec_js functions."""
 
@@ -2659,6 +2668,8 @@ class TestQueryset(unittest.TestCase):
             title = StringField(primary_key=True)
             tags = ListField(StringField())
 
+        BlogPost.drop_collection()
+
         post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"])
         post2 = BlogPost(title="Post #2", tags=["django", "mongodb"])
         post3 = BlogPost(title="Post #3", tags=["hitchcock films"])
@@ -2687,12 +2698,15 @@ class TestQueryset(unittest.TestCase):
             }
         """
 
-        results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults")
+        results = BlogPost.objects.order_by("_id").map_reduce(
+            map_f, reduce_f, "myresults2"
+        )
         results = list(results)
 
-        assert results[0].object == post1
-        assert results[1].object == post2
-        assert results[2].object == post3
+        assert len(results) == 3
+        assert results[0].object.id == post1.id
+        assert results[1].object.id == post2.id
+        assert results[2].object.id == post3.id
 
         BlogPost.drop_collection()
 
@@ -2700,7 +2714,6 @@ class TestQueryset(unittest.TestCase):
         """
         Test map/reduce custom output
         """
-        register_connection("test2", "mongoenginetest2")
 
         class Family(Document):
             id = IntField(primary_key=True)
@@ -2773,6 +2786,7 @@ class TestQueryset(unittest.TestCase):
                             family.persons.push(person);
                             family.totalAge += person.age;
                         });
+                        family.persons.sort((a, b) => (a.age > b.age))
                     }
                 });
 
@@ -2801,10 +2815,10 @@ class TestQueryset(unittest.TestCase):
             "_id": 1,
             "value": {
                 "persons": [
-                    {"age": 21, "name": "Wilson Jr"},
-                    {"age": 45, "name": "Wilson Father"},
-                    {"age": 40, "name": "Eliana Costa"},
                     {"age": 17, "name": "Tayza Mariana"},
+                    {"age": 21, "name": "Wilson Jr"},
+                    {"age": 40, "name": "Eliana Costa"},
+                    {"age": 45, "name": "Wilson Father"},
                 ],
                 "totalAge": 123,
             },
@@ -2814,9 +2828,9 @@ class TestQueryset(unittest.TestCase):
             "_id": 2,
             "value": {
                 "persons": [
+                    {"age": 10, "name": "Igor Gabriel"},
                     {"age": 16, "name": "Isabella Luanna"},
                     {"age": 36, "name": "Sandra Mara"},
-                    {"age": 10, "name": "Igor Gabriel"},
                 ],
                 "totalAge": 62,
             },
@@ -2826,8 +2840,8 @@ class TestQueryset(unittest.TestCase):
             "_id": 3,
             "value": {
                 "persons": [
-                    {"age": 30, "name": "Arthur WA"},
                     {"age": 25, "name": "Paula Leonel"},
+                    {"age": 30, "name": "Arthur WA"},
                 ],
                 "totalAge": 55,
             },
@@ -3108,6 +3122,7 @@ class TestQueryset(unittest.TestCase):
         freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True)
         assert freq == {"CRB": 0.5, None: 0.5}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_null_embedded(self):
         class Data(EmbeddedDocument):
             name = StringField()
@@ -3136,6 +3151,7 @@ class TestQueryset(unittest.TestCase):
         ot = Person.objects.item_frequencies("extra.tag", map_reduce=True)
         assert ot == {None: 1.0, "friend": 1.0}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_0_values(self):
         class Test(Document):
             val = IntField()
@@ -3150,6 +3166,7 @@ class TestQueryset(unittest.TestCase):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {0: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_False_values(self):
         class Test(Document):
             val = BooleanField()
@@ -3164,6 +3181,7 @@ class TestQueryset(unittest.TestCase):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {False: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_normalize(self):
         class Test(Document):
             val = IntField()
@@ -3550,7 +3568,8 @@ class TestQueryset(unittest.TestCase):
         Book.objects.create(title="The Stories", authors=[mark_twain, john_tolkien])
 
         authors = Book.objects.distinct("authors")
-        assert authors == [mark_twain, john_tolkien]
+        authors_names = {author.name for author in authors}
+        assert authors_names == {mark_twain.name, john_tolkien.name}
 
     def test_distinct_ListField_EmbeddedDocumentField_EmbeddedDocumentField(self):
         class Continent(EmbeddedDocument):
@@ -3587,7 +3606,8 @@ class TestQueryset(unittest.TestCase):
         assert country_list == [scotland, tibet]
 
         continent_list = Book.objects.distinct("authors.country.continent")
-        assert continent_list == [europe, asia]
+        continent_list_names = {c.continent_name for c in continent_list}
+        assert continent_list_names == {europe.continent_name, asia.continent_name}
 
     def test_distinct_ListField_ReferenceField(self):
         class Bar(Document):
@@ -5656,6 +5676,31 @@ class TestQueryset(unittest.TestCase):
         qs = self.Person.objects().timeout(False)
         assert qs._cursor_args == {"no_cursor_timeout": True}
 
+    @requires_mongodb_gte_44
+    def test_allow_disk_use(self):
+        qs = self.Person.objects()
+        assert qs._cursor_args == {}
+
+        qs = self.Person.objects().allow_disk_use(False)
+        assert qs._cursor_args == {}
+
+        qs = self.Person.objects().allow_disk_use(True)
+        assert qs._cursor_args == {"allow_disk_use": True}
+
+        # Test if allow_disk_use changes the results
+        self.Person.drop_collection()
+        self.Person.objects.create(name="Foo", age=12)
+        self.Person.objects.create(name="Baz", age=17)
+        self.Person.objects.create(name="Bar", age=13)
+
+        qs_disk = self.Person.objects().order_by("age").allow_disk_use(True)
+        qs = self.Person.objects().order_by("age")
+
+        assert qs_disk.count() == qs.count()
+
+        for index in range(qs_disk.count()):
+            assert qs_disk[index] == qs[index]
+
 
 if __name__ == "__main__":
     unittest.main()

tests/queryset/test_queryset_aggregation.py

@@ -248,6 +248,34 @@ class TestQuerysetAggregate(MongoDBTestCase):
 
         assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}]
 
+    def test_queryset_aggregation_geonear_aggregation_on_pointfield(self):
+        """test ensures that $geonear can be used as a 1-stage pipeline and that
+        MongoEngine does not interfer with such pipeline (#2473)
+        """
+
+        class Aggr(Document):
+            name = StringField()
+            c = PointField()
+
+        Aggr.drop_collection()
+
+        agg1 = Aggr(name="X", c=[10.634584, 35.8245029]).save()
+        agg2 = Aggr(name="Y", c=[10.634584, 35.8245029]).save()
+
+        pipeline = [
+            {
+                "$geoNear": {
+                    "near": {"type": "Point", "coordinates": [10.634584, 35.8245029]},
+                    "distanceField": "c",
+                    "spherical": True,
+                }
+            }
+        ]
+        assert list(Aggr.objects.aggregate(*pipeline)) == [
+            {"_id": agg1.id, "c": 0.0, "name": "X"},
+            {"_id": agg2.id, "c": 0.0, "name": "Y"},
+        ]
+
 
 if __name__ == "__main__":
     unittest.main()

tests/queryset/test_transform.py

@@ -1,7 +1,7 @@
 import unittest
 
-from bson.son import SON
 import pytest
+from bson.son import SON
 
 from mongoengine import *
 from mongoengine.queryset import Q, transform

tests/queryset/test_visitor.py

@@ -2,8 +2,8 @@ import datetime
 import re
 import unittest
 
-from bson import ObjectId
 import pytest
+from bson import ObjectId
 
 from mongoengine import *
 from mongoengine.errors import InvalidQueryError

tests/test_connection.py

@@ -1,11 +1,11 @@
 import datetime
 import unittest
 
-from bson.tz_util import utc
 import pymongo
+import pytest
+from bson.tz_util import utc
 from pymongo import MongoClient, ReadPreference
 from pymongo.errors import InvalidName, OperationFailure
-import pytest
 
 import mongoengine.connection
 from mongoengine import (
@@ -17,8 +17,8 @@ from mongoengine import (
     register_connection,
 )
 from mongoengine.connection import (
-    ConnectionFailure,
     DEFAULT_DATABASE_NAME,
+    ConnectionFailure,
     disconnect,
     get_connection,
     get_db,

tests/test_connection_mongomock.py

@@ -3,15 +3,9 @@ import unittest
 import pytest
 
 import mongoengine.connection
-from mongoengine import (
-    Document,
-    StringField,
-    connect,
-    disconnect_all,
-)
+from mongoengine import Document, StringField, connect, disconnect_all
 from mongoengine.connection import get_connection
 
-
 try:
     import mongomock
 

tests/test_datastructures.py

@@ -3,7 +3,11 @@ import unittest
 import pytest
 
 from mongoengine import Document
-from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    StrictDict,
+)
 
 
 class DocumentStub:

tests/test_replicaset_connection.py

@@ -5,7 +5,6 @@ from pymongo import MongoClient, ReadPreference
 import mongoengine
 from mongoengine.connection import ConnectionFailure
 
-
 CONN_CLASS = MongoClient
 READ_PREF = ReadPreference.SECONDARY
 

tests/test_signals.py

@@ -249,7 +249,7 @@ class TestSignal(unittest.TestCase):
         assert self.pre_signals == post_signals
 
     def test_model_signals(self):
-        """ Model saves should throw some signals. """
+        """Model saves should throw some signals."""
 
         def create_author():
             self.Author(name="Bill Shakespeare")
@@ -340,7 +340,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_signal_kwargs(self):
-        """ Make sure signal_kwargs is passed to signals calls. """
+        """Make sure signal_kwargs is passed to signals calls."""
 
         def live_and_let_die():
             a = self.Author(name="Bill Shakespeare")
@@ -385,7 +385,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_queryset_delete_signals(self):
-        """ Queryset delete should throw some signals. """
+        """Queryset delete should throw some signals."""
 
         self.Another(name="Bill Shakespeare").save()
         assert self.get_signal_output(self.Another.objects.delete) == [
@@ -396,7 +396,7 @@ class TestSignal(unittest.TestCase):
         ]
 
     def test_signals_with_explicit_doc_ids(self):
-        """ Model saves must have a created flag the first time."""
+        """Model saves must have a created flag the first time."""
         ei = self.ExplicitId(id=123)
         # post save must received the created flag, even if there's already
         # an object id present

tests/utils.py

@@ -1,3 +1,4 @@
+import operator
 import unittest
 
 import pytest
@@ -6,7 +7,6 @@ from mongoengine import connect
 from mongoengine.connection import disconnect_all, get_db
 from mongoengine.mongodb_support import get_mongodb_version
 
-
 MONGO_TEST_DB = "mongoenginetest"  # standard name for the test database
 
 
@@ -33,6 +33,14 @@ def get_as_pymongo(doc):
     return doc.__class__.objects.as_pymongo().get(id=doc.id)
 
 
+def requires_mongodb_lt_42(func):
+    return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt)
+
+
+def requires_mongodb_gte_44(func):
+    return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge)
+
+
 def _decorated_with_ver_requirement(func, mongo_version_req, oper):
     """Return a MongoDB version requirement decorator.