fix yaml float - string conversion

test against python 3.10

.github/workflows/github-actions.yml

@@ -44,7 +44,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, pypy3]
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
         MONGODB: [$MONGODB_4_0]
         PYMONGO: [$PYMONGO_3_11]
         include:

.pre-commit-config.yaml

@@ -1,22 +1,22 @@
 fail_fast: false
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: check-merge-conflict
       - id: debug-statements
       - id: trailing-whitespace
       - id: end-of-file-fixer
   - repo: https://github.com/ambv/black
-    rev: 21.4b2
+    rev: 21.5b2
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.1
+    rev: 3.9.2
     hooks:
       - id: flake8
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.14.0
+    rev: v2.19.1
     hooks:
       - id: pyupgrade
         args: [--py36-plus]

.travis_.yml

@@ -1,108 +0,0 @@
-## For full coverage, we'd have to test all supported Python, MongoDB, and
-## PyMongo combinations. However, that would result in an overly long build
-## with a very large number of jobs, hence we only test a subset of all the
-## combinations.
-## * Python3.7, MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-## Other combinations are tested. See below for the details or check the travis jobs
-#
-## We should periodically check MongoDB Server versions supported by MongoDB
-## Inc., add newly released versions to the test matrix, and remove versions
-## which have reached their End of Life. See:
-## 1. https://www.mongodb.com/support-policy.
-## 2. https://docs.mongodb.com/ecosystem/drivers/driver-compatibility-reference/#python-driver-compatibility
-##
-## Reminder: Update README.rst if you change MongoDB versions we test.
-#
-#language: python
-#dist: xenial
-#python:
-#  - 3.6
-#  - 3.7
-#  - 3.8
-#  - 3.9
-#  - pypy3
-#
-#env:
-#  global:
-#    - MONGODB_3_4=3.4.19
-#    - MONGODB_3_6=3.6.13
-#    - MONGODB_4_0=4.0.13
-#
-#    - PYMONGO_3_4=3.4
-#    - PYMONGO_3_6=3.6
-#    - PYMONGO_3_9=3.9
-#    - PYMONGO_3_11=3.11
-#
-#    - MAIN_PYTHON_VERSION=3.7
-#  matrix:
-#    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_11}
-#
-#matrix:
-#  # Finish the build as soon as one job fails
-#  fast_finish: true
-#
-#  include:
-#    - python: 3.7
-#      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
-#    - python: 3.7
-#      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-#    - python: 3.7
-#      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_11}
-#    - python: 3.8
-#      env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_11}
-#
-#install:
-#  # Install Mongo
-#  - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB}.tgz
-#  - tar xzf mongodb-linux-x86_64-${MONGODB}.tgz
-#  - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --version
-#  # Install Python dependencies.
-#  - pip install --upgrade pip
-#  - pip install coveralls
-#  - pip install pre-commit
-#  - pip install tox
-#  # tox dryrun to setup the tox venv (we run a mock test).
-#  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
-#
-#before_script:
-#  - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data
-#  - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
-#  # Run pre-commit hooks (black, flake8, etc) on entire codebase
-#  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
-#  - mongo --eval 'db.version();' # Make sure mongo is awake
-#
-#script:
-#  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
-#
-#after_success:
-#  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi
-#
-#notifications:
-#  irc: irc.freenode.org#mongoengine
-#
-## Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
-#branches:
-#  # Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
-#  only:
-#    - master
-#    - /^v.*$/
-#
-## Whenever a new release is created via GitHub, publish it on PyPI.
-#deploy:
-#  provider: pypi
-#  user: the_drow
-#  password:
-#    secure: QMyatmWBnC6ZN3XLW2+fTBDU4LQcp1m/LjR2/0uamyeUzWKdlOoh/Wx5elOgLwt/8N9ppdPeG83ose1jOz69l5G0MUMjv8n/RIcMFSpCT59tGYqn3kh55b0cIZXFT9ar+5cxlif6a5rS72IHm5li7QQyxexJIII6Uxp0kpvUmek=
-#
-#  # Create a source distribution and a pure python wheel for faster installs.
-#  distributions: "sdist bdist_wheel"
-#
-#  # Only deploy on tagged commits (aka GitHub releases) and only for the parent
-#  # repo's builds running Python v3.7 along with PyMongo v3.x and MongoDB v3.4.
-#  # We run Travis against many different Python, PyMongo, and MongoDB versions
-#  # and we don't want the deploy to occur multiple times).
-#  on:
-#    tags: true
-#    repo: MongoEngine/mongoengine
-#    condition: ($PYMONGO = ${PYMONGO_3_11}) && ($MONGODB = ${MONGODB_3_4})
-#    python: 3.7

AUTHORS

@@ -260,3 +260,6 @@ that much better:
  * Stankiewicz Mateusz (https://github.com/mas15)
  * Felix Schultheiß (https://github.com/felix-smashdocs)
  * Jan Stein (https://github.com/janste63)
+ * Timothé Perez (https://github.com/AchilleAsh)
+ * oleksandr-l5 (https://github.com/oleksandr-l5)
+ * Ido Shraga (https://github.com/idoshr)

docs/apireference.rst

@@ -75,6 +75,7 @@ Fields
 .. autoclass:: mongoengine.fields.StringField
 .. autoclass:: mongoengine.fields.URLField
 .. autoclass:: mongoengine.fields.EmailField
+.. autoclass:: mongoengine.fields.EnumField
 .. autoclass:: mongoengine.fields.IntField
 .. autoclass:: mongoengine.fields.LongField
 .. autoclass:: mongoengine.fields.FloatField

docs/changelog.rst

@@ -7,6 +7,10 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- EnumField improvements: now `choices` limits the values of an enum to allow
+- Fix deepcopy of EmbeddedDocument #2202
+- Fix error when using precision=0 with DecimalField #2535
+- Add support for regex and whole word text search query #2568
 
 Changes in 0.23.1
 ===========

docs/guide/defining-documents.rst

@@ -27,6 +27,8 @@ objects** as class attributes to the document class::
 As BSON (the binary format for storing data in mongodb) is order dependent,
 documents are serialized based on their field order.
 
+.. _dynamic-document-schemas:
+
 Dynamic document schemas
 ========================
 One of the benefits of MongoDB is dynamic schemas for a collection, whilst data
@@ -231,6 +233,9 @@ document class as the first argument::
     comment2 = Comment(content='Nice article!')
     page = Page(comments=[comment1, comment2])
 
+Embedded documents can also leverage the flexibility of :ref:`dynamic-document-schemas:`
+by inheriting :class:`~mongoengine.DynamicEmbeddedDocument`.
+
 Dictionary Fields
 -----------------
 Often, an embedded document may be used instead of a dictionary – generally
@@ -336,7 +341,6 @@ supplying the :attr:`reverse_delete_rule` attributes on the
 :class:`ReferenceField` definition, like this::
 
     class ProfilePage(Document):
-        ...
         employee = ReferenceField('Employee', reverse_delete_rule=mongoengine.CASCADE)
 
 The declaration in this example means that when an :class:`Employee` object is
@@ -473,7 +477,7 @@ dictionary containing a full index definition.
 
 A direction may be specified on fields by prefixing the field name with a
 **+** (for ascending) or a **-** sign (for descending). Note that direction
-only matters on multi-field indexes. Text indexes may be specified by prefixing
+only matters on compound indexes. Text indexes may be specified by prefixing
 the field name with a **$**. Hashed indexes may be specified by prefixing
 the field name with a **#**::
 
@@ -484,14 +488,14 @@ the field name with a **#**::
         created = DateTimeField()
         meta = {
             'indexes': [
-                'title',
+                'title',   # single-field index
                 '$title',  # text index
                 '#title',  # hashed index
-                ('title', '-rating'),
-                ('category', '_cls'),
+                ('title', '-rating'),  # compound index
+                ('category', '_cls'),  # compound index
                 {
                     'fields': ['created'],
-                    'expireAfterSeconds': 3600
+                    'expireAfterSeconds': 3600  # ttl index
                 }
             ]
         }
@@ -624,8 +628,8 @@ point. To create a geospatial index you must prefix the field with the
             ],
         }
 
-Time To Live indexes
---------------------
+Time To Live (TTL) indexes
+--------------------------
 
 A special index type that allows you to automatically expire data from a
 collection after a given period. See the official

docs/guide/migration.rst

@@ -223,6 +223,47 @@ it is often useful for complex migrations of Document models.
 
 .. warning:: Be aware of this `flaw <https://groups.google.com/g/mongodb-user/c/AFC1ia7MHzk>`_ if you modify documents while iterating
 
+Example 4: Index removal
+========================
+
+If you remove an index from your Document class, or remove an indexed Field from your Document class,
+you'll need to manually drop the corresponding index. MongoEngine will not do that for you.
+
+The way to deal with this case is to identify the name of the index to drop with `index_information()`, and then drop
+it with `drop_index()`
+
+Let's for instance assume that you start with the following Document class
+
+.. code-block:: python
+
+    class User(Document):
+        name = StringField(index=True)
+
+        meta = {"indexes": ["name"]}
+
+    User(name="John Doe").save()
+
+As soon as you start interacting with the Document collection (when `.save()` is called in this case),
+it would create the following indexes:
+
+.. code-block:: python
+
+    print(User._get_collection().index_information())
+    # {
+    #  '_id_': {'key': [('_id', 1)], 'v': 2},
+    #  'name_1': {'background': False, 'key': [('name', 1)], 'v': 2},
+    # }
+
+Thus: '_id' which is the default index and 'name_1' which is our custom index.
+If you would remove the 'name' field or its index, you would have to call:
+
+.. code-block:: python
+
+    User._get_collection().drop_index('name_1')
+
+.. note:: When adding new fields or new indexes, MongoEngine will take care of creating them
+    (unless `auto_create_index` is disabled) ::
+
 Recommendations
 ===============
 

docs/guide/querying.rst

@@ -86,6 +86,10 @@ expressions:
 * ``istartswith`` -- string field starts with value (case insensitive)
 * ``endswith`` -- string field ends with value
 * ``iendswith`` -- string field ends with value (case insensitive)
+* ``wholeword`` -- string field contains whole word
+* ``iwholeword`` -- string field contains whole word (case insensitive)
+* ``regex`` -- string field match by regex
+* ``iregex`` -- string field match by regex (case insensitive)
 * ``match``  -- performs an $elemMatch so you can match an entire document within an array
 
 
@@ -239,7 +243,7 @@ Limiting and skipping results
 Just as with traditional ORMs, you may limit the number of results returned or
 skip a number or results in you query.
 :meth:`~mongoengine.queryset.QuerySet.limit` and
-:meth:`~mongoengine.queryset.QuerySet.skip` and methods are available on
+:meth:`~mongoengine.queryset.QuerySet.skip` methods are available on
 :class:`~mongoengine.queryset.QuerySet` objects, but the `array-slicing` syntax
 is preferred for achieving this::
 
@@ -543,7 +547,10 @@ Documents may be updated atomically by using the
 There are several different "modifiers" that you may use with these methods:
 
 * ``set`` -- set a particular value
+* ``set_on_insert`` -- set only if this is new document  `need to add upsert=True`_
 * ``unset`` -- delete a particular value (since MongoDB v1.3)
+* ``max`` -- update only if value is bigger
+* ``min`` -- update only if value is smaller
 * ``inc`` -- increment a value by a given amount
 * ``dec`` -- decrement a value by a given amount
 * ``push`` -- append a value to a list
@@ -552,6 +559,7 @@ There are several different "modifiers" that you may use with these methods:
 * ``pull`` -- remove a value from a list
 * ``pull_all`` -- remove several values from a list
 * ``add_to_set`` -- add value to a list only if its not in the list already
+* ``rename`` -- rename the key name
 
 .. _depending on the value: http://docs.mongodb.org/manual/reference/operator/update/pop/
 

mongoengine/connection.py

@@ -58,20 +58,20 @@ def _get_connection_settings(
 ):
     """Get the connection settings as a dict
 
-    : param db: the name of the database to use, for compatibility with connect
-    : param name: the name of the specific database to use
-    : param host: the host name of the: program: `mongod` instance to connect to
-    : param port: the port that the: program: `mongod` instance is running on
-    : param read_preference: The read preference for the collection
-    : param username: username to authenticate with
-    : param password: password to authenticate with
-    : param authentication_source: database to authenticate against
-    : param authentication_mechanism: database authentication mechanisms.
+    :param db: the name of the database to use, for compatibility with connect
+    :param name: the name of the specific database to use
+    :param host: the host name of the: program: `mongod` instance to connect to
+    :param port: the port that the: program: `mongod` instance is running on
+    :param read_preference: The read preference for the collection
+    :param username: username to authenticate with
+    :param password: password to authenticate with
+    :param authentication_source: database to authenticate against
+    :param authentication_mechanism: database authentication mechanisms.
         By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
         MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
-    : param is_mock: explicitly use mongomock for this connection
+    :param is_mock: explicitly use mongomock for this connection
         (can also be done by using `mongomock: // ` as db host prefix)
-    : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
     """
@@ -181,22 +181,21 @@ def register_connection(
 ):
     """Register the connection settings.
 
-    : param alias: the name that will be used to refer to this connection
-        throughout MongoEngine
-    : param db: the name of the database to use, for compatibility with connect
-    : param name: the name of the specific database to use
-    : param host: the host name of the: program: `mongod` instance to connect to
-    : param port: the port that the: program: `mongod` instance is running on
-    : param read_preference: The read preference for the collection
-    : param username: username to authenticate with
-    : param password: password to authenticate with
-    : param authentication_source: database to authenticate against
-    : param authentication_mechanism: database authentication mechanisms.
+    :param alias: the name that will be used to refer to this connection throughout MongoEngine
+    :param db: the name of the database to use, for compatibility with connect
+    :param name: the name of the specific database to use
+    :param host: the host name of the: program: `mongod` instance to connect to
+    :param port: the port that the: program: `mongod` instance is running on
+    :param read_preference: The read preference for the collection
+    :param username: username to authenticate with
+    :param password: password to authenticate with
+    :param authentication_source: database to authenticate against
+    :param authentication_mechanism: database authentication mechanisms.
         By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
         MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
-    : param is_mock: explicitly use mongomock for this connection
+    :param is_mock: explicitly use mongomock for this connection
         (can also be done by using `mongomock: // ` as db host prefix)
-    : param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
         for example maxpoolsize, tz_aware, etc. See the documentation
         for pymongo's `MongoClient` for a full list.
     """

mongoengine/context_managers.py

@@ -177,14 +177,28 @@ class query_counter:
     This was designed for debugging purpose. In fact it is a global counter so queries issued by other threads/processes
     can interfere with it
 
+    Usage:
+
+    .. code-block:: python
+
+        class User(Document):
+            name = StringField()
+
+        with query_counter() as q:
+            user = User(name='Bob')
+            assert q == 0       # no query fired yet
+            user.save()
+            assert q == 1       # 1 query was fired, an 'insert'
+            user_bis = User.objects().first()
+            assert q == 2       # a 2nd query was fired, a 'find_one'
+
     Be aware that:
-    - Iterating over large amount of documents (>101) makes pymongo issue `getmore` queries to fetch the next batch of
-        documents (https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches)
+
+    - Iterating over large amount of documents (>101) makes pymongo issue `getmore` queries to fetch the next batch of documents (https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches)
     - Some queries are ignored by default by the counter (killcursors, db.system.indexes)
     """
 
     def __init__(self, alias=DEFAULT_CONNECTION_NAME):
-        """Construct the query_counter"""
         self.db = get_db(alias=alias)
         self.initial_profiling_level = None
         self._ctx_query_counter = 0  # number of queries issued by the context

mongoengine/document.py

@@ -99,6 +99,15 @@ class EmbeddedDocument(BaseDocument, metaclass=DocumentMetaclass):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __getstate__(self):
+        data = super().__getstate__()
+        data["_instance"] = None
+        return data
+
+    def __setstate__(self, state):
+        super().__setstate__(state)
+        self._instance = state["_instance"]
+
     def to_mongo(self, *args, **kwargs):
         data = super().to_mongo(*args, **kwargs)
 
@@ -126,7 +135,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     create a specialised version of the document that will be stored in the
     same collection. To facilitate this behaviour a `_cls`
     field is added to documents (hidden though the MongoEngine interface).
-    To enable this behaviourset :attr:`allow_inheritance` to ``True`` in the
+    To enable this behaviour set :attr:`allow_inheritance` to ``True`` in the
     :attr:`meta` dictionary.
 
     A :class:`~mongoengine.Document` may use a **Capped Collection** by
@@ -574,7 +583,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
     def _qs(self):
         """Return the default queryset corresponding to this document."""
         if not hasattr(self, "__objects"):
-            self.__objects = QuerySet(self, self._get_collection())
+            self.__objects = QuerySet(self.__class__, self._get_collection())
         return self.__objects
 
     @property
@@ -866,6 +875,10 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
         Global defaults can be set in the meta - see :doc:`guide/defining-documents`
 
+        By default, this will get called automatically upon first interaction with the
+        Document collection (query, save, etc) so unless you disabled `auto_create_index`, you
+        shouldn't have to call this manually.
+
         .. note:: You can disable automatic index creation by setting
                   `auto_create_index` to False in the documents meta data
         """
@@ -915,8 +928,10 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass):
 
     @classmethod
     def list_indexes(cls):
-        """Lists all of the indexes that should be created for given
-        collection. It includes all the indexes from super- and sub-classes.
+        """Lists all indexes that should be created for the Document collection.
+        It includes all the indexes from super- and sub-classes.
+
+        Note that it will only return the indexes' fields, not the indexes' options
         """
         if cls._meta.get("abstract"):
             return []

mongoengine/errors.py

@@ -17,11 +17,15 @@ __all__ = (
 )
 
 
-class NotRegistered(Exception):
+class MongoEngineException(Exception):
     pass
 
 
-class InvalidDocumentError(Exception):
+class NotRegistered(MongoEngineException):
+    pass
+
+
+class InvalidDocumentError(MongoEngineException):
     pass
 
 
@@ -29,19 +33,19 @@ class LookUpError(AttributeError):
     pass
 
 
-class DoesNotExist(Exception):
+class DoesNotExist(MongoEngineException):
     pass
 
 
-class MultipleObjectsReturned(Exception):
+class MultipleObjectsReturned(MongoEngineException):
     pass
 
 
-class InvalidQueryError(Exception):
+class InvalidQueryError(MongoEngineException):
     pass
 
 
-class OperationError(Exception):
+class OperationError(MongoEngineException):
     pass
 
 
@@ -57,7 +61,7 @@ class SaveConditionError(OperationError):
     pass
 
 
-class FieldDoesNotExist(Exception):
+class FieldDoesNotExist(MongoEngineException):
     """Raised when trying to set a field
     not declared in a :class:`~mongoengine.Document`
     or an :class:`~mongoengine.EmbeddedDocument`.
@@ -155,7 +159,7 @@ class ValidationError(AssertionError):
         return " ".join([f"{k}: {v}" for k, v in error_dict.items()])
 
 
-class DeprecatedError(Exception):
+class DeprecatedError(MongoEngineException):
     """Raise when a user uses a feature that has been Deprecated"""
 
     pass

mongoengine/fields.py

@@ -157,10 +157,17 @@ class StringField(BaseField):
                 regex = r"%s$"
             elif op == "exact":
                 regex = r"^%s$"
+            elif op == "wholeword":
+                regex = r"\b%s\b"
+            elif op == "regex":
+                regex = value
 
-            # escape unsafe characters which could lead to a re.error
-            value = re.escape(value)
-            value = re.compile(regex % value, flags)
+            if op == "regex":
+                value = re.compile(regex, flags)
+            else:
+                # escape unsafe characters which could lead to a re.error
+                value = re.escape(value)
+                value = re.compile(regex % value, flags)
         return super().prepare_query_value(op, value)
 
 
@@ -468,6 +475,10 @@ class DecimalField(BaseField):
         self.min_value = min_value
         self.max_value = max_value
         self.force_string = force_string
+
+        if precision < 0 or not isinstance(precision, int):
+            self.error("precision must be a positive integer")
+
         self.precision = precision
         self.rounding = rounding
 
@@ -482,9 +493,12 @@ class DecimalField(BaseField):
             value = decimal.Decimal("%s" % value)
         except (TypeError, ValueError, decimal.InvalidOperation):
             return value
-        return value.quantize(
-            decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding
-        )
+        if self.precision > 0:
+            return value.quantize(
+                decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding
+            )
+        else:
+            return value.quantize(decimal.Decimal(), rounding=self.rounding)
 
     def to_mongo(self, value):
         if value is None:
@@ -1079,16 +1093,7 @@ class DictField(ComplexBaseField):
         return DictField(db_field=member_name)
 
     def prepare_query_value(self, op, value):
-        match_operators = [
-            "contains",
-            "icontains",
-            "startswith",
-            "istartswith",
-            "endswith",
-            "iendswith",
-            "exact",
-            "iexact",
-        ]
+        match_operators = [*STRING_OPERATORS]
 
         if op in match_operators and isinstance(value, str):
             return StringField().prepare_query_value(op, value)
@@ -1611,11 +1616,14 @@ class EnumField(BaseField):
     """Enumeration Field. Values are stored underneath as is,
     so it will only work with simple types (str, int, etc) that
     are bson encodable
-     Example usage:
+
+    Example usage:
+
     .. code-block:: python
 
         class Status(Enum):
             NEW = 'new'
+            ONGOING = 'ongoing'
             DONE = 'done'
 
         class ModelWithEnum(Document):
@@ -1625,23 +1633,31 @@ class EnumField(BaseField):
         ModelWithEnum(status=Status.DONE)
 
     Enum fields can be searched using enum or its value:
+
     .. code-block:: python
 
         ModelWithEnum.objects(status='new').count()
         ModelWithEnum.objects(status=Status.NEW).count()
 
-    Note that choices cannot be set explicitly, they are derived
-    from the provided enum class.
+    The values can be restricted to a subset of the enum by using the ``choices`` parameter:
+
+    .. code-block:: python
+
+        class ModelWithEnum(Document):
+            status = EnumField(Status, choices=[Status.NEW, Status.DONE])
     """
 
     def __init__(self, enum, **kwargs):
         self._enum_cls = enum
-        if "choices" in kwargs:
-            raise ValueError(
-                "'choices' can't be set on EnumField, "
-                "it is implicitly set as the enum class"
-            )
-        kwargs["choices"] = list(self._enum_cls)  # Implicit validator
+        if kwargs.get("choices"):
+            invalid_choices = []
+            for choice in kwargs["choices"]:
+                if not isinstance(choice, enum):
+                    invalid_choices.append(choice)
+            if invalid_choices:
+                raise ValueError("Invalid choices: %r" % invalid_choices)
+        else:
+            kwargs["choices"] = list(self._enum_cls)  # Implicit validator
         super().__init__(**kwargs)
 
     def __set__(self, instance, value):

mongoengine/queryset/base.py

@@ -720,7 +720,7 @@ class BaseQuerySet:
         return queryset.filter(pk=object_id).first()
 
     def in_bulk(self, object_ids):
-        """ "Retrieve a set of documents by their ids.
+        """Retrieve a set of documents by their ids.
 
         :param object_ids: a list or tuple of ObjectId's
         :rtype: dict of ObjectId's as keys and collection-specific

mongoengine/queryset/transform.py

@@ -51,6 +51,10 @@ STRING_OPERATORS = (
     "iendswith",
     "exact",
     "iexact",
+    "regex",
+    "iregex",
+    "wholeword",
+    "iwholeword",
 )
 CUSTOM_OPERATORS = ("match",)
 MATCH_OPERATORS = (

setup.py

@@ -101,6 +101,8 @@ CLASSIFIERS = [
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Database",
@@ -110,7 +112,7 @@ CLASSIFIERS = [
 extra_opts = {
     "packages": find_packages(exclude=["tests", "tests.*"]),
     "tests_require": [
-        "pytest<5.0",
+        "pytest",
         "pytest-cov",
         "coverage",
         "blinker",

tests/document/test_instance.py

@@ -65,12 +65,12 @@ class TestDocumentInstance(MongoDBTestCase):
         for collection in list_collection_names(self.db):
             self.db.drop_collection(collection)
 
-    def assertDbEqual(self, docs):
+    def _assert_db_equal(self, docs):
         assert list(self.Person._get_collection().find().sort("id")) == sorted(
             docs, key=lambda doc: doc["_id"]
         )
 
-    def assertHasInstance(self, field, instance):
+    def _assert_has_instance(self, field, instance):
         assert hasattr(field, "_instance")
         assert field._instance is not None
         if isinstance(field._instance, weakref.ProxyType):
@@ -407,6 +407,16 @@ class TestDocumentInstance(MongoDBTestCase):
         assert person.name == "Test User"
         assert person.age == 30
 
+    def test__qs_property_does_not_raise(self):
+        # ensures no regression of #2500
+        class MyDocument(Document):
+            pass
+
+        MyDocument.drop_collection()
+        object = MyDocument()
+        object._qs().insert([MyDocument()])
+        assert MyDocument.objects.count() == 1
+
     def test_to_dbref(self):
         """Ensure that you can get a dbref of a document."""
         person = self.Person(name="Test User", age=30)
@@ -730,11 +740,11 @@ class TestDocumentInstance(MongoDBTestCase):
         Doc.drop_collection()
 
         doc = Doc(embedded_field=Embedded(string="Hi"))
-        self.assertHasInstance(doc.embedded_field, doc)
+        self._assert_has_instance(doc.embedded_field, doc)
 
         doc.save()
         doc = Doc.objects.get()
-        self.assertHasInstance(doc.embedded_field, doc)
+        self._assert_has_instance(doc.embedded_field, doc)
 
     def test_embedded_document_complex_instance(self):
         """Ensure that embedded documents in complex fields can reference
@@ -749,11 +759,11 @@ class TestDocumentInstance(MongoDBTestCase):
 
         Doc.drop_collection()
         doc = Doc(embedded_field=[Embedded(string="Hi")])
-        self.assertHasInstance(doc.embedded_field[0], doc)
+        self._assert_has_instance(doc.embedded_field[0], doc)
 
         doc.save()
         doc = Doc.objects.get()
-        self.assertHasInstance(doc.embedded_field[0], doc)
+        self._assert_has_instance(doc.embedded_field[0], doc)
 
     def test_embedded_document_complex_instance_no_use_db_field(self):
         """Ensure that use_db_field is propagated to list of Emb Docs."""
@@ -782,11 +792,11 @@ class TestDocumentInstance(MongoDBTestCase):
 
         acc = Account()
         acc.email = Email(email="test@example.com")
-        self.assertHasInstance(acc._data["email"], acc)
+        self._assert_has_instance(acc._data["email"], acc)
         acc.save()
 
         acc1 = Account.objects.first()
-        self.assertHasInstance(acc1._data["email"], acc1)
+        self._assert_has_instance(acc1._data["email"], acc1)
 
     def test_instance_is_set_on_setattr_on_embedded_document_list(self):
         class Email(EmbeddedDocument):
@@ -798,11 +808,11 @@ class TestDocumentInstance(MongoDBTestCase):
         Account.drop_collection()
         acc = Account()
         acc.emails = [Email(email="test@example.com")]
-        self.assertHasInstance(acc._data["emails"][0], acc)
+        self._assert_has_instance(acc._data["emails"][0], acc)
         acc.save()
 
         acc1 = Account.objects.first()
-        self.assertHasInstance(acc1._data["emails"][0], acc1)
+        self._assert_has_instance(acc1._data["emails"][0], acc1)
 
     def test_save_checks_that_clean_is_called(self):
         class CustomError(Exception):
@@ -911,7 +921,7 @@ class TestDocumentInstance(MongoDBTestCase):
         with pytest.raises(InvalidDocumentError):
             self.Person().modify(set__age=10)
 
-        self.assertDbEqual([dict(doc.to_mongo())])
+        self._assert_db_equal([dict(doc.to_mongo())])
 
     def test_modify_invalid_query(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -921,7 +931,7 @@ class TestDocumentInstance(MongoDBTestCase):
         with pytest.raises(InvalidQueryError):
             doc1.modify({"id": doc2.id}, set__value=20)
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_match_another_document(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -931,7 +941,7 @@ class TestDocumentInstance(MongoDBTestCase):
         n_modified = doc1.modify({"name": doc2.name}, set__age=100)
         assert n_modified == 0
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_not_exists(self):
         doc1 = self.Person(name="bob", age=10).save()
@@ -941,7 +951,7 @@ class TestDocumentInstance(MongoDBTestCase):
         n_modified = doc2.modify({"name": doc2.name}, set__age=100)
         assert n_modified == 0
 
-        self.assertDbEqual(docs)
+        self._assert_db_equal(docs)
 
     def test_modify_update(self):
         other_doc = self.Person(name="bob", age=10).save()
@@ -967,7 +977,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert doc.to_json() == doc_copy.to_json()
         assert doc._get_changed_fields() == []
 
-        self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
+        self._assert_db_equal([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
     def test_modify_with_positional_push(self):
         class Content(EmbeddedDocument):

tests/fields/test_decimal_field.py

@@ -118,3 +118,23 @@ class TestDecimalField(MongoDBTestCase):
         assert 2 == Person.objects(money__gt="7").count()
 
         assert 3 == Person.objects(money__gte="7").count()
+
+    def test_precision_0(self):
+        """prevent regression of a bug that was raising an exception when using precision=0"""
+
+        class TestDoc(Document):
+            d = DecimalField(precision=0)
+
+        TestDoc.drop_collection()
+
+        td = TestDoc(d=Decimal("12.00032678131263"))
+        assert td.d == Decimal("12")
+
+    def test_precision_negative_raise(self):
+        """prevent regression of a bug that was raising an exception when using precision=0"""
+        with pytest.raises(
+            ValidationError, match="precision must be a positive integer"
+        ):
+
+            class TestDoc(Document):
+                dneg = DecimalField(precision=-1)

tests/fields/test_embedded_document_field.py

@@ -1,4 +1,7 @@
+from copy import deepcopy
+
 import pytest
+from bson import ObjectId
 
 from mongoengine import (
     Document,
@@ -9,6 +12,7 @@ from mongoengine import (
     InvalidQueryError,
     ListField,
     LookUpError,
+    MapField,
     StringField,
     ValidationError,
 )
@@ -350,3 +354,30 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase):
         # Test existing attribute
         assert Person.objects(settings__base_foo="basefoo").first().id == p.id
         assert Person.objects(settings__sub_foo="subfoo").first().id == p.id
+
+    def test_deepcopy_set__instance(self):
+        """Ensure that the _instance attribute on EmbeddedDocument exists after a deepcopy"""
+
+        class Wallet(EmbeddedDocument):
+            money = IntField()
+
+        class Person(Document):
+            wallet = EmbeddedDocumentField(Wallet)
+            wallet_map = MapField(EmbeddedDocumentField(Wallet))
+
+        # Test on fresh EmbeddedDoc
+        emb_doc = Wallet(money=1)
+        assert emb_doc._instance is None
+        copied_emb_doc = deepcopy(emb_doc)
+        assert copied_emb_doc._instance is None
+
+        # Test on attached EmbeddedDoc
+        doc = Person(
+            id=ObjectId(), wallet=Wallet(money=2), wallet_map={"test": Wallet(money=2)}
+        )
+        assert doc.wallet._instance == doc
+        copied_emb_doc = deepcopy(doc.wallet)
+        assert copied_emb_doc._instance is None
+
+        copied_map_emb_doc = deepcopy(doc.wallet_map)
+        assert copied_map_emb_doc["test"]._instance is None

tests/fields/test_enum_field.py

@@ -12,6 +12,11 @@ class Status(Enum):
     DONE = "done"
 
 
+class Color(Enum):
+    RED = 1
+    BLUE = 2
+
+
 class ModelWithEnum(Document):
     status = EnumField(Status)
 
@@ -74,14 +79,27 @@ class TestStringEnumField(MongoDBTestCase):
         with pytest.raises(ValidationError):
             m.validate()
 
-    def test_user_is_informed_when_tries_to_set_choices(self):
-        with pytest.raises(ValueError, match="'choices' can't be set on EnumField"):
+    def test_partial_choices(self):
+        partial = [Status.DONE]
+        enum_field = EnumField(Status, choices=partial)
+        assert enum_field.choices == partial
+
+        class FancyDoc(Document):
+            z = enum_field
+
+        FancyDoc(z=Status.DONE).validate()
+        with pytest.raises(
+            ValidationError, match=r"Value must be one of .*Status.DONE"
+        ):
+            FancyDoc(z=Status.NEW).validate()
+
+    def test_wrong_choices(self):
+        with pytest.raises(ValueError, match="Invalid choices"):
             EnumField(Status, choices=["my", "custom", "options"])
-
-
-class Color(Enum):
-    RED = 1
-    BLUE = 2
+        with pytest.raises(ValueError, match="Invalid choices"):
+            EnumField(Status, choices=[Color.RED])
+        with pytest.raises(ValueError, match="Invalid choices"):
+            EnumField(Status, choices=[Status.DONE, Color.RED])
 
 
 class ModelWithColor(Document):

tests/fields/test_fields.py

@@ -2077,7 +2077,7 @@ class TestField(MongoDBTestCase):
         a ComplexBaseField.
         """
 
-        class EnumField(BaseField):
+        class SomeField(BaseField):
             def __init__(self, **kwargs):
                 super().__init__(**kwargs)
 
@@ -2088,7 +2088,7 @@ class TestField(MongoDBTestCase):
                 return tuple(value)
 
         class TestDoc(Document):
-            items = ListField(EnumField())
+            items = ListField(SomeField())
 
         TestDoc.drop_collection()
 

tests/queryset/test_modify.py

@@ -19,7 +19,7 @@ class TestFindAndModify(unittest.TestCase):
         connect(db="mongoenginetest")
         Doc.drop_collection()
 
-    def assertDbEqual(self, docs):
+    def _assert_db_equal(self, docs):
         assert list(Doc._collection.find().sort("id")) == docs
 
     def test_modify(self):
@@ -28,7 +28,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(set__value=-1)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_new(self):
         Doc(id=0, value=0).save()
@@ -37,18 +37,18 @@ class TestFindAndModify(unittest.TestCase):
         new_doc = Doc.objects(id=1).modify(set__value=-1, new=True)
         doc.value = -1
         assert new_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_not_existing(self):
         Doc(id=0, value=0).save()
         assert Doc.objects(id=1).modify(set__value=-1) is None
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_modify_with_upsert(self):
         Doc(id=0, value=0).save()
         old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True)
         assert old_doc is None
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
 
     def test_modify_with_upsert_existing(self):
         Doc(id=0, value=0).save()
@@ -56,13 +56,13 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_upsert_with_new(self):
         Doc(id=0, value=0).save()
         new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1)
         assert new_doc.to_mongo() == {"_id": 1, "value": 1}
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
 
     def test_modify_with_remove(self):
         Doc(id=0, value=0).save()
@@ -70,12 +70,12 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).modify(remove=True)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_find_and_modify_with_remove_not_existing(self):
         Doc(id=0, value=0).save()
         assert Doc.objects(id=1).modify(remove=True) is None
-        self.assertDbEqual([{"_id": 0, "value": 0}])
+        self._assert_db_equal([{"_id": 0, "value": 0}])
 
     def test_modify_with_order_by(self):
         Doc(id=0, value=3).save()
@@ -85,7 +85,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects().order_by("-id").modify(set__value=-1)
         assert old_doc.to_json() == doc.to_json()
-        self.assertDbEqual(
+        self._assert_db_equal(
             [
                 {"_id": 0, "value": 3},
                 {"_id": 1, "value": 2},
@@ -100,7 +100,7 @@ class TestFindAndModify(unittest.TestCase):
 
         old_doc = Doc.objects(id=1).only("id").modify(set__value=-1)
         assert old_doc.to_mongo() == {"_id": 1}
-        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+        self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
     def test_modify_with_push(self):
         class BlogPost(Document):

tests/queryset/test_queryset.py

@@ -867,6 +867,21 @@ class TestQueryset(unittest.TestCase):
         assert "Bob" == bob.name
         assert 30 == bob.age
 
+    def test_rename(self):
+        self.Person.drop_collection()
+        self.Person.objects.create(name="Foo", age=11)
+
+        bob = self.Person.objects.as_pymongo().first()
+        assert "age" in bob
+        assert bob["age"] == 11
+
+        self.Person.objects(name="Foo").update(rename__age="person_age")
+
+        bob = self.Person.objects.as_pymongo().first()
+        assert "age" not in bob
+        assert "person_age" in bob
+        assert bob["person_age"] == 11
+
     def test_save_and_only_on_fields_with_default(self):
         class Embed(EmbeddedDocument):
             field = IntField()
@@ -1240,6 +1255,34 @@ class TestQueryset(unittest.TestCase):
         obj = self.Person.objects(name__iexact="gUIDO VAN rOSSU").first()
         assert obj is None
 
+        # Test wholeword
+        obj = self.Person.objects(name__wholeword="Guido").first()
+        assert obj == person
+        obj = self.Person.objects(name__wholeword="rossum").first()
+        assert obj is None
+        obj = self.Person.objects(name__wholeword="Rossu").first()
+        assert obj is None
+
+        # Test iwholeword
+        obj = self.Person.objects(name__iwholeword="rOSSUM").first()
+        assert obj == person
+        obj = self.Person.objects(name__iwholeword="rOSSU").first()
+        assert obj is None
+
+        # Test regex
+        obj = self.Person.objects(name__regex="^[Guido].*[Rossum]$").first()
+        assert obj == person
+        obj = self.Person.objects(name__regex="^[guido].*[rossum]$").first()
+        assert obj is None
+        obj = self.Person.objects(name__regex="^[uido].*[Rossum]$").first()
+        assert obj is None
+
+        # Test iregex
+        obj = self.Person.objects(name__iregex="^[guido].*[rossum]$").first()
+        assert obj == person
+        obj = self.Person.objects(name__iregex="^[Uido].*[Rossum]$").first()
+        assert obj is None
+
         # Test unsafe expressions
         person = self.Person(name="Guido van Rossum [.'Geek']")
         person.save()
@@ -1324,7 +1367,14 @@ class TestQueryset(unittest.TestCase):
         person.save()
 
         people = self.Person.objects
-        people = people.filter(name__startswith="Gui").filter(name__not__endswith="tum")
+        people = (
+            people.filter(name__startswith="Gui")
+            .filter(name__not__endswith="tum")
+            .filter(name__icontains="VAN")
+            .filter(name__regex="^Guido")
+            .filter(name__wholeword="Guido")
+            .filter(name__wholeword="van")
+        )
         assert people.count() == 1
 
     def assertSequence(self, qs, expected):