Merge commit master into bugfix-save-sharding

AUTHORS

@@ -257,4 +257,5 @@ that much better:
  * Matthew Simpson (https://github.com/mcsimps2)
  * Leonardo Domingues (https://github.com/leodmgs)
  * Agustin Barto (https://github.com/abarto)
+ * Stankiewicz Mateusz (https://github.com/mas15)
  * Felix Schultheiß (https://github.com/felix-smashdocs)

docs/changelog.rst

@@ -13,6 +13,7 @@ Development
 - Fix the behavior of Doc.objects.limit(0) which should return all documents (similar to mongodb) #2311
 - Bug fix in ListField when updating the first item, it was saving the whole list, instead of
     just replacing the first item (as it's usually done) #2392
+- Add EnumField: ``mongoengine.fields.EnumField``
 
 Changes in 0.20.0
 =================

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # MongoEngine documentation build configuration file, created by
 # sphinx-quickstart on Sun Nov 22 18:14:13 2009.

docs/guide/defining-documents.rst

@@ -76,6 +76,7 @@ are as follows:
 * :class:`~mongoengine.fields.EmailField`
 * :class:`~mongoengine.fields.EmbeddedDocumentField`
 * :class:`~mongoengine.fields.EmbeddedDocumentListField`
+* :class:`~mongoengine.fields.EnumField`
 * :class:`~mongoengine.fields.FileField`
 * :class:`~mongoengine.fields.FloatField`
 * :class:`~mongoengine.fields.GenericEmbeddedDocumentField`

mongoengine/base/document.py

@@ -538,6 +538,9 @@ class BaseDocument:
         """Using _get_changed_fields iterate and remove any fields that
         are marked as changed.
         """
+        ReferenceField = _import_class("ReferenceField")
+        GenericReferenceField = _import_class("GenericReferenceField")
+
         for changed in self._get_changed_fields():
             parts = changed.split(".")
             data = self
@@ -550,7 +553,8 @@ class BaseDocument:
                 elif isinstance(data, dict):
                     data = data.get(part, None)
                 else:
-                    data = getattr(data, part, None)
+                    field_name = data._reverse_db_field_map.get(part, part)
+                    data = getattr(data, field_name, None)
 
                 if not isinstance(data, LazyReference) and hasattr(
                     data, "_changed_fields"
@@ -559,10 +563,40 @@ class BaseDocument:
                         continue
 
                     data._changed_fields = []
+                elif isinstance(data, (list, tuple, dict)):
+                    if hasattr(data, "field") and isinstance(
+                        data.field, (ReferenceField, GenericReferenceField)
+                    ):
+                        continue
+                    BaseDocument._nestable_types_clear_changed_fields(data)
 
         self._changed_fields = []
 
-    def _nestable_types_changed_fields(self, changed_fields, base_key, data):
+    @staticmethod
+    def _nestable_types_clear_changed_fields(data):
+        """Inspect nested data for changed fields
+
+        :param data: data to inspect for changes
+        """
+        Document = _import_class("Document")
+
+        # Loop list / dict fields as they contain documents
+        # Determine the iterator to use
+        if not hasattr(data, "items"):
+            iterator = enumerate(data)
+        else:
+            iterator = data.items()
+
+        for index_or_key, value in iterator:
+            if hasattr(value, "_get_changed_fields") and not isinstance(
+                value, Document
+            ):  # don't follow references
+                value._clear_changed_fields()
+            elif isinstance(value, (list, tuple, dict)):
+                BaseDocument._nestable_types_clear_changed_fields(value)
+
+    @staticmethod
+    def _nestable_types_changed_fields(changed_fields, base_key, data):
         """Inspect nested data for changed fields
 
         :param changed_fields: Previously collected changed fields
@@ -587,7 +621,9 @@ class BaseDocument:
                 changed = value._get_changed_fields()
                 changed_fields += ["{}{}".format(item_key, k) for k in changed if k]
             elif isinstance(value, (list, tuple, dict)):
-                self._nestable_types_changed_fields(changed_fields, item_key, value)
+                BaseDocument._nestable_types_changed_fields(
+                    changed_fields, item_key, value
+                )
 
     def _get_changed_fields(self):
         """Return a list of all fields that have explicitly been changed.

mongoengine/fields.py

@@ -87,6 +87,7 @@ __all__ = (
     "PolygonField",
     "SequenceField",
     "UUIDField",
+    "EnumField",
     "MultiPointField",
     "MultiLineStringField",
     "MultiPolygonField",
@@ -847,8 +848,7 @@ class DynamicField(BaseField):
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
     def to_mongo(self, value, use_db_field=True, fields=None):
-        """Convert a Python type to a MongoDB compatible type.
-        """
+        """Convert a Python type to a MongoDB compatible type."""
 
         if isinstance(value, str):
             return value
@@ -1622,6 +1622,70 @@ class BinaryField(BaseField):
         return super().prepare_query_value(op, self.to_mongo(value))
 
 
+class EnumField(BaseField):
+    """Enumeration Field. Values are stored underneath as is,
+    so it will only work with simple types (str, int, etc) that
+    are bson encodable
+     Example usage:
+    .. code-block:: python
+
+        class Status(Enum):
+            NEW = 'new'
+            DONE = 'done'
+
+        class ModelWithEnum(Document):
+            status = EnumField(Status, default=Status.NEW)
+
+        ModelWithEnum(status='done')
+        ModelWithEnum(status=Status.DONE)
+
+    Enum fields can be searched using enum or its value:
+    .. code-block:: python
+
+        ModelWithEnum.objects(status='new').count()
+        ModelWithEnum.objects(status=Status.NEW).count()
+
+    Note that choices cannot be set explicitly, they are derived
+    from the provided enum class.
+    """
+
+    def __init__(self, enum, **kwargs):
+        self._enum_cls = enum
+        if "choices" in kwargs:
+            raise ValueError(
+                "'choices' can't be set on EnumField, "
+                "it is implicitly set as the enum class"
+            )
+        kwargs["choices"] = list(self._enum_cls)
+        super().__init__(**kwargs)
+
+    def __set__(self, instance, value):
+        is_legal_value = value is None or isinstance(value, self._enum_cls)
+        if not is_legal_value:
+            try:
+                value = self._enum_cls(value)
+            except Exception:
+                pass
+        return super().__set__(instance, value)
+
+    def to_mongo(self, value):
+        if isinstance(value, self._enum_cls):
+            return value.value
+        return value
+
+    def validate(self, value):
+        if value and not isinstance(value, self._enum_cls):
+            try:
+                self._enum_cls(value)
+            except Exception as e:
+                self.error(str(e))
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+        return super().prepare_query_value(op, self.to_mongo(value))
+
+
 class GridFSError(Exception):
     pass
 

tests/document/test_class_methods.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *

tests/document/test_delta.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import SON
@@ -546,6 +545,7 @@ class TestDelta(MongoDBTestCase):
             {},
         )
         doc.save()
+        assert doc._get_changed_fields() == []
         doc = doc.reload(10)
 
         assert doc.embedded_field.list_field[0] == "1"
@@ -777,9 +777,7 @@ class TestDelta(MongoDBTestCase):
 
         MyDoc.drop_collection()
 
-        mydoc = MyDoc(
-            name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}
-        ).save()
+        MyDoc(name="testcase1", subs={"a": {"b": EmbeddedDoc(name="foo")}}).save()
 
         mydoc = MyDoc.objects.first()
         subdoc = mydoc.subs["a"]["b"]
@@ -791,6 +789,35 @@ class TestDelta(MongoDBTestCase):
         mydoc._clear_changed_fields()
         assert mydoc._get_changed_fields() == []
 
+    def test_nested_nested_fields_db_field_set__gets_mark_as_changed_and_cleaned(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            name = StringField(db_field="db_name")
+
+        class MyDoc(Document):
+            embed = EmbeddedDocumentField(EmbeddedDoc, db_field="db_embed")
+            name = StringField(db_field="db_name")
+
+        MyDoc.drop_collection()
+
+        MyDoc(name="testcase1", embed=EmbeddedDoc(name="foo")).save()
+
+        mydoc = MyDoc.objects.first()
+        mydoc.embed.name = "foo1"
+
+        assert mydoc.embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed.db_name"]
+
+        mydoc = MyDoc.objects.first()
+        embed = EmbeddedDoc(name="foo2")
+        embed.name = "bar"
+        mydoc.embed = embed
+
+        assert embed._get_changed_fields() == ["db_name"]
+        assert mydoc._get_changed_fields() == ["db_embed"]
+
+        mydoc._clear_changed_fields()
+        assert mydoc._get_changed_fields() == []
+
     def test_lower_level_mark_as_changed(self):
         class EmbeddedDoc(EmbeddedDocument):
             name = StringField()

tests/document/test_indexes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 

tests/document/test_inheritance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 import warnings
 

tests/document/test_instance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import pickle
 import unittest

tests/document/test_validation.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from datetime import datetime
 

tests/fields/test_binary_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 from bson import Binary

tests/fields/test_boolean_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *

tests/fields/test_cached_reference_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest

tests/fields/test_complex_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import itertools
 import math

tests/fields/test_date_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest

tests/fields/test_datetime_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime as dt
 
 import pytest

tests/fields/test_decimal_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from decimal import Decimal
 
 import pytest

tests/fields/test_dict_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from bson import InvalidDocument
 import pytest
 

tests/fields/test_email_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import sys
 
 import pytest

tests/fields/test_embedded_document_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import (

tests/fields/test_enum_field.py

@@ -0,0 +1,122 @@
+from enum import Enum
+
+from bson import InvalidDocument
+import pytest
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class Status(Enum):
+    NEW = "new"
+    DONE = "done"
+
+
+class ModelWithEnum(Document):
+    status = EnumField(Status)
+
+
+class TestStringEnumField(MongoDBTestCase):
+    def test_storage(self):
+        model = ModelWithEnum(status=Status.NEW).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "status": "new"}
+
+    def test_set_enum(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=Status.NEW).save()
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_set_by_value(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects.first().status == Status.NEW
+
+    def test_filter(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status="new").save()
+        assert ModelWithEnum.objects(status="new").count() == 1
+        assert ModelWithEnum.objects(status=Status.NEW).count() == 1
+        assert ModelWithEnum.objects(status=Status.DONE).count() == 0
+
+    def test_change_value(self):
+        m = ModelWithEnum(status="new")
+        m.status = Status.DONE
+        m.save()
+        assert m.status == Status.DONE
+
+    def test_set_default(self):
+        class ModelWithDefault(Document):
+            status = EnumField(Status, default=Status.DONE)
+
+        m = ModelWithDefault().save()
+        assert m.status == Status.DONE
+
+    def test_enum_field_can_be_empty(self):
+        ModelWithEnum.drop_collection()
+        m = ModelWithEnum().save()
+        assert m.status is None
+        assert ModelWithEnum.objects()[0].status is None
+        assert ModelWithEnum.objects(status=None).count() == 1
+
+    def test_set_none_explicitly(self):
+        ModelWithEnum.drop_collection()
+        ModelWithEnum(status=None).save()
+        assert ModelWithEnum.objects.first().status is None
+
+    def test_cannot_create_model_with_wrong_enum_value(self):
+        m = ModelWithEnum(status="wrong_one")
+        with pytest.raises(ValidationError):
+            m.validate()
+
+    def test_user_is_informed_when_tries_to_set_choices(self):
+        with pytest.raises(ValueError, match="'choices' can't be set on EnumField"):
+            EnumField(Status, choices=["my", "custom", "options"])
+
+
+class Color(Enum):
+    RED = 1
+    BLUE = 2
+
+
+class ModelWithColor(Document):
+    color = EnumField(Color, default=Color.RED)
+
+
+class TestIntEnumField(MongoDBTestCase):
+    def test_enum_with_int(self):
+        ModelWithColor.drop_collection()
+        m = ModelWithColor().save()
+        assert m.color == Color.RED
+        assert ModelWithColor.objects(color=Color.RED).count() == 1
+        assert ModelWithColor.objects(color=1).count() == 1
+        assert ModelWithColor.objects(color=2).count() == 0
+
+    def test_create_int_enum_by_value(self):
+        model = ModelWithColor(color=2).save()
+        assert model.color == Color.BLUE
+
+    def test_storage_enum_with_int(self):
+        model = ModelWithColor(color=Color.BLUE).save()
+        assert get_as_pymongo(model) == {"_id": model.id, "color": 2}
+
+    def test_validate_model(self):
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color=3).validate()
+
+        with pytest.raises(ValidationError, match="Value must be one of"):
+            ModelWithColor(color="wrong_type").validate()
+
+
+class TestFunkyEnumField(MongoDBTestCase):
+    def test_enum_incompatible_bson_type_fails_during_save(self):
+        class FunkyColor(Enum):
+            YELLOW = object()
+
+        class ModelWithFunkyColor(Document):
+            color = EnumField(FunkyColor)
+
+        m = ModelWithFunkyColor(color=FunkyColor.YELLOW)
+
+        with pytest.raises(InvalidDocument, match="[cC]annot encode object"):
+            m.save()

tests/fields/test_fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import unittest
 

tests/fields/test_file_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import copy
 import os
 import tempfile

tests/fields/test_float_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *

tests/fields/test_geo_fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *

tests/fields/test_int_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *

tests/fields/test_lazy_reference_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from bson import DBRef, ObjectId
 import pytest
 

tests/fields/test_map_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 
 import pytest

tests/fields/test_reference_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from bson import DBRef, SON
 import pytest
 

tests/fields/test_sequence_field.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 from mongoengine import *
 
 from tests.utils import MongoDBTestCase

tests/fields/test_url_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 
 from mongoengine import *

tests/fields/test_uuid_field.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 import pytest

tests/queryset/test_queryset.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import datetime
 import unittest
 import uuid

tests/queryset/test_queryset_aggregation.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 import warnings
 

tests/test_dereference.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from bson import DBRef, ObjectId
@@ -370,8 +369,7 @@ class FieldTest(unittest.TestCase):
         assert Post.objects.all()[0].user_lists == [[u1, u2], [u3]]
 
     def test_circular_reference(self):
-        """Ensure you can handle circular references
-        """
+        """Ensure you can handle circular references"""
 
         class Relation(EmbeddedDocument):
             name = StringField()
@@ -426,6 +424,7 @@ class FieldTest(unittest.TestCase):
 
         daughter.relations.append(mother)
         daughter.relations.append(daughter)
+        assert daughter._get_changed_fields() == ["relations"]
         daughter.save()
 
         assert "[<Person: Mother>, <Person: Daughter>]" == "%s" % Person.objects()

tests/test_signals.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from mongoengine import *