Merge branch 'master' into pr/625

mongoengine/base/document.py

@@ -1,6 +1,7 @@
 import copy
 import operator
 import numbers
+from collections import Hashable
 from functools import partial
 
 import pymongo
@@ -12,8 +13,7 @@ from mongoengine import signals
 from mongoengine.common import _import_class
 from mongoengine.errors import (ValidationError, InvalidDocumentError,
                                 LookUpError)
-from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type,
-                                        to_str_keys_recursive)
+from mongoengine.python_support import PY3, txt_type
 
 from mongoengine.base.common import get_document, ALLOW_INHERITANCE
 from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict, SemiStrictDict 
@@ -197,7 +197,7 @@ class BaseDocument(object):
         """Dictionary-style field access, set a field's value.
         """
         # Ensure that the field exists before settings its value
-        if name not in self._fields:
+        if not self._dynamic and name not in self._fields:
             raise KeyError(name)
         return setattr(self, name, value)
 
@@ -391,20 +391,41 @@ class BaseDocument(object):
             self._changed_fields.append(key)
 
     def _clear_changed_fields(self):
+        """Using get_changed_fields iterate and remove any fields that are
+        marked as changed"""
+        for changed in self._get_changed_fields():
+            parts = changed.split(".")
+            data = self
+            for part in parts:
+                if isinstance(data, list):
+                    try:
+                        data = data[int(part)]
+                    except IndexError:
+                        data = None
+                elif isinstance(data, dict):
+                    data = data.get(part, None)
+                else:
+                    data = getattr(data, part, None)
+                if hasattr(data, "_changed_fields"):
+                    data._changed_fields = []
         self._changed_fields = []
-        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
-        for field_name, field in self._fields.iteritems():
-            if (isinstance(field, ComplexBaseField) and
-               isinstance(field.field, EmbeddedDocumentField)):
-                field_value = getattr(self, field_name, None)
-                if field_value:
-                    for idx in (field_value if isinstance(field_value, dict)
-                                else xrange(len(field_value))):
-                        field_value[idx]._clear_changed_fields()
-            elif isinstance(field, EmbeddedDocumentField):
-                field_value = getattr(self, field_name, None)
-                if field_value:
-                    field_value._clear_changed_fields()
+
+    def _nestable_types_changed_fields(self, changed_fields, key, data, inspected):
+        # Loop list / dict fields as they contain documents
+        # Determine the iterator to use
+        if not hasattr(data, 'items'):
+            iterator = enumerate(data)
+        else:
+            iterator = data.iteritems()
+
+        for index, value in iterator:
+            list_key = "%s%s." % (key, index)
+            if hasattr(value, '_get_changed_fields'):
+                changed = value._get_changed_fields(inspected)
+                changed_fields += ["%s%s" % (list_key, k)
+                                    for k in changed if k]
+            elif isinstance(value, (list, tuple, dict)):
+                self._nestable_types_changed_fields(changed_fields, list_key, value, inspected)
 
     def _get_changed_fields(self, inspected=None):
         """Returns a list of all fields that have explicitly been changed.
@@ -412,13 +433,12 @@ class BaseDocument(object):
         EmbeddedDocument = _import_class("EmbeddedDocument")
         DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument")
         ReferenceField = _import_class("ReferenceField")
-        _changed_fields = []
-        _changed_fields += getattr(self, '_changed_fields', [])
-
+        changed_fields = []
+        changed_fields += getattr(self, '_changed_fields', [])
         inspected = inspected or set()
-        if hasattr(self, 'id'):
+        if hasattr(self, 'id') and isinstance(self.id, Hashable):
             if self.id in inspected:
-                return _changed_fields
+                return changed_fields
             inspected.add(self.id)
 
         for field_name in self._fields_ordered:
@@ -434,29 +454,17 @@ class BaseDocument(object):
             if isinstance(field, ReferenceField):
                 continue
             elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
-               and db_field_name not in _changed_fields):
+               and db_field_name not in changed_fields):
                  # Find all embedded fields that have been changed
                 changed = data._get_changed_fields(inspected)
-                _changed_fields += ["%s%s" % (key, k) for k in changed if k]
+                changed_fields += ["%s%s" % (key, k) for k in changed if k]
             elif (isinstance(data, (list, tuple, dict)) and
-                    db_field_name not in _changed_fields):
-                # Loop list / dict fields as they contain documents
-                # Determine the iterator to use
-                if not hasattr(data, 'items'):
-                    iterator = enumerate(data)
-                else:
-                    iterator = data.iteritems()
-                for index, value in iterator:
-                    if not hasattr(value, '_get_changed_fields'):
-                        continue
-                    if (hasattr(field, 'field') and
-                        isinstance(field.field, ReferenceField)):
-                        continue
-                    list_key = "%s%s." % (key, index)
-                    changed = value._get_changed_fields(inspected)
-                    _changed_fields += ["%s%s" % (list_key, k)
-                                        for k in changed if k]
-        return _changed_fields
+                    db_field_name not in changed_fields):
+                if (hasattr(field, 'field') and
+                    isinstance(field.field, ReferenceField)):
+                    continue
+                self._nestable_types_changed_fields(changed_fields, key, data, inspected)
+        return changed_fields
 
     def _delta(self):
         """Returns the delta (set, unset) of the changes for a document.
@@ -552,10 +560,6 @@ class BaseDocument(object):
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
         data = dict(("%s" % key, value) for key, value in son.iteritems())
-        if not UNICODE_KWARGS:
-            # python 2.6.4 and lower cannot handle unicode keys
-            # passed to class constructor example: cls(**data)
-            to_str_keys_recursive(data)
 
         # Return correct subclass for document type
         if class_name != cls._class_name:
@@ -773,6 +777,9 @@ class BaseDocument(object):
         """Lookup a field based on its attribute and return a list containing
         the field's parents and the field.
         """
+
+        ListField = _import_class("ListField")
+
         if not isinstance(parts, (list, tuple)):
             parts = [parts]
         fields = []
@@ -780,7 +787,7 @@ class BaseDocument(object):
 
         for field_name in parts:
             # Handle ListField indexing:
-            if field_name.isdigit() and hasattr(field, 'field'):
+            if field_name.isdigit() and isinstance(field, ListField):
                 new_field = field.field
                 fields.append(field_name)
                 continue

mongoengine/base/fields.py

@@ -89,12 +89,7 @@ class BaseField(object):
             return self
 
         # Get value from document instance if available
-        value = instance._data.get(self.name)
-
-        EmbeddedDocument = _import_class('EmbeddedDocument')
-        if isinstance(value, EmbeddedDocument) and value._instance is None:
-            value._instance = weakref.proxy(instance)
-        return value
+        return instance._data.get(self.name)
 
     def __set__(self, instance, value):
         """Descriptor for assigning a value to a field in a document.
@@ -116,6 +111,10 @@ class BaseField(object):
                 # Values cant be compared eg: naive and tz datetimes
                 # So mark it as changed
                 instance._mark_as_changed(self.name)
+
+        EmbeddedDocument = _import_class('EmbeddedDocument')
+        if isinstance(value, EmbeddedDocument) and value._instance is None:
+            value._instance = weakref.proxy(instance)
         instance._data[self.name] = value
 
     def error(self, message="", errors=None, field_name=None):
@@ -203,7 +202,7 @@ class ComplexBaseField(BaseField):
         _dereference = _import_class("DeReference")()
 
         self._auto_dereference = instance._fields[self.name]._auto_dereference
-        if instance._initialised and dereference:
+        if instance._initialised and dereference and instance._data.get(self.name):
             instance._data[self.name] = _dereference(
                 instance._data.get(self.name), max_depth=1, instance=instance,
                 name=self.name