Merge pull request #1128 from iici-gli/master

Fixed: ListField minus index assignment does not work #1119

AUTHORS

@@ -228,6 +228,7 @@ that much better:
  * Vicki Donchenko (https://github.com/kivistein)
  * Emile Caron (https://github.com/emilecaron)
  * Amit Lichtenberg (https://github.com/amitlicht)
+ * Gang Li (https://github.com/iici-gli)
  * Lars Butler (https://github.com/larsbutler)
  * George Macon (https://github.com/gmacon)
  * Ashley Whetter (https://github.com/AWhetter)

docs/changelog.rst

@@ -52,6 +52,8 @@ Changes in 0.10.1
 - Document save's save_condition error raises `SaveConditionError` exception #1070
 - Fix Document.reload for DynamicDocument. #1050
 - StrictDict & SemiStrictDict are shadowed at init time. #1105
+- Fix ListField minus index assignment does not work. #1119
+- Remove code that marks field as changed when the field has default but not existed in database #1126
 - Remove test dependencies (nose and rednose) from install dependencies list. #1079
 - Recursively build query when using elemMatch operator. #1130
 - Fix instance back references for lists of embedded documents. #1131

mongoengine/base/document.py

@@ -310,7 +310,7 @@ class BaseDocument(object):
         data = SON()
         data["_id"] = None
         data['_cls'] = self._class_name
-        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+        
         # only root fields ['test1.a', 'test2'] => ['test1', 'test2']
         root_fields = set([f.split('.')[0] for f in fields])
 
@@ -325,18 +325,20 @@ class BaseDocument(object):
                 field = self._dynamic_fields.get(field_name)
 
             if value is not None:
-
-                if fields:
+                f_inputs = field.to_mongo.__code__.co_varnames
+                ex_vars = {}
+                if fields and 'fields' in f_inputs:
                     key = '%s.' % field_name
                     embedded_fields = [
                         i.replace(key, '') for i in fields
                         if i.startswith(key)]
 
-                else:
-                    embedded_fields = []
+                    ex_vars['fields'] = embedded_fields 
 
-                value = field.to_mongo(value, use_db_field=use_db_field,
-                                        fields=embedded_fields)
+                if 'use_db_field' in f_inputs:
+                    ex_vars['use_db_field'] = use_db_field
+                     
+                value = field.to_mongo(value, **ex_vars)
 
             # Handle self generating fields
             if value is None and field._auto_gen:
@@ -489,7 +491,7 @@ class BaseDocument(object):
                 # remove lower level changed fields
                 level = '.'.join(levels[:idx]) + '.'
                 remove = self._changed_fields.remove
-                for field in self._changed_fields:
+                for field in self._changed_fields[:]:
                     if field.startswith(level):
                         remove(field)
 
@@ -604,7 +606,9 @@ class BaseDocument(object):
                 for p in parts:
                     if isinstance(d, (ObjectId, DBRef)):
                         break
-                    elif isinstance(d, list) and p.isdigit():
+                    elif isinstance(d, list) and p.lstrip('-').isdigit():
+                        if p[0] == '-':
+                            p = str(len(d)+int(p))
                         try:
                             d = d[int(p)]
                         except IndexError:
@@ -638,7 +642,9 @@ class BaseDocument(object):
                 parts = path.split('.')
                 db_field_name = parts.pop()
                 for p in parts:
-                    if isinstance(d, list) and p.isdigit():
+                    if isinstance(d, list) and p.lstrip('-').isdigit():
+                        if p[0] == '-':
+                            p = str(len(d)+int(p))
                         d = d[int(p)]
                     elif (hasattr(d, '__getattribute__') and
                           not isinstance(d, dict)):
@@ -706,14 +712,6 @@ class BaseDocument(object):
                         del data[field.db_field]
                 except (AttributeError, ValueError), e:
                     errors_dict[field_name] = e
-            elif field.default:
-                default = field.default
-                if callable(default):
-                    default = default()
-                if isinstance(default, BaseDocument):
-                    changed_fields.append(field_name)
-                elif not only_fields or field_name in only_fields:
-                    changed_fields.append(field_name)
 
         if errors_dict:
             errors = "\n".join(["%s - %s" % (k, v)

mongoengine/base/fields.py

@@ -158,11 +158,24 @@ class BaseField(object):
         """
         return value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         """Convert a Python type to a MongoDB-compatible type.
         """
         return self.to_python(value)
 
+    def _to_mongo_safe_call(self, value, use_db_field=True, fields=None):
+        """A helper method to call to_mongo with proper inputs
+        """
+        f_inputs = self.to_mongo.__code__.co_varnames
+        ex_vars = {}
+        if 'fields' in f_inputs:
+            ex_vars['fields'] = fields 
+
+        if 'use_db_field' in f_inputs:
+            ex_vars['use_db_field'] = use_db_field
+             
+        return self.to_mongo(value, **ex_vars)
+
     def prepare_query_value(self, op, value):
         """Prepare a value that is being used in a query for PyMongo.
         """
@@ -324,7 +337,7 @@ class ComplexBaseField(BaseField):
                                          key=operator.itemgetter(0))]
         return value_dict
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB-compatible type.
         """
         Document = _import_class("Document")
@@ -336,10 +349,9 @@ class ComplexBaseField(BaseField):
 
         if hasattr(value, 'to_mongo'):
             if isinstance(value, Document):
-                return GenericReferenceField().to_mongo(
-                    value, **kwargs)
+                return GenericReferenceField().to_mongo(value)
             cls = value.__class__
-            val = value.to_mongo(**kwargs)
+            val = value.to_mongo(use_db_field, fields)
             # If it's a document that is not inherited add _cls
             if isinstance(value, EmbeddedDocument):
                 val['_cls'] = cls.__name__
@@ -354,7 +366,7 @@ class ComplexBaseField(BaseField):
                 return value
 
         if self.field:
-            value_dict = dict([(key, self.field.to_mongo(item, **kwargs))
+            value_dict = dict([(key, self.field._to_mongo_safe_call(item, use_db_field, fields))
                                for key, item in value.iteritems()])
         else:
             value_dict = {}
@@ -373,20 +385,19 @@ class ComplexBaseField(BaseField):
                         meta.get('allow_inheritance', ALLOW_INHERITANCE)
                         is True)
                     if not allow_inheritance and not self.field:
-                        value_dict[k] = GenericReferenceField().to_mongo(
-                            v, **kwargs)
+                        value_dict[k] = GenericReferenceField().to_mongo(v)
                     else:
                         collection = v._get_collection_name()
                         value_dict[k] = DBRef(collection, v.pk)
                 elif hasattr(v, 'to_mongo'):
                     cls = v.__class__
-                    val = v.to_mongo(**kwargs)
+                    val = v.to_mongo(use_db_field, fields)
                     # If it's a document that is not inherited add _cls
                     if isinstance(v, (Document, EmbeddedDocument)):
                         val['_cls'] = cls.__name__
                     value_dict[k] = val
                 else:
-                    value_dict[k] = self.to_mongo(v, **kwargs)
+                    value_dict[k] = self.to_mongo(v, use_db_field, fields)
 
         if is_list:  # Convert back to a list
             return [v for _, v in sorted(value_dict.items(),
@@ -444,7 +455,7 @@ class ObjectIdField(BaseField):
             pass
         return value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         if not isinstance(value, ObjectId):
             try:
                 return ObjectId(unicode(value))
@@ -619,7 +630,7 @@ class GeoJsonBaseField(BaseField):
         if errors:
             return "Invalid MultiPolygon:\n%s" % ", ".join(errors)
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         if isinstance(value, dict):
             return value
         return SON([("type", self._type), ("coordinates", value)])

mongoengine/fields.py

@@ -231,7 +231,7 @@ class LongField(BaseField):
             pass
         return value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         return Int64(value)
 
     def validate(self, value):
@@ -338,7 +338,7 @@ class DecimalField(BaseField):
             return value
         return value.quantize(decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding)
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         if value is None:
             return value
         if self.force_string:
@@ -401,7 +401,7 @@ class DateTimeField(BaseField):
         if not isinstance(new_value, (datetime.datetime, datetime.date)):
             self.error(u'cannot parse date "%s"' % value)
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         if value is None:
             return value
         if isinstance(value, datetime.datetime):
@@ -524,7 +524,7 @@ class ComplexDateTimeField(StringField):
         except Exception:
             return original_value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         value = self.to_python(value)
         return self._convert_from_datetime(value)
 
@@ -559,10 +559,10 @@ class EmbeddedDocumentField(BaseField):
             return self.document_type._from_son(value, _auto_dereference=self._auto_dereference)
         return value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         if not isinstance(value, self.document_type):
             return value
-        return self.document_type.to_mongo(value, **kwargs)
+        return self.document_type.to_mongo(value, use_db_field, fields)
 
     def validate(self, value, clean=True):
         """Make sure that the document instance is an instance of the
@@ -612,11 +612,11 @@ class GenericEmbeddedDocumentField(BaseField):
 
         value.validate(clean=clean)
 
-    def to_mongo(self, document, **kwargs):
+    def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
 
-        data = document.to_mongo(**kwargs)
+        data = document.to_mongo(use_db_field, fields)
         if '_cls' not in data:
             data['_cls'] = document._class_name
         return data
@@ -628,7 +628,7 @@ class DynamicField(BaseField):
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB compatible type.
         """
 
@@ -637,7 +637,7 @@ class DynamicField(BaseField):
 
         if hasattr(value, 'to_mongo'):
             cls = value.__class__
-            val = value.to_mongo(**kwargs)
+            val = value.to_mongo(use_db_field, fields)
             # If we its a document thats not inherited add _cls
             if isinstance(value, Document):
                 val = {"_ref": value.to_dbref(), "_cls": cls.__name__}
@@ -655,7 +655,7 @@ class DynamicField(BaseField):
 
         data = {}
         for k, v in value.iteritems():
-            data[k] = self.to_mongo(v, **kwargs)
+            data[k] = self.to_mongo(v, use_db_field, fields)
 
         value = data
         if is_list:  # Convert back to a list
@@ -767,8 +767,8 @@ class SortedListField(ListField):
             self._order_reverse = kwargs.pop('reverse')
         super(SortedListField, self).__init__(field, **kwargs)
 
-    def to_mongo(self, value, **kwargs):
-        value = super(SortedListField, self).to_mongo(value, **kwargs)
+    def to_mongo(self, value, use_db_field=True, fields=None):
+        value = super(SortedListField, self).to_mongo(value, use_db_field, fields)
         if self._ordering is not None:
             return sorted(value, key=itemgetter(self._ordering),
                           reverse=self._order_reverse)
@@ -954,7 +954,7 @@ class ReferenceField(BaseField):
 
         return super(ReferenceField, self).__get__(instance, owner)
 
-    def to_mongo(self, document, **kwargs):
+    def to_mongo(self, document):
         if isinstance(document, DBRef):
             if not self.dbref:
                 return document.id
@@ -977,7 +977,7 @@ class ReferenceField(BaseField):
         id_field_name = cls._meta['id_field']
         id_field = cls._fields[id_field_name]
 
-        id_ = id_field.to_mongo(id_, **kwargs)
+        id_ = id_field.to_mongo(id_)
         if self.document_type._meta.get('abstract'):
             collection = cls._get_collection_name()
             return DBRef(collection, id_, cls=cls._class_name)
@@ -1100,7 +1100,7 @@ class CachedReferenceField(BaseField):
 
         return super(CachedReferenceField, self).__get__(instance, owner)
 
-    def to_mongo(self, document, **kwargs):
+    def to_mongo(self, document, use_db_field=True, fields=None):
         id_field_name = self.document_type._meta['id_field']
         id_field = self.document_type._fields[id_field_name]
 
@@ -1115,11 +1115,15 @@ class CachedReferenceField(BaseField):
             # TODO: should raise here or will fail next statement
 
         value = SON((
-            ("_id", id_field.to_mongo(id_, **kwargs)),
+            ("_id", id_field.to_mongo(id_)),
         ))
 
-        kwargs['fields'] = self.fields
-        value.update(dict(document.to_mongo(**kwargs)))
+        if fields:
+            new_fields = [f for f in self.fields if f in fields]
+        else:
+            new_fields = self.fields
+            
+        value.update(dict(document.to_mongo(use_db_field, fields=new_fields)))
         return value
 
     def prepare_query_value(self, op, value):
@@ -1235,7 +1239,7 @@ class GenericReferenceField(BaseField):
             doc = doc_cls._from_son(doc)
         return doc
 
-    def to_mongo(self, document, **kwargs):
+    def to_mongo(self, document):
         if document is None:
             return None
 
@@ -1254,7 +1258,7 @@ class GenericReferenceField(BaseField):
         else:
             id_ = document
 
-        id_ = id_field.to_mongo(id_, **kwargs)
+        id_ = id_field.to_mongo(id_)
         collection = document._get_collection_name()
         ref = DBRef(collection, id_)
         return SON((
@@ -1283,7 +1287,7 @@ class BinaryField(BaseField):
             value = bin_type(value)
         return super(BinaryField, self).__set__(instance, value)
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         return Binary(value)
 
     def validate(self, value):
@@ -1508,7 +1512,7 @@ class FileField(BaseField):
                                 db_alias=db_alias,
                                 collection_name=collection_name)
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         # Store the GridFS file id in MongoDB
         if isinstance(value, self.proxy_class) and value.grid_id is not None:
             return value.grid_id
@@ -1858,7 +1862,7 @@ class UUIDField(BaseField):
                 return original_value
         return value
 
-    def to_mongo(self, value, **kwargs):
+    def to_mongo(self, value):
         if not self._binary:
             return unicode(value)
         elif isinstance(value, basestring):

tests/document/instance.py

@@ -3118,6 +3118,17 @@ class InstanceTest(unittest.TestCase):
         p4 = Person.objects()[0]
         p4.save()
         self.assertEquals(p4.height, 189)
+        
+        # However the default will not be fixed in DB
+        self.assertEquals(Person.objects(height=189).count(), 0)
+        
+        # alter DB for the new default
+        coll = Person._get_collection()
+        for person in Person.objects.as_pymongo():
+            if 'height' not in person:
+                person['height'] = 189
+                coll.save(person)
+                
         self.assertEquals(Person.objects(height=189).count(), 1)
 
     def test_from_son(self):

tests/fields/fields.py

@@ -1046,6 +1046,54 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(BlogPost.objects(info=['1', '2', '3', '4', '1', '2', '3', '4']).count(), 1)
         BlogPost.drop_collection()
 
+    def test_list_assignment(self):
+        """Ensure that list field element assignment and slicing work 
+        """
+        class BlogPost(Document):
+            info = ListField()
+
+        BlogPost.drop_collection()
+
+        post = BlogPost()
+        post.info = ['e1', 'e2', 3, '4', 5]
+        post.save()
+        
+        post.info[0] = 1
+        post.save()
+        post.reload()
+        self.assertEqual(post.info[0], 1)
+        
+        post.info[1:3] = ['n2', 'n3']
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', '4', 5])
+
+        post.info[-1] = 'n5'
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', '4', 'n5'])
+
+        post.info[-2] = 4
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', 4, 'n5'])
+
+        post.info[1:-1] = [2]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 2, 'n5'])
+
+        post.info[:-1] = [1, 'n2', 'n3', 4]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', 4, 'n5'])
+
+        post.info[-4:3] = [2, 3]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 2, 3, 4, 'n5'])
+
+
     def test_list_field_passed_in_value(self):
         class Foo(Document):
             bars = ListField(ReferenceField("Bar"))
@@ -3479,7 +3527,7 @@ class FieldTest(unittest.TestCase):
             def __init__(self, **kwargs):
                 super(EnumField, self).__init__(**kwargs)
 
-            def to_mongo(self, value, **kwargs):
+            def to_mongo(self, value):
                 return value
 
             def to_python(self, value):