Merge branch 'master' into Fix_update_full_result_documentation

docs/changelog.rst

@@ -8,6 +8,7 @@ Development
 - Fix .only() working improperly after using .count() of the same instance of QuerySet
 - POTENTIAL BREAKING CHANGE: All result fields are now passed, including internal fields (_cls, _id) when using `QuerySet.as_pymongo` #1976
 - Document a BREAKING CHANGE introduced in 0.15.3 and not reported at that time (#1995)
+- Fix InvalidStringData error when using modify on a BinaryField #1127
 
 =================
 Changes in 0.16.3

mongoengine/fields.py

@@ -1483,6 +1483,12 @@ class BinaryField(BaseField):
         if self.max_bytes is not None and len(value) > self.max_bytes:
             self.error('Binary value is too long')
 
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+        return super(BinaryField, self).prepare_query_value(
+            op, self.to_mongo(value))
+
 
 class GridFSError(Exception):
     pass

tests/fields/fields.py

@@ -17,7 +17,7 @@ except ImportError:
 
 from decimal import Decimal
 
-from bson import Binary, DBRef, ObjectId, SON
+from bson import DBRef, ObjectId, SON
 try:
     from bson.int64 import Int64
 except ImportError:
@@ -1699,7 +1699,7 @@ class FieldTest(MongoDBTestCase):
         post.save()
 
         post = BlogPost()
-        post.info = {'title' : 'dollar_sign', 'details' : {'te$t' : 'test'} }
+        post.info = {'title': 'dollar_sign', 'details': {'te$t': 'test'}}
         post.save()
 
         post = BlogPost()
@@ -1718,7 +1718,7 @@ class FieldTest(MongoDBTestCase):
 
         post = BlogPost.objects.filter(info__title__exact='dollar_sign').first()
         self.assertIn('te$t', post['info']['details'])
-         
+
         # Confirm handles non strings or non existing keys
         self.assertEqual(
             BlogPost.objects.filter(info__details__test__exact=5).count(), 0)
@@ -2934,78 +2934,6 @@ class FieldTest(MongoDBTestCase):
         doc = Doc.objects.get(ref=doc1.pk)
         self.assertEqual(doc, doc2)
 
-    def test_binary_fields(self):
-        """Ensure that binary fields can be stored and retrieved.
-        """
-        class Attachment(Document):
-            content_type = StringField()
-            blob = BinaryField()
-
-        BLOB = six.b('\xe6\x00\xc4\xff\x07')
-        MIME_TYPE = 'application/octet-stream'
-
-        Attachment.drop_collection()
-
-        attachment = Attachment(content_type=MIME_TYPE, blob=BLOB)
-        attachment.save()
-
-        attachment_1 = Attachment.objects().first()
-        self.assertEqual(MIME_TYPE, attachment_1.content_type)
-        self.assertEqual(BLOB, six.binary_type(attachment_1.blob))
-
-    def test_binary_validation_succeeds(self):
-        """Ensure that valid values can be assigned to binary fields.
-        """
-        class AttachmentRequired(Document):
-            blob = BinaryField(required=True)
-
-        class AttachmentSizeLimit(Document):
-            blob = BinaryField(max_bytes=4)
-
-        attachment_required = AttachmentRequired()
-        self.assertRaises(ValidationError, attachment_required.validate)
-        attachment_required.blob = Binary(six.b('\xe6\x00\xc4\xff\x07'))
-        attachment_required.validate()
-
-        _5_BYTES = six.b('\xe6\x00\xc4\xff\x07')
-        _4_BYTES = six.b('\xe6\x00\xc4\xff')
-        self.assertRaises(ValidationError, AttachmentSizeLimit(blob=_5_BYTES).validate)
-        AttachmentSizeLimit(blob=_4_BYTES).validate()
-
-    def test_binary_validation_fails(self):
-        """Ensure that invalid values cannot be assigned to binary fields."""
-
-        class Attachment(Document):
-            blob = BinaryField()
-
-        for invalid_data in (2, u'Im_a_unicode', ['some_str']):
-            self.assertRaises(ValidationError, Attachment(blob=invalid_data).validate)
-
-    def test_binary_field_primary(self):
-        class Attachment(Document):
-            id = BinaryField(primary_key=True)
-
-        Attachment.drop_collection()
-        binary_id = uuid.uuid4().bytes
-        att = Attachment(id=binary_id).save()
-        self.assertEqual(1, Attachment.objects.count())
-        self.assertEqual(1, Attachment.objects.filter(id=att.id).count())
-        att.delete()
-        self.assertEqual(0, Attachment.objects.count())
-
-    def test_binary_field_primary_filter_by_binary_pk_as_str(self):
-        raise SkipTest("Querying by id as string is not currently supported")
-
-        class Attachment(Document):
-            id = BinaryField(primary_key=True)
-
-        Attachment.drop_collection()
-        binary_id = uuid.uuid4().bytes
-        att = Attachment(id=binary_id).save()
-        self.assertEqual(1, Attachment.objects.filter(id=binary_id).count())
-        att.delete()
-        self.assertEqual(0, Attachment.objects.count())
-
     def test_choices_allow_using_sets_as_choices(self):
         """Ensure that sets can be used when setting choices
         """

tests/fields/test_binary_field.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+import uuid
+
+from nose.plugins.skip import SkipTest
+import six
+
+from bson import Binary
+
+from mongoengine import *
+from tests.utils import MongoDBTestCase
+
+BIN_VALUE = six.b('\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde\xe6\x87\xb1\x7f\xc6\xe6\xd9r\x18\xf5')
+
+
+class TestBinaryField(MongoDBTestCase):
+    def test_binary_fields(self):
+        """Ensure that binary fields can be stored and retrieved.
+        """
+        class Attachment(Document):
+            content_type = StringField()
+            blob = BinaryField()
+
+        BLOB = six.b('\xe6\x00\xc4\xff\x07')
+        MIME_TYPE = 'application/octet-stream'
+
+        Attachment.drop_collection()
+
+        attachment = Attachment(content_type=MIME_TYPE, blob=BLOB)
+        attachment.save()
+
+        attachment_1 = Attachment.objects().first()
+        self.assertEqual(MIME_TYPE, attachment_1.content_type)
+        self.assertEqual(BLOB, six.binary_type(attachment_1.blob))
+
+    def test_validation_succeeds(self):
+        """Ensure that valid values can be assigned to binary fields.
+        """
+        class AttachmentRequired(Document):
+            blob = BinaryField(required=True)
+
+        class AttachmentSizeLimit(Document):
+            blob = BinaryField(max_bytes=4)
+
+        attachment_required = AttachmentRequired()
+        self.assertRaises(ValidationError, attachment_required.validate)
+        attachment_required.blob = Binary(six.b('\xe6\x00\xc4\xff\x07'))
+        attachment_required.validate()
+
+        _5_BYTES = six.b('\xe6\x00\xc4\xff\x07')
+        _4_BYTES = six.b('\xe6\x00\xc4\xff')
+        self.assertRaises(ValidationError, AttachmentSizeLimit(blob=_5_BYTES).validate)
+        AttachmentSizeLimit(blob=_4_BYTES).validate()
+
+    def test_validation_fails(self):
+        """Ensure that invalid values cannot be assigned to binary fields."""
+
+        class Attachment(Document):
+            blob = BinaryField()
+
+        for invalid_data in (2, u'Im_a_unicode', ['some_str']):
+            self.assertRaises(ValidationError, Attachment(blob=invalid_data).validate)
+
+    def test__primary(self):
+        class Attachment(Document):
+            id = BinaryField(primary_key=True)
+
+        Attachment.drop_collection()
+        binary_id = uuid.uuid4().bytes
+        att = Attachment(id=binary_id).save()
+        self.assertEqual(1, Attachment.objects.count())
+        self.assertEqual(1, Attachment.objects.filter(id=att.id).count())
+        att.delete()
+        self.assertEqual(0, Attachment.objects.count())
+
+    def test_primary_filter_by_binary_pk_as_str(self):
+        raise SkipTest("Querying by id as string is not currently supported")
+
+        class Attachment(Document):
+            id = BinaryField(primary_key=True)
+
+        Attachment.drop_collection()
+        binary_id = uuid.uuid4().bytes
+        att = Attachment(id=binary_id).save()
+        self.assertEqual(1, Attachment.objects.filter(id=binary_id).count())
+        att.delete()
+        self.assertEqual(0, Attachment.objects.count())
+
+    def test_match_querying_with_bytes(self):
+        class MyDocument(Document):
+            bin_field = BinaryField()
+
+        MyDocument.drop_collection()
+
+        doc = MyDocument(bin_field=BIN_VALUE).save()
+        matched_doc = MyDocument.objects(bin_field=BIN_VALUE).first()
+        self.assertEqual(matched_doc.id, doc.id)
+
+    def test_match_querying_with_binary(self):
+        class MyDocument(Document):
+            bin_field = BinaryField()
+
+        MyDocument.drop_collection()
+
+        doc = MyDocument(bin_field=BIN_VALUE).save()
+
+        matched_doc = MyDocument.objects(bin_field=Binary(BIN_VALUE)).first()
+        self.assertEqual(matched_doc.id, doc.id)
+
+    def test_modify_operation__set(self):
+        """Ensures no regression of bug #1127"""
+        class MyDocument(Document):
+            some_field = StringField()
+            bin_field = BinaryField()
+
+        MyDocument.drop_collection()
+
+        doc = MyDocument.objects(some_field='test').modify(
+            upsert=True, new=True,
+            set__bin_field=BIN_VALUE
+        )
+        self.assertEqual(doc.some_field, 'test')
+        if six.PY3:
+            self.assertEqual(doc.bin_field, BIN_VALUE)
+        else:
+            self.assertEqual(doc.bin_field, Binary(BIN_VALUE))
+
+    def test_update_one(self):
+        """Ensures no regression of bug #1127"""
+        class MyDocument(Document):
+            bin_field = BinaryField()
+
+        MyDocument.drop_collection()
+
+        bin_data = six.b('\xe6\x00\xc4\xff\x07')
+        doc = MyDocument(bin_field=bin_data).save()
+
+        n_updated = MyDocument.objects(bin_field=bin_data).update_one(bin_field=BIN_VALUE)
+        self.assertEqual(n_updated, 1)
+        fetched = MyDocument.objects.with_id(doc.id)
+        if six.PY3:
+            self.assertEqual(fetched.bin_field, BIN_VALUE)
+        else:
+            self.assertEqual(fetched.bin_field, Binary(BIN_VALUE))

tests/queryset/queryset.py

@@ -4641,8 +4641,6 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(doc_objects, Doc.objects.from_json(json_data))
 
     def test_as_pymongo(self):
-        from decimal import Decimal
-
         class LastLogin(EmbeddedDocument):
             location = StringField()
             ip = StringField()
@@ -4707,6 +4705,24 @@ class QuerySetTest(unittest.TestCase):
             }
         })
 
+    def test_as_pymongo_returns_cls_attribute_when_using_inheritance(self):
+        class User(Document):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        User.drop_collection()
+
+        user = User(name="Bob Dole").save()
+        result = User.objects.as_pymongo().first()
+        self.assertEqual(
+            result,
+            {
+                '_cls': 'User',
+                '_id': user.id,
+                'name': 'Bob Dole'
+            }
+        )
+
     def test_as_pymongo_json_limit_fields(self):
 
         class User(Document):