From 54d8c64ad5c643c1a3acda82e1b9f6da22e73146 Mon Sep 17 00:00:00 2001 From: Freyr Date: Thu, 25 May 2017 10:44:29 -0400 Subject: [PATCH 1/5] Fixes query operations on BinaryFields Overrides prepare_query_value in BinaryField to wrap the query value in bson.binary.Binary. This was already done in the to_mongo call when saving a document, but query operations required the user to convert the value manually. Fixes https://github.com/MongoEngine/mongoengine/issues/1127 --- mongoengine/fields.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0d402712..c52c6786 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1393,6 +1393,12 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') + def prepare_query_value(self, op, value): + if value is None: + return value + return super(BinaryField, self).prepare_query_value( + op, self.to_mongo(value)) + class GridFSError(Exception): pass From 2919e6765c71abe6a07ed5a8fefa5f8e6a31d5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 17 Feb 2019 21:45:05 +0100 Subject: [PATCH 2/5] separate test on binary field (#1983) --- tests/fields/fields.py | 78 +---------------------- tests/fields/test_binary_field.py | 100 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 75 deletions(-) create mode 100644 tests/fields/test_binary_field.py diff --git a/tests/fields/fields.py b/tests/fields/fields.py index b09c0a2d..05810f2c 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -17,7 +17,7 @@ except ImportError: from decimal import Decimal -from bson import Binary, DBRef, ObjectId, SON +from bson import DBRef, ObjectId, SON try: from bson.int64 import Int64 except ImportError: @@ -1699,7 +1699,7 @@ class FieldTest(MongoDBTestCase): post.save() post = BlogPost() - post.info = {'title' : 'dollar_sign', 'details' : {'te$t' : 'test'} } + post.info = {'title': 'dollar_sign', 'details': {'te$t': 'test'}} post.save() post = BlogPost() @@ -1718,7 +1718,7 @@ class FieldTest(MongoDBTestCase): post = BlogPost.objects.filter(info__title__exact='dollar_sign').first() self.assertIn('te$t', post['info']['details']) - + # Confirm handles non strings or non existing keys self.assertEqual( BlogPost.objects.filter(info__details__test__exact=5).count(), 0) @@ -2934,78 +2934,6 @@ class FieldTest(MongoDBTestCase): doc = Doc.objects.get(ref=doc1.pk) self.assertEqual(doc, doc2) - def test_binary_fields(self): - """Ensure that binary fields can be stored and retrieved. - """ - class Attachment(Document): - content_type = StringField() - blob = BinaryField() - - BLOB = six.b('\xe6\x00\xc4\xff\x07') - MIME_TYPE = 'application/octet-stream' - - Attachment.drop_collection() - - attachment = Attachment(content_type=MIME_TYPE, blob=BLOB) - attachment.save() - - attachment_1 = Attachment.objects().first() - self.assertEqual(MIME_TYPE, attachment_1.content_type) - self.assertEqual(BLOB, six.binary_type(attachment_1.blob)) - - def test_binary_validation_succeeds(self): - """Ensure that valid values can be assigned to binary fields. - """ - class AttachmentRequired(Document): - blob = BinaryField(required=True) - - class AttachmentSizeLimit(Document): - blob = BinaryField(max_bytes=4) - - attachment_required = AttachmentRequired() - self.assertRaises(ValidationError, attachment_required.validate) - attachment_required.blob = Binary(six.b('\xe6\x00\xc4\xff\x07')) - attachment_required.validate() - - _5_BYTES = six.b('\xe6\x00\xc4\xff\x07') - _4_BYTES = six.b('\xe6\x00\xc4\xff') - self.assertRaises(ValidationError, AttachmentSizeLimit(blob=_5_BYTES).validate) - AttachmentSizeLimit(blob=_4_BYTES).validate() - - def test_binary_validation_fails(self): - """Ensure that invalid values cannot be assigned to binary fields.""" - - class Attachment(Document): - blob = BinaryField() - - for invalid_data in (2, u'Im_a_unicode', ['some_str']): - self.assertRaises(ValidationError, Attachment(blob=invalid_data).validate) - - def test_binary_field_primary(self): - class Attachment(Document): - id = BinaryField(primary_key=True) - - Attachment.drop_collection() - binary_id = uuid.uuid4().bytes - att = Attachment(id=binary_id).save() - self.assertEqual(1, Attachment.objects.count()) - self.assertEqual(1, Attachment.objects.filter(id=att.id).count()) - att.delete() - self.assertEqual(0, Attachment.objects.count()) - - def test_binary_field_primary_filter_by_binary_pk_as_str(self): - raise SkipTest("Querying by id as string is not currently supported") - - class Attachment(Document): - id = BinaryField(primary_key=True) - - Attachment.drop_collection() - binary_id = uuid.uuid4().bytes - att = Attachment(id=binary_id).save() - self.assertEqual(1, Attachment.objects.filter(id=binary_id).count()) - att.delete() - self.assertEqual(0, Attachment.objects.count()) - def test_choices_allow_using_sets_as_choices(self): """Ensure that sets can be used when setting choices """ diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py new file mode 100644 index 00000000..d96f59e9 --- /dev/null +++ b/tests/fields/test_binary_field.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +import datetime +import unittest +import uuid +import math +import itertools +import re +import sys + +from nose.plugins.skip import SkipTest +import six + +try: + import dateutil +except ImportError: + dateutil = None + +from bson import Binary + +try: + from bson.int64 import Int64 +except ImportError: + Int64 = long + +from mongoengine import * +from tests.utils import MongoDBTestCase + + +class TestBinaryField(MongoDBTestCase): + def test_binary_fields(self): + """Ensure that binary fields can be stored and retrieved. + """ + class Attachment(Document): + content_type = StringField() + blob = BinaryField() + + BLOB = six.b('\xe6\x00\xc4\xff\x07') + MIME_TYPE = 'application/octet-stream' + + Attachment.drop_collection() + + attachment = Attachment(content_type=MIME_TYPE, blob=BLOB) + attachment.save() + + attachment_1 = Attachment.objects().first() + self.assertEqual(MIME_TYPE, attachment_1.content_type) + self.assertEqual(BLOB, six.binary_type(attachment_1.blob)) + + def test_binary_validation_succeeds(self): + """Ensure that valid values can be assigned to binary fields. + """ + class AttachmentRequired(Document): + blob = BinaryField(required=True) + + class AttachmentSizeLimit(Document): + blob = BinaryField(max_bytes=4) + + attachment_required = AttachmentRequired() + self.assertRaises(ValidationError, attachment_required.validate) + attachment_required.blob = Binary(six.b('\xe6\x00\xc4\xff\x07')) + attachment_required.validate() + + _5_BYTES = six.b('\xe6\x00\xc4\xff\x07') + _4_BYTES = six.b('\xe6\x00\xc4\xff') + self.assertRaises(ValidationError, AttachmentSizeLimit(blob=_5_BYTES).validate) + AttachmentSizeLimit(blob=_4_BYTES).validate() + + def test_binary_validation_fails(self): + """Ensure that invalid values cannot be assigned to binary fields.""" + + class Attachment(Document): + blob = BinaryField() + + for invalid_data in (2, u'Im_a_unicode', ['some_str']): + self.assertRaises(ValidationError, Attachment(blob=invalid_data).validate) + + def test_binary_field_primary(self): + class Attachment(Document): + id = BinaryField(primary_key=True) + + Attachment.drop_collection() + binary_id = uuid.uuid4().bytes + att = Attachment(id=binary_id).save() + self.assertEqual(1, Attachment.objects.count()) + self.assertEqual(1, Attachment.objects.filter(id=att.id).count()) + att.delete() + self.assertEqual(0, Attachment.objects.count()) + + def test_binary_field_primary_filter_by_binary_pk_as_str(self): + raise SkipTest("Querying by id as string is not currently supported") + + class Attachment(Document): + id = BinaryField(primary_key=True) + + Attachment.drop_collection() + binary_id = uuid.uuid4().bytes + att = Attachment(id=binary_id).save() + self.assertEqual(1, Attachment.objects.filter(id=binary_id).count()) + att.delete() + self.assertEqual(0, Attachment.objects.count()) From e02261be6d86489945ad022ed73ae45c39fcc9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 17 Feb 2019 21:54:36 +0100 Subject: [PATCH 3/5] add test coverage for #1557 --- docs/changelog.rst | 1 + tests/fields/test_binary_field.py | 77 +++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d8bed7e6..25e1a585 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development - (Fill this out as you fix issues and develop your features). - Fix .only() working improperly after using .count() of the same instance of QuerySet - POTENTIAL BREAKING CHANGE: All result fields are now passed, including internal fields (_cls, _id) when using `QuerySet.as_pymongo` #1976 +- Fix InvalidStringData error when using modify on a BinaryField #1127 ================= Changes in 0.16.3 diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py index d96f59e9..b0427766 100644 --- a/tests/fields/test_binary_field.py +++ b/tests/fields/test_binary_field.py @@ -1,30 +1,16 @@ # -*- coding: utf-8 -*- -import datetime -import unittest import uuid -import math -import itertools -import re -import sys from nose.plugins.skip import SkipTest import six -try: - import dateutil -except ImportError: - dateutil = None - from bson import Binary -try: - from bson.int64 import Int64 -except ImportError: - Int64 = long - from mongoengine import * from tests.utils import MongoDBTestCase +BIN_VALUE = six.b('\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde\xe6\x87\xb1\x7f\xc6\xe6\xd9r\x18\xf5') + class TestBinaryField(MongoDBTestCase): def test_binary_fields(self): @@ -46,7 +32,7 @@ class TestBinaryField(MongoDBTestCase): self.assertEqual(MIME_TYPE, attachment_1.content_type) self.assertEqual(BLOB, six.binary_type(attachment_1.blob)) - def test_binary_validation_succeeds(self): + def test_validation_succeeds(self): """Ensure that valid values can be assigned to binary fields. """ class AttachmentRequired(Document): @@ -65,7 +51,7 @@ class TestBinaryField(MongoDBTestCase): self.assertRaises(ValidationError, AttachmentSizeLimit(blob=_5_BYTES).validate) AttachmentSizeLimit(blob=_4_BYTES).validate() - def test_binary_validation_fails(self): + def test_validation_fails(self): """Ensure that invalid values cannot be assigned to binary fields.""" class Attachment(Document): @@ -74,7 +60,7 @@ class TestBinaryField(MongoDBTestCase): for invalid_data in (2, u'Im_a_unicode', ['some_str']): self.assertRaises(ValidationError, Attachment(blob=invalid_data).validate) - def test_binary_field_primary(self): + def test__primary(self): class Attachment(Document): id = BinaryField(primary_key=True) @@ -86,7 +72,7 @@ class TestBinaryField(MongoDBTestCase): att.delete() self.assertEqual(0, Attachment.objects.count()) - def test_binary_field_primary_filter_by_binary_pk_as_str(self): + def test_primary_filter_by_binary_pk_as_str(self): raise SkipTest("Querying by id as string is not currently supported") class Attachment(Document): @@ -98,3 +84,54 @@ class TestBinaryField(MongoDBTestCase): self.assertEqual(1, Attachment.objects.filter(id=binary_id).count()) att.delete() self.assertEqual(0, Attachment.objects.count()) + + def test_match_querying_with_bytes(self): + class MyDocument(Document): + bin_field = BinaryField() + + MyDocument.drop_collection() + + doc = MyDocument(bin_field=BIN_VALUE).save() + matched_doc = MyDocument.objects(bin_field=BIN_VALUE).first() + self.assertEqual(matched_doc.id, doc.id) + + def test_match_querying_with_binary(self): + class MyDocument(Document): + bin_field = BinaryField() + + MyDocument.drop_collection() + + doc = MyDocument(bin_field=BIN_VALUE).save() + + matched_doc = MyDocument.objects(bin_field=Binary(BIN_VALUE)).first() + self.assertEqual(matched_doc.id, doc.id) + + def test_modify_operation__set(self): + """Ensures no regression of bug #1127""" + class MyDocument(Document): + some_field = StringField() + bin_field = BinaryField() + + MyDocument.drop_collection() + + doc = MyDocument.objects(some_field='test').modify( + upsert=True, new=True, + set__bin_field=BIN_VALUE + ) + self.assertEqual(doc.some_field, 'test') + self.assertEqual(doc.bin_field, Binary(BIN_VALUE)) + + def test_update_one(self): + """Ensures no regression of bug #1127""" + class MyDocument(Document): + bin_field = BinaryField() + + MyDocument.drop_collection() + + bin_data = six.b('\xe6\x00\xc4\xff\x07') + doc = MyDocument(bin_field=bin_data).save() + + n_updated = MyDocument.objects(bin_field=bin_data).update_one(bin_field=BIN_VALUE) + self.assertEqual(n_updated, 1) + fetched = MyDocument.objects.with_id(doc.id) + self.assertEqual(fetched.bin_field, Binary(BIN_VALUE)) From f4873fee18793543266d10ac5d7fbd0341ba2442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 17 Feb 2019 22:50:42 +0100 Subject: [PATCH 4/5] add additional test for #1976 --- tests/queryset/queryset.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c183aa86..d3a2418a 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4628,8 +4628,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) def test_as_pymongo(self): - from decimal import Decimal - class LastLogin(EmbeddedDocument): location = StringField() ip = StringField() @@ -4694,6 +4692,24 @@ class QuerySetTest(unittest.TestCase): } }) + def test_as_pymongo_returns_cls_attribute_when_using_inheritance(self): + class User(Document): + name = StringField() + meta = {'allow_inheritance': True} + + User.drop_collection() + + user = User(name="Bob Dole").save() + result = User.objects.as_pymongo().first() + self.assertEqual( + result, + { + '_cls': 'User', + '_id': user.id, + 'name': 'Bob Dole' + } + ) + def test_as_pymongo_json_limit_fields(self): class User(Document): From 201f3008b1c451c93d112e5905fe7e64259a8d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 17 Feb 2019 22:56:56 +0100 Subject: [PATCH 5/5] Fix for py3 in binary field test --- tests/fields/test_binary_field.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py index b0427766..8af75d4e 100644 --- a/tests/fields/test_binary_field.py +++ b/tests/fields/test_binary_field.py @@ -119,7 +119,10 @@ class TestBinaryField(MongoDBTestCase): set__bin_field=BIN_VALUE ) self.assertEqual(doc.some_field, 'test') - self.assertEqual(doc.bin_field, Binary(BIN_VALUE)) + if six.PY3: + self.assertEqual(doc.bin_field, BIN_VALUE) + else: + self.assertEqual(doc.bin_field, Binary(BIN_VALUE)) def test_update_one(self): """Ensures no regression of bug #1127""" @@ -134,4 +137,7 @@ class TestBinaryField(MongoDBTestCase): n_updated = MyDocument.objects(bin_field=bin_data).update_one(bin_field=BIN_VALUE) self.assertEqual(n_updated, 1) fetched = MyDocument.objects.with_id(doc.id) - self.assertEqual(fetched.bin_field, Binary(BIN_VALUE)) + if six.PY3: + self.assertEqual(fetched.bin_field, BIN_VALUE) + else: + self.assertEqual(fetched.bin_field, Binary(BIN_VALUE))