From b04dc90cdf4e71afb38594af7113a93302527a31 Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Thu, 20 Dec 2018 00:39:32 +0200 Subject: [PATCH 1/6] Bring back _cls and _id fields just as they return in pymongo --- mongoengine/queryset/base.py | 5 +---- tests/queryset/queryset.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 862a1826..5f11c01e 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1845,10 +1845,7 @@ class BaseQuerySet(object): # remove it from the doc (we always fetch it so that we can properly # construct documents). fields = self._loaded_fields - if fields and '_id' in doc and ( - (fields.value == QueryFieldList.ONLY and '_id' not in fields.fields) or - (fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields) - ): + if fields and '_id' in doc and fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields: del doc['_id'] return doc diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b44c36df..586913aa 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4635,7 +4635,7 @@ class QuerySetTest(unittest.TestCase): ip = StringField() class User(Document): - id = ObjectIdField('_id') + id = StringField(primary_key=True) name = StringField() age = IntField() price = DecimalField() @@ -4643,9 +4643,10 @@ class QuerySetTest(unittest.TestCase): User.drop_collection() - User.objects.create(name="Bob Dole", age=89, price=Decimal('1.11')) + User.objects.create(id='Bob', name="Bob Dole", age=89, price=Decimal('1.11')) User.objects.create( - name="Barack Obama", + id='Barak', + name="Barak Obama", age=51, price=Decimal('2.22'), last_login=LastLogin( @@ -4673,7 +4674,7 @@ class QuerySetTest(unittest.TestCase): self.assertIsInstance(results[1], dict) self.assertEqual(results[0]['name'], 'Bob Dole') self.assertEqual(results[0]['price'], 1.11) - self.assertEqual(results[1]['name'], 'Barack Obama') + self.assertEqual(results[1]['name'], 'Barak Obama') self.assertEqual(results[1]['price'], 2.22) users = User.objects.only('name', 'last_login').as_pymongo() @@ -4681,10 +4682,12 @@ class QuerySetTest(unittest.TestCase): self.assertIsInstance(results[0], dict) self.assertIsInstance(results[1], dict) self.assertEqual(results[0], { + '_id': 'Bob', 'name': 'Bob Dole' }) self.assertEqual(results[1], { - 'name': 'Barack Obama', + '_id': 'Barak', + 'name': 'Barak Obama', 'last_login': { 'location': 'White House', 'ip': '104.107.108.116' @@ -4701,8 +4704,8 @@ class QuerySetTest(unittest.TestCase): db_field='password_salt', required=True) User.drop_collection() - User(email="ross@example.com", password_salt="SomeSalt", - password_hash="SomeHash").save() + user = User(email="ross@example.com", password_salt="SomeSalt", + password_hash="SomeHash").save() serialized_user = User.objects.exclude( 'password_salt', 'password_hash').as_pymongo()[0] @@ -4714,11 +4717,16 @@ class QuerySetTest(unittest.TestCase): serialized_user = User.objects.exclude( 'password_salt').only('email').as_pymongo()[0] + self.assertEqual(set(['_id', 'email']), set(serialized_user.keys())) + + serialized_user = User.objects.exclude( + 'password_salt', 'id').only('email').as_pymongo()[0] self.assertEqual(set(['email']), set(serialized_user.keys())) serialized_user = User.objects.exclude( 'password_salt').only('email').to_json() - self.assertEqual('[{"email": "ross@example.com"}]', serialized_user) + self.assertEqual('[{"_id": {"$oid": "%s"}, "email": "ross@example.com"}]' % user.id, + serialized_user) def test_only_after_count(self): """Test that only() works after count()""" From 6fbdde63d8d39e00f9d33c8f5ed0d55caee09de6 Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Thu, 20 Dec 2018 00:57:02 +0200 Subject: [PATCH 2/6] Fix id in test_only_after_count ut --- tests/queryset/queryset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 586913aa..7490cad3 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4736,19 +4736,19 @@ class QuerySetTest(unittest.TestCase): age = IntField() address = StringField() User.drop_collection() - User(name="User", age=50, - address="Moscow, Russia").save() + user = User(name="User", age=50, + address="Moscow, Russia").save() user_queryset = User.objects(age=50) result = user_queryset.only("name", "age").as_pymongo().first() - self.assertEqual(result, {"name": "User", "age": 50}) + self.assertEqual(result, {"_id": user.id, "name": "User", "age": 50}) result = user_queryset.count() self.assertEqual(result, 1) result = user_queryset.only("name", "age").as_pymongo().first() - self.assertEqual(result, {"name": "User", "age": 50}) + self.assertEqual(result, {"_id": user.id, "name": "User", "age": 50}) def test_no_dereference(self): From 7fd4f792ba30d1ac8061bdb017fe7703493ff0e0 Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Thu, 20 Dec 2018 01:02:42 +0200 Subject: [PATCH 3/6] Don't hide _cls field when using as_pymongo --- mongoengine/queryset/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 5f11c01e..5493ffac 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1837,10 +1837,6 @@ class BaseQuerySet(object): """Clean up a PyMongo doc, removing fields that were only fetched for the sake of MongoEngine's implementation, and return it. """ - # Always remove _cls as a MongoEngine's implementation detail. - if '_cls' in doc: - del doc['_cls'] - # If the _id was not included in a .only or was excluded in a .exclude, # remove it from the doc (we always fetch it so that we can properly # construct documents). From 363aefe399efc6acc035a846241acd136ca8277d Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Thu, 20 Dec 2018 01:09:28 +0200 Subject: [PATCH 4/6] Get rid of _get_as_pymongo wrapper altogether as we return raw output --- mongoengine/queryset/base.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 5493ffac..8ae88422 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -188,7 +188,7 @@ class BaseQuerySet(object): ) if queryset._as_pymongo: - return queryset._get_as_pymongo(queryset._cursor[key]) + return queryset._cursor[key] return queryset._document._from_son( queryset._cursor[key], @@ -690,7 +690,7 @@ class BaseQuerySet(object): self._document._from_son(doc, only_fields=self.only_fields)) elif self._as_pymongo: for doc in docs: - doc_map[doc['_id']] = self._get_as_pymongo(doc) + doc_map[doc['_id']] = doc else: for doc in docs: doc_map[doc['_id']] = self._document._from_son( @@ -1486,7 +1486,7 @@ class BaseQuerySet(object): raw_doc = six.next(self._cursor) if self._as_pymongo: - return self._get_as_pymongo(raw_doc) + return raw_doc doc = self._document._from_son( raw_doc, _auto_dereference=self._auto_dereference, @@ -1833,19 +1833,6 @@ class BaseQuerySet(object): return tuple(data) - def _get_as_pymongo(self, doc): - """Clean up a PyMongo doc, removing fields that were only fetched - for the sake of MongoEngine's implementation, and return it. - """ - # If the _id was not included in a .only or was excluded in a .exclude, - # remove it from the doc (we always fetch it so that we can properly - # construct documents). - fields = self._loaded_fields - if fields and '_id' in doc and fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields: - del doc['_id'] - - return doc - def _sub_js_fields(self, code): """When fields are specified with [~fieldname] syntax, where *fieldname* is the Python name of a field, *fieldname* will be From 56d9f7a8af1b628c52c69dfa23a6d4374d2a2ac3 Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Thu, 20 Dec 2018 01:14:23 +0200 Subject: [PATCH 5/6] Fix ut to pass constantly --- tests/queryset/queryset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7490cad3..faadfe0c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4704,8 +4704,8 @@ class QuerySetTest(unittest.TestCase): db_field='password_salt', required=True) User.drop_collection() - user = User(email="ross@example.com", password_salt="SomeSalt", - password_hash="SomeHash").save() + User(email="ross@example.com", password_salt="SomeSalt", + password_hash="SomeHash").save() serialized_user = User.objects.exclude( 'password_salt', 'password_hash').as_pymongo()[0] @@ -4724,8 +4724,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(set(['email']), set(serialized_user.keys())) serialized_user = User.objects.exclude( - 'password_salt').only('email').to_json() - self.assertEqual('[{"_id": {"$oid": "%s"}, "email": "ross@example.com"}]' % user.id, + 'password_salt', 'id').only('email').to_json() + self.assertEqual('[{"email": "ross@example.com"}]', serialized_user) def test_only_after_count(self): From c6c68abfccff298fbe3f173aa337fe6e62d85d4d Mon Sep 17 00:00:00 2001 From: Eddie Linder Date: Fri, 11 Jan 2019 19:23:47 +0200 Subject: [PATCH 6/6] Update changelog and add another ut case --- docs/changelog.rst | 1 + tests/queryset/queryset.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index def5cc34..d8bed7e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Fix .only() working improperly after using .count() of the same instance of QuerySet +- POTENTIAL BREAKING CHANGE: All result fields are now passed, including internal fields (_cls, _id) when using `QuerySet.as_pymongo` #1976 ================= Changes in 0.16.3 diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index faadfe0c..c183aa86 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4709,19 +4709,22 @@ class QuerySetTest(unittest.TestCase): serialized_user = User.objects.exclude( 'password_salt', 'password_hash').as_pymongo()[0] - self.assertEqual(set(['_id', 'email']), set(serialized_user.keys())) + self.assertEqual({'_id', 'email'}, set(serialized_user.keys())) serialized_user = User.objects.exclude( 'id', 'password_salt', 'password_hash').to_json() self.assertEqual('[{"email": "ross@example.com"}]', serialized_user) + serialized_user = User.objects.only('email').as_pymongo()[0] + self.assertEqual({'_id', 'email'}, set(serialized_user.keys())) + serialized_user = User.objects.exclude( 'password_salt').only('email').as_pymongo()[0] - self.assertEqual(set(['_id', 'email']), set(serialized_user.keys())) + self.assertEqual({'_id', 'email'}, set(serialized_user.keys())) serialized_user = User.objects.exclude( 'password_salt', 'id').only('email').as_pymongo()[0] - self.assertEqual(set(['email']), set(serialized_user.keys())) + self.assertEqual({'email'}, set(serialized_user.keys())) serialized_user = User.objects.exclude( 'password_salt', 'id').only('email').to_json()