Merge pull request #1976 from 3ddi/eddie/revert_hidden_fields

Bring back _cls and _id fields just as they return in pymongo

docs/changelog.rst

@@ -6,6 +6,7 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 - Fix .only() working improperly after using .count() of the same instance of QuerySet
+- POTENTIAL BREAKING CHANGE: All result fields are now passed, including internal fields (_cls, _id) when using `QuerySet.as_pymongo` #1976
 
 =================
 Changes in 0.16.3

mongoengine/queryset/base.py

@@ -188,7 +188,7 @@ class BaseQuerySet(object):
                 )
 
             if queryset._as_pymongo:
-                return queryset._get_as_pymongo(queryset._cursor[key])
+                return queryset._cursor[key]
 
             return queryset._document._from_son(
                 queryset._cursor[key],
@@ -690,7 +690,7 @@ class BaseQuerySet(object):
                     self._document._from_son(doc, only_fields=self.only_fields))
         elif self._as_pymongo:
             for doc in docs:
-                doc_map[doc['_id']] = self._get_as_pymongo(doc)
+                doc_map[doc['_id']] = doc
         else:
             for doc in docs:
                 doc_map[doc['_id']] = self._document._from_son(
@@ -1486,7 +1486,7 @@ class BaseQuerySet(object):
         raw_doc = six.next(self._cursor)
 
         if self._as_pymongo:
-            return self._get_as_pymongo(raw_doc)
+            return raw_doc
 
         doc = self._document._from_son(
             raw_doc, _auto_dereference=self._auto_dereference,
@@ -1833,26 +1833,6 @@ class BaseQuerySet(object):
 
         return tuple(data)
 
-    def _get_as_pymongo(self, doc):
-        """Clean up a PyMongo doc, removing fields that were only fetched
-        for the sake of MongoEngine's implementation, and return it.
-        """
-        # Always remove _cls as a MongoEngine's implementation detail.
-        if '_cls' in doc:
-            del doc['_cls']
-
-        # If the _id was not included in a .only or was excluded in a .exclude,
-        # remove it from the doc (we always fetch it so that we can properly
-        # construct documents).
-        fields = self._loaded_fields
-        if fields and '_id' in doc and (
-            (fields.value == QueryFieldList.ONLY and '_id' not in fields.fields) or
-            (fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields)
-        ):
-            del doc['_id']
-
-        return doc
-
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where
         *fieldname* is the Python name of a field, *fieldname* will be

tests/queryset/queryset.py

@@ -4635,7 +4635,7 @@ class QuerySetTest(unittest.TestCase):
             ip = StringField()
 
         class User(Document):
-            id = ObjectIdField('_id')
+            id = StringField(primary_key=True)
             name = StringField()
             age = IntField()
             price = DecimalField()
@@ -4643,9 +4643,10 @@ class QuerySetTest(unittest.TestCase):
 
         User.drop_collection()
 
-        User.objects.create(name="Bob Dole", age=89, price=Decimal('1.11'))
+        User.objects.create(id='Bob', name="Bob Dole", age=89, price=Decimal('1.11'))
         User.objects.create(
-            name="Barack Obama",
+            id='Barak',
+            name="Barak Obama",
             age=51,
             price=Decimal('2.22'),
             last_login=LastLogin(
@@ -4673,7 +4674,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertIsInstance(results[1], dict)
         self.assertEqual(results[0]['name'], 'Bob Dole')
         self.assertEqual(results[0]['price'], 1.11)
-        self.assertEqual(results[1]['name'], 'Barack Obama')
+        self.assertEqual(results[1]['name'], 'Barak Obama')
         self.assertEqual(results[1]['price'], 2.22)
 
         users = User.objects.only('name', 'last_login').as_pymongo()
@@ -4681,10 +4682,12 @@ class QuerySetTest(unittest.TestCase):
         self.assertIsInstance(results[0], dict)
         self.assertIsInstance(results[1], dict)
         self.assertEqual(results[0], {
+            '_id': 'Bob',
             'name': 'Bob Dole'
         })
         self.assertEqual(results[1], {
-            'name': 'Barack Obama',
+            '_id': 'Barak',
+            'name': 'Barak Obama',
             'last_login': {
                 'location': 'White House',
                 'ip': '104.107.108.116'
@@ -4706,19 +4709,27 @@ class QuerySetTest(unittest.TestCase):
 
         serialized_user = User.objects.exclude(
             'password_salt', 'password_hash').as_pymongo()[0]
-        self.assertEqual(set(['_id', 'email']), set(serialized_user.keys()))
+        self.assertEqual({'_id', 'email'}, set(serialized_user.keys()))
 
         serialized_user = User.objects.exclude(
             'id', 'password_salt', 'password_hash').to_json()
         self.assertEqual('[{"email": "ross@example.com"}]', serialized_user)
 
-        serialized_user = User.objects.exclude(
+        serialized_user = User.objects.only('email').as_pymongo()[0]
-            'password_salt').only('email').as_pymongo()[0]
+        self.assertEqual({'_id', 'email'}, set(serialized_user.keys()))
-        self.assertEqual(set(['email']), set(serialized_user.keys()))
 
         serialized_user = User.objects.exclude(
-            'password_salt').only('email').to_json()
+            'password_salt').only('email').as_pymongo()[0]
-        self.assertEqual('[{"email": "ross@example.com"}]', serialized_user)
+        self.assertEqual({'_id', 'email'}, set(serialized_user.keys()))
+
+        serialized_user = User.objects.exclude(
+            'password_salt', 'id').only('email').as_pymongo()[0]
+        self.assertEqual({'email'}, set(serialized_user.keys()))
+
+        serialized_user = User.objects.exclude(
+            'password_salt', 'id').only('email').to_json()
+        self.assertEqual('[{"email": "ross@example.com"}]',
+                         serialized_user)
 
     def test_only_after_count(self):
         """Test that only() works after count()"""
@@ -4728,19 +4739,19 @@ class QuerySetTest(unittest.TestCase):
             age = IntField()
             address = StringField()
         User.drop_collection()
-        User(name="User", age=50,
+        user = User(name="User", age=50,
                     address="Moscow, Russia").save()
 
         user_queryset = User.objects(age=50)
 
         result = user_queryset.only("name", "age").as_pymongo().first()
-        self.assertEqual(result, {"name": "User", "age": 50})
+        self.assertEqual(result, {"_id": user.id, "name": "User", "age": 50})
 
         result = user_queryset.count()
         self.assertEqual(result, 1)
 
         result = user_queryset.only("name", "age").as_pymongo().first()
-        self.assertEqual(result, {"name": "User", "age": 50})
+        self.assertEqual(result, {"_id": user.id, "name": "User", "age": 50})
 
     def test_no_dereference(self):