extra note in the docstring [ci skip]

This reverts commit 944d1c0a4a.

.travis.yml

@@ -16,8 +16,6 @@ python:
 - 2.7
 - 3.5
 - pypy
-- pypy3.3-5.2-alpha1
-
 
 env:
 - MONGODB=2.6 PYMONGO=2.7

mongoengine/base/fields.py

@@ -81,7 +81,14 @@ class BaseField(object):
         self.sparse = sparse
         self._owner_document = None
 
-        # Validate the db_field
+        # Make sure db_field is a string (if it's explicitly defined).
+        if (
+            self.db_field is not None and
+            not isinstance(self.db_field, six.string_types)
+        ):
+            raise TypeError('db_field should be a string.')
+
+        # Make sure db_field doesn't contain any forbidden characters.
         if isinstance(self.db_field, six.string_types) and (
             '.' in self.db_field or
             '\0' in self.db_field or

mongoengine/dereference.py

@@ -1,4 +1,3 @@
-from collections import OrderedDict
 from bson import DBRef, SON
 import six
 
@@ -202,10 +201,6 @@ class DeReference(object):
             as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
-        elif isinstance(items, OrderedDict):
-            is_list = False
-            iterator = items.iteritems()
-            data = OrderedDict()
         else:
             is_list = False
             iterator = items.iteritems()

mongoengine/fields.py

@@ -6,7 +6,6 @@ import socket
 import time
 import uuid
 import warnings
-from collections import Mapping
 from operator import itemgetter
 
 from bson import Binary, DBRef, ObjectId, SON
@@ -705,14 +704,6 @@ class DynamicField(BaseField):
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
-    def __init__(self, container_class=dict, *args, **kwargs):
-        self._container_cls = container_class
-        if not issubclass(self._container_cls, Mapping):
-            self.error('The class that is specified in `container_class` parameter '
-                       'must be a subclass of `dict`.')
-
-        super(DynamicField, self).__init__(*args, **kwargs)
-
     def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB compatible type.
         """
@@ -738,7 +729,7 @@ class DynamicField(BaseField):
             is_list = True
             value = {k: v for k, v in enumerate(value)}
 
-        data = self._container_cls()
+        data = {}
         for k, v in value.iteritems():
             data[k] = self.to_mongo(v, use_db_field, fields)
 

mongoengine/queryset/base.py

@@ -67,7 +67,6 @@ class BaseQuerySet(object):
         self._scalar = []
         self._none = False
         self._as_pymongo = False
-        self._as_pymongo_coerce = False
         self._search_text = None
 
         # If inheritance is allowed, only return instances and instances of
@@ -728,11 +727,12 @@ class BaseQuerySet(object):
                 '%s is not a subclass of BaseQuerySet' % new_qs.__name__)
 
         copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj',
-                      '_where_clause', '_loaded_fields', '_ordering', '_snapshot',
+                      '_where_clause', '_loaded_fields', '_ordering',
-                      '_timeout', '_class_check', '_slave_okay', '_read_preference',
+                      '_snapshot', '_timeout', '_class_check', '_slave_okay',
-                      '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce',
+                      '_read_preference', '_iter', '_scalar', '_as_pymongo',
                       '_limit', '_skip', '_hint', '_auto_dereference',
-                      '_search_text', 'only_fields', '_max_time_ms', '_comment')
+                      '_search_text', 'only_fields', '_max_time_ms',
+                      '_comment')
 
         for prop in copy_props:
             val = getattr(self, prop)
@@ -939,7 +939,8 @@ class BaseQuerySet(object):
 
             posts = BlogPost.objects(...).fields(slice__comments=5)
 
-        :param kwargs: A set keywors arguments identifying what to include.
+        :param kwargs: A set of keyword arguments identifying what to
+            include, exclude, or slice.
 
         .. versionadded:: 0.5
         """
@@ -1128,16 +1129,15 @@ class BaseQuerySet(object):
         """An alias for scalar"""
         return self.scalar(*fields)
 
-    def as_pymongo(self, coerce_types=False):
+    def as_pymongo(self):
         """Instead of returning Document instances, return raw values from
         pymongo.
 
-        :param coerce_types: Field types (if applicable) would be use to
+        This method is particularly useful if you don't need dereferencing
-            coerce types.
+        and care primarily about the speed of data retrieval.
         """
         queryset = self.clone()
         queryset._as_pymongo = True
-        queryset._as_pymongo_coerce = coerce_types
         return queryset
 
     def max_time_ms(self, ms):
@@ -1799,59 +1799,25 @@ class BaseQuerySet(object):
 
         return tuple(data)
 
-    def _get_as_pymongo(self, row):
+    def _get_as_pymongo(self, doc):
-        # Extract which fields paths we should follow if .fields(...) was
+        """Clean up a PyMongo doc, removing fields that were only fetched
-        # used. If not, handle all fields.
+        for the sake of MongoEngine's implementation, and return it.
-        if not getattr(self, '__as_pymongo_fields', None):
+        """
-            self.__as_pymongo_fields = []
+        # Always remove _cls as a MongoEngine's implementation detail.
+        if '_cls' in doc:
+            del doc['_cls']
 
-            for field in self._loaded_fields.fields - set(['_cls']):
+        # If the _id was not included in a .only or was excluded in a .exclude,
-                self.__as_pymongo_fields.append(field)
+        # remove it from the doc (we always fetch it so that we can properly
-                while '.' in field:
+        # construct documents).
-                    field, _ = field.rsplit('.', 1)
+        fields = self._loaded_fields
-                    self.__as_pymongo_fields.append(field)
+        if fields and '_id' in doc and (
+            (fields.value == QueryFieldList.ONLY and '_id' not in fields.fields) or
+            (fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields)
+        ):
+            del doc['_id']
 
-        all_fields = not self.__as_pymongo_fields
+        return doc
-
-        def clean(data, path=None):
-            path = path or ''
-
-            if isinstance(data, dict):
-                new_data = {}
-                for key, value in data.iteritems():
-                    new_path = '%s.%s' % (path, key) if path else key
-
-                    if all_fields:
-                        include_field = True
-                    elif self._loaded_fields.value == QueryFieldList.ONLY:
-                        include_field = new_path in self.__as_pymongo_fields
-                    else:
-                        include_field = new_path not in self.__as_pymongo_fields
-
-                    if include_field:
-                        new_data[key] = clean(value, path=new_path)
-                data = new_data
-            elif isinstance(data, list):
-                data = [clean(d, path=path) for d in data]
-            else:
-                if self._as_pymongo_coerce:
-                    # If we need to coerce types, we need to determine the
-                    # type of this field and use the corresponding
-                    # .to_python(...)
-                    EmbeddedDocumentField = _import_class('EmbeddedDocumentField')
-
-                    obj = self._document
-                    for chunk in path.split('.'):
-                        obj = getattr(obj, chunk, None)
-                        if obj is None:
-                            break
-                        elif isinstance(obj, EmbeddedDocumentField):
-                            obj = obj.document_type
-                    if obj and data is not None:
-                        data = obj.to_python(data)
-            return data
-
-        return clean(row)
 
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where

tests/document/instance.py

@@ -242,7 +242,7 @@ class InstanceTest(unittest.TestCase):
         Zoo.drop_collection()
 
         class Zoo(Document):
-            animals = ListField(GenericReferenceField(Animal))
+            animals = ListField(GenericReferenceField())
 
         # Save a reference to each animal
         zoo = Zoo(animals=Animal.objects)

tests/fields/fields.py

@@ -5,11 +5,9 @@ import uuid
 import math
 import itertools
 import re
-import pymongo
 import sys
 
 from nose.plugins.skip import SkipTest
-from collections import OrderedDict
 import six
 
 try:
@@ -28,12 +26,9 @@ except ImportError:
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.base import (BaseDict, BaseField, EmbeddedDocumentList,
-                              _document_registry, TopLevelDocumentMetaclass)
+                              _document_registry)
 
-from tests.utils import MongoDBTestCase, MONGO_TEST_DB
+from tests.utils import MongoDBTestCase
-from mongoengine.python_support import IS_PYMONGO_3
-if IS_PYMONGO_3:
-    from bson import CodecOptions
 
 __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
 
@@ -4188,67 +4183,6 @@ class EmbeddedDocumentListFieldTestCase(MongoDBTestCase):
         self.assertTrue(hasattr(CustomData.c_field, 'custom_data'))
         self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a'])
 
-    def test_dynamicfield_with_container_class(self):
-        """
-        Tests that object can be stored in order by DynamicField class
-        with container_class parameter.
-        """
-        raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
-
-        class Doc(Document):
-            ordered_data = DynamicField(container_class=OrderedDict)
-            unordered_data = DynamicField()
-
-        Doc.drop_collection()
-
-        doc = Doc(ordered_data=OrderedDict(raw_data), unordered_data=dict(raw_data)).save()
-
-        # checks that the data is in order
-        self.assertEqual(type(doc.ordered_data), OrderedDict)
-        self.assertEqual(type(doc.unordered_data), dict)
-        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
-
-        # checks that the data is stored to the database in order
-        pymongo_db = pymongo.MongoClient()[MONGO_TEST_DB]
-        if IS_PYMONGO_3:
-            codec_option = CodecOptions(document_class=OrderedDict)
-            db_doc = pymongo_db.doc.with_options(codec_options=codec_option).find_one()
-        else:
-            db_doc = pymongo_db.doc.find_one(as_class=OrderedDict)
-
-        self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
-
-    def test_dynamicfield_with_wrong_container_class(self):
-        with self.assertRaises(ValidationError):
-            class DocWithInvalidField:
-                data = DynamicField(container_class=list)
-
-    def test_dynamicfield_with_wrong_container_class_and_reload_docuemnt(self):
-        # This is because 'codec_options' is supported on pymongo3 or later
-        if IS_PYMONGO_3:
-            class OrderedDocument(Document):
-                my_metaclass = TopLevelDocumentMetaclass
-                __metaclass__ = TopLevelDocumentMetaclass
-
-                @classmethod
-                def _get_collection(cls):
-                    collection = super(OrderedDocument, cls)._get_collection()
-                    opts = CodecOptions(document_class=OrderedDict)
-
-                    return collection.with_options(codec_options=opts)
-
-            raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
-
-            class Doc(OrderedDocument):
-                data = DynamicField(container_class=OrderedDict)
-
-            Doc.drop_collection()
-
-            doc = Doc(data=OrderedDict(raw_data)).save()
-            doc.reload()
-
-            self.assertEqual(type(doc.data), OrderedDict)
-            self.assertEqual(','.join(doc.data.keys()), 'd,c,b,a')
 
 class CachedReferenceFieldTest(MongoDBTestCase):
 

tests/queryset/queryset.py

@@ -4047,6 +4047,35 @@ class QuerySetTest(unittest.TestCase):
         plist = list(Person.objects.scalar('name', 'state'))
         self.assertEqual(plist, [(u'Wilson JR', s1)])
 
+    def test_generic_reference_field_with_only_and_as_pymongo(self):
+        class TestPerson(Document):
+            name = StringField()
+
+        class TestActivity(Document):
+            name = StringField()
+            owner = GenericReferenceField()
+
+        TestPerson.drop_collection()
+        TestActivity.drop_collection()
+
+        person = TestPerson(name='owner')
+        person.save()
+
+        a1 = TestActivity(name='a1', owner=person)
+        a1.save()
+
+        activity = TestActivity.objects(owner=person).scalar('id', 'owner').no_dereference().first()
+        self.assertEqual(activity[0], a1.pk)
+        self.assertEqual(activity[1]['_ref'], DBRef('test_person', person.pk))
+
+        activity = TestActivity.objects(owner=person).only('id', 'owner')[0]
+        self.assertEqual(activity.pk, a1.pk)
+        self.assertEqual(activity.owner, person)
+
+        activity = TestActivity.objects(owner=person).only('id', 'owner').as_pymongo().first()
+        self.assertEqual(activity['_id'], a1.pk)
+        self.assertTrue(activity['owner']['_ref'], DBRef('test_person', person.pk))
+
     def test_scalar_db_field(self):
 
         class TestDoc(Document):
@@ -4392,21 +4421,44 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(doc_objects, Doc.objects.from_json(json_data))
 
     def test_as_pymongo(self):
-
         from decimal import Decimal
 
+        class LastLogin(EmbeddedDocument):
+            location = StringField()
+            ip = StringField()
+
         class User(Document):
             id = ObjectIdField('_id')
             name = StringField()
             age = IntField()
             price = DecimalField()
+            last_login = EmbeddedDocumentField(LastLogin)
 
         User.drop_collection()
-        User(name="Bob Dole", age=89, price=Decimal('1.11')).save()
+
-        User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
+        User.objects.create(name="Bob Dole", age=89, price=Decimal('1.11'))
+        User.objects.create(
+            name="Barack Obama",
+            age=51,
+            price=Decimal('2.22'),
+            last_login=LastLogin(
+                location='White House',
+                ip='104.107.108.116'
+            )
+        )
+
+        results = User.objects.as_pymongo()
+        self.assertEqual(
+            set(results[0].keys()),
+            set(['_id', 'name', 'age', 'price'])
+        )
+        self.assertEqual(
+            set(results[1].keys()),
+            set(['_id', 'name', 'age', 'price', 'last_login'])
+        )
 
         results = User.objects.only('id', 'name').as_pymongo()
-        self.assertEqual(sorted(results[0].keys()), sorted(['_id', 'name']))
+        self.assertEqual(set(results[0].keys()), set(['_id', 'name']))
 
         users = User.objects.only('name', 'price').as_pymongo()
         results = list(users)
@@ -4417,16 +4469,20 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(results[1]['name'], 'Barack Obama')
         self.assertEqual(results[1]['price'], 2.22)
 
-        # Test coerce_types
+        users = User.objects.only('name', 'last_login').as_pymongo()
-        users = User.objects.only(
-            'name', 'price').as_pymongo(coerce_types=True)
         results = list(users)
         self.assertTrue(isinstance(results[0], dict))
         self.assertTrue(isinstance(results[1], dict))
-        self.assertEqual(results[0]['name'], 'Bob Dole')
+        self.assertEqual(results[0], {
-        self.assertEqual(results[0]['price'], Decimal('1.11'))
+            'name': 'Bob Dole'
-        self.assertEqual(results[1]['name'], 'Barack Obama')
+        })
-        self.assertEqual(results[1]['price'], Decimal('2.22'))
+        self.assertEqual(results[1], {
+            'name': 'Barack Obama',
+            'last_login': {
+                'location': 'White House',
+                'ip': '104.107.108.116'
+            }
+        })
 
     def test_as_pymongo_json_limit_fields(self):
 
@@ -4590,7 +4646,6 @@ class QuerySetTest(unittest.TestCase):
 
     def test_no_cache(self):
         """Ensure you can add meta data to file"""
-
         class Noddy(Document):
             fields = DictField()
 
@@ -4608,15 +4663,19 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(len(list(docs)), 100)
 
+        # Can't directly get a length of a no-cache queryset.
         with self.assertRaises(TypeError):
             len(docs)
 
+        # Another iteration over the queryset should result in another db op.
         with query_counter() as q:
-            self.assertEqual(q, 0)
             list(docs)
             self.assertEqual(q, 1)
+
+        # ... and another one to double-check.
+        with query_counter() as q:
             list(docs)
-            self.assertEqual(q, 2)
+            self.assertEqual(q, 1)
 
     def test_nested_queryset_iterator(self):
         # Try iterating the same queryset twice, nested.

tests/test_dereference.py

@@ -2,15 +2,10 @@
 import unittest
 
 from bson import DBRef, ObjectId
-from collections import OrderedDict
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter
-from mongoengine.python_support import IS_PYMONGO_3
-from mongoengine.base import TopLevelDocumentMetaclass
-if IS_PYMONGO_3:
-    from bson import CodecOptions
 
 
 class FieldTest(unittest.TestCase):
@@ -1292,70 +1287,5 @@ class FieldTest(unittest.TestCase):
 
             self.assertEqual(q, 2)
 
-    def test_dynamic_field_dereference(self):
-        class Merchandise(Document):
-            name = StringField()
-            price = IntField()
-
-        class Store(Document):
-            merchandises = DynamicField()
-
-        Merchandise.drop_collection()
-        Store.drop_collection()
-
-        merchandises = {
-            '#1': Merchandise(name='foo', price=100).save(),
-            '#2': Merchandise(name='bar', price=120).save(),
-            '#3': Merchandise(name='baz', price=110).save(),
-        }
-        Store(merchandises=merchandises).save()
-
-        store = Store.objects().first()
-        for obj in store.merchandises.values():
-            self.assertFalse(isinstance(obj, Merchandise))
-
-        store.select_related()
-        for obj in store.merchandises.values():
-            self.assertTrue(isinstance(obj, Merchandise))
-
-    def test_dynamic_field_dereference_with_ordering_guarantee_on_pymongo3(self):
-        # This is because 'codec_options' is supported on pymongo3 or later
-        if IS_PYMONGO_3:
-            class OrderedDocument(Document):
-                my_metaclass = TopLevelDocumentMetaclass
-                __metaclass__ = TopLevelDocumentMetaclass
-
-                @classmethod
-                def _get_collection(cls):
-                    collection = super(OrderedDocument, cls)._get_collection()
-                    opts = CodecOptions(document_class=OrderedDict)
-
-                    return collection.with_options(codec_options=opts)
-
-            class Merchandise(Document):
-                name = StringField()
-                price = IntField()
-
-            class Store(OrderedDocument):
-                merchandises = DynamicField(container_class=OrderedDict)
-
-            Merchandise.drop_collection()
-            Store.drop_collection()
-
-            merchandises = OrderedDict()
-            merchandises['#1'] = Merchandise(name='foo', price=100).save()
-            merchandises['#2'] = Merchandise(name='bar', price=120).save()
-            merchandises['#3'] = Merchandise(name='baz', price=110).save()
-
-            Store(merchandises=merchandises).save()
-
-            store = Store.objects().first()
-
-            store.select_related()
-
-            # confirms that the load data order is same with the one at storing
-            self.assertTrue(type(store.merchandises), OrderedDict)
-            self.assertEqual(','.join(store.merchandises.keys()), '#1,#2,#3')
-
 if __name__ == '__main__':
     unittest.main()