v0.14.1 version bump + updated changelog

Add missing dunder method  - __ne__ to the class GridFSProxy class

.install_mongodb_on_travis.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+sudo apt-get remove mongodb-org-server
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 
 if [ "$MONGODB" = "2.4" ]; then
@@ -13,7 +14,7 @@ elif [ "$MONGODB" = "2.6" ]; then
     sudo apt-get install mongodb-org-server=2.6.12
     # service should be started automatically
 elif [ "$MONGODB" = "3.0" ]; then
-    echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
     sudo apt-get update
     sudo apt-get install mongodb-org-server=3.0.14
     # service should be started automatically
@@ -21,3 +22,6 @@ else
     echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
     exit 1
 fi;
+
+mkdir db
+1>db/logs mongod --dbpath=db &

.travis.yml

@@ -15,6 +15,7 @@ language: python
 python:
 - 2.7
 - 3.5
+- 3.6
 - pypy
 
 env:
@@ -39,9 +40,15 @@ matrix:
     env: MONGODB=2.4 PYMONGO=3.0
   - python: 3.5
     env: MONGODB=3.0 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=2.4 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=3.0 PYMONGO=3.0
 
 before_install:
 - bash .install_mongodb_on_travis.sh
+- sleep 15  # https://docs.travis-ci.com/user/database-setup/#MongoDB-does-not-immediately-accept-connections
+- mongo --eval 'db.version();'
 
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
@@ -90,7 +97,7 @@ deploy:
   distributions: "sdist bdist_wheel"
 
   # only deploy on tagged commits (aka GitHub releases) and only for the
-  # parent repo's builds running Python 2.7 along with dev PyMongo (we run
+  # parent repo's builds running Python 2.7 along with PyMongo v3.0 (we run
   # Travis against many different Python and PyMongo versions and we don't
   # want the deploy to occur multiple times).
   on:

AUTHORS

@@ -243,3 +243,5 @@ that much better:
  * Victor Varvaryuk
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
+ * Renjianxin (https://github.com/Davidrjx)
+ * Erdenezul Batmunkh (https://github.com/erdenezul)

docs/changelog.rst

@@ -6,6 +6,20 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 
+Changes in 0.14.1
+=================
+- Removed SemiStrictDict and started using a regular dict for `BaseDocument._data` #1630
+- Added support for the `$position` param in the `$push` operator #1566
+- Fixed `DateTimeField` interpreting an empty string as today #1533
+- Added a missing `__ne__` method to the `GridFSProxy` class #1632
+- Fixed `BaseQuerySet._fields_to_db_fields` #1553
+
+Changes in 0.14.0
+=================
+- BREAKING CHANGE: Removed the `coerce_types` param from `QuerySet.as_pymongo` #1549
+- POTENTIAL BREAKING CHANGE: Made EmbeddedDocument not hashable by default #1528
+- Improved code quality #1531, #1540, #1541, #1547
+
 Changes in 0.13.0
 =================
 - POTENTIAL BREAKING CHANGE: Added Unicode support to the `EmailField`, see

docs/guide/querying.rst

@@ -565,6 +565,15 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`::
     >>> post.tags
     ['database', 'mongodb']
 
+From MongoDB version 2.6, push operator supports $position value which allows
+to push values with index.
+    >>> post = BlogPost(title="Test", tags=["mongo"])
+    >>> post.save()
+    >>> post.update(push__tags__0=["database", "code"])
+    >>> post.reload()
+    >>> post.tags
+    ['database', 'code', 'mongo']
+
 .. note::
     Currently only top level lists are handled, future versions of mongodb /
     pymongo plan to support nested positional operators.  See `The $ positional

docs/upgrade.rst

@@ -6,6 +6,18 @@ Development
 ***********
 (Fill this out whenever you introduce breaking changes to MongoEngine)
 
+0.14.0
+******
+This release includes a few bug fixes and a significant code cleanup. The most
+important change is that `QuerySet.as_pymongo` no longer supports a
+`coerce_types` mode. If you used it in the past, a) please let us know of your
+use case, b) you'll need to override `as_pymongo` to get the desired outcome.
+
+This release also makes the EmbeddedDocument not hashable by default. If you
+use embedded documents in sets or dictionaries, you might have to override
+`__hash__` and implement a hashing logic specific to your use case. See #1528
+for the reason behind this change.
+
 0.13.0
 ******
 This release adds Unicode support to the `EmailField` and changes its

mongoengine/__init__.py

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 13, 0)
+VERSION = (0, 14, 1)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -127,7 +127,7 @@ class BaseList(list):
         return value
 
     def __iter__(self):
-        for i in xrange(self.__len__()):
+        for i in six.moves.range(self.__len__()):
             yield self[i]
 
     def __setitem__(self, key, value, *args, **kwargs):
@@ -445,42 +445,3 @@ class StrictDict(object):
 
             cls._classes[allowed_keys] = SpecificStrictDict
         return cls._classes[allowed_keys]
-
-
-class SemiStrictDict(StrictDict):
-    __slots__ = ('_extras', )
-    _classes = {}
-
-    def __getattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__getattr__(attr)
-        except AttributeError:
-            try:
-                return self.__getattribute__('_extras')[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __setattr__(self, attr, value):
-        try:
-            super(SemiStrictDict, self).__setattr__(attr, value)
-        except AttributeError:
-            try:
-                self._extras[attr] = value
-            except AttributeError:
-                self._extras = {attr: value}
-
-    def __delattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__delattr__(attr)
-        except AttributeError:
-            try:
-                del self._extras[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __iter__(self):
-        try:
-            extras_iter = iter(self.__getattribute__('_extras'))
-        except AttributeError:
-            extras_iter = ()
-        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)

mongoengine/base/document.py

@@ -13,13 +13,13 @@ from mongoengine import signals
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
-                                             SemiStrictDict, StrictDict)
+                                             StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
 from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
                                 LookUpError, OperationError, ValidationError)
 
-__all__ = ('BaseDocument',)
+__all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
 NON_FIELD_ERRORS = '__all__'
 
@@ -79,8 +79,7 @@ class BaseDocument(object):
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
         else:
-            self._data = SemiStrictDict.create(
-                allowed_keys=self._fields_ordered)()
+            self._data = {}
 
         self._dynamic_fields = SON()
 

mongoengine/connection.py

@@ -146,13 +146,14 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         raise MongoEngineConnectionError(msg)
 
     def _clean_settings(settings_dict):
-        irrelevant_fields = set([
-            'name', 'username', 'password', 'authentication_source',
-            'authentication_mechanism'
-        ])
+        # set literal more efficient than calling set function
+        irrelevant_fields_set = {
+            'name', 'username', 'password',
+            'authentication_source', 'authentication_mechanism'
+        }
         return {
             k: v for k, v in settings_dict.items()
-            if k not in irrelevant_fields
+            if k not in irrelevant_fields_set
         }
 
     # Retrieve a copy of the connection settings associated with the requested

mongoengine/document.py

@@ -320,7 +320,7 @@ class Document(BaseDocument):
         :param save_condition: only perform save if matching record in db
             satisfies condition(s) (e.g. version number).
             Raises :class:`OperationError` if the conditions are not satisfied
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
 
         .. versionchanged:: 0.5

mongoengine/fields.py

@@ -483,6 +483,10 @@ class DateTimeField(BaseField):
         if not isinstance(value, six.string_types):
             return None
 
+        value = value.strip()
+        if not value:
+            return None
+
         # Attempt to parse a datetime:
         if dateutil:
             try:
@@ -1461,6 +1465,9 @@ class GridFSProxy(object):
         else:
             return False
 
+    def __ne__(self, other):
+        return not self == other
+
     @property
     def fs(self):
         if not self._fs:

mongoengine/queryset/base.py

@@ -67,7 +67,6 @@ class BaseQuerySet(object):
         self._scalar = []
         self._none = False
         self._as_pymongo = False
-        self._as_pymongo_coerce = False
         self._search_text = None
 
         # If inheritance is allowed, only return instances and instances of
@@ -728,11 +727,12 @@ class BaseQuerySet(object):
                 '%s is not a subclass of BaseQuerySet' % new_qs.__name__)
 
         copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj',
-                      '_where_clause', '_loaded_fields', '_ordering', '_snapshot',
-                      '_timeout', '_class_check', '_slave_okay', '_read_preference',
-                      '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce',
+                      '_where_clause', '_loaded_fields', '_ordering',
+                      '_snapshot', '_timeout', '_class_check', '_slave_okay',
+                      '_read_preference', '_iter', '_scalar', '_as_pymongo',
                       '_limit', '_skip', '_hint', '_auto_dereference',
-                      '_search_text', 'only_fields', '_max_time_ms', '_comment')
+                      '_search_text', 'only_fields', '_max_time_ms',
+                      '_comment')
 
         for prop in copy_props:
             val = getattr(self, prop)
@@ -939,7 +939,8 @@ class BaseQuerySet(object):
 
             posts = BlogPost.objects(...).fields(slice__comments=5)
 
-        :param kwargs: A set keywors arguments identifying what to include.
+        :param kwargs: A set of keyword arguments identifying what to
+            include, exclude, or slice.
 
         .. versionadded:: 0.5
         """
@@ -1128,16 +1129,15 @@ class BaseQuerySet(object):
         """An alias for scalar"""
         return self.scalar(*fields)
 
-    def as_pymongo(self, coerce_types=False):
+    def as_pymongo(self):
         """Instead of returning Document instances, return raw values from
         pymongo.
 
-        :param coerce_types: Field types (if applicable) would be use to
-            coerce types.
+        This method is particularly useful if you don't need dereferencing
+        and care primarily about the speed of data retrieval.
         """
         queryset = self.clone()
         queryset._as_pymongo = True
-        queryset._as_pymongo_coerce = coerce_types
         return queryset
 
     def max_time_ms(self, ms):
@@ -1722,25 +1722,33 @@ class BaseQuerySet(object):
         return frequencies
 
     def _fields_to_dbfields(self, fields):
-        """Translate fields paths to its db equivalents"""
-        ret = []
+        """Translate fields' paths to their db equivalents."""
         subclasses = []
-        document = self._document
-        if document._meta['allow_inheritance']:
+        if self._document._meta['allow_inheritance']:
             subclasses = [get_document(x)
-                          for x in document._subclasses][1:]
+                          for x in self._document._subclasses][1:]
+
+        db_field_paths = []
         for field in fields:
+            field_parts = field.split('.')
             try:
-                field = '.'.join(f.db_field for f in
-                                 document._lookup_field(field.split('.')))
-                ret.append(field)
+                field = '.'.join(
+                    f if isinstance(f, six.string_types) else f.db_field
+                    for f in self._document._lookup_field(field_parts)
+                )
+                db_field_paths.append(field)
             except LookUpError as err:
                 found = False
+
+                # If a field path wasn't found on the main document, go
+                # through its subclasses and see if it exists on any of them.
                 for subdoc in subclasses:
                     try:
-                        subfield = '.'.join(f.db_field for f in
-                                            subdoc._lookup_field(field.split('.')))
-                        ret.append(subfield)
+                        subfield = '.'.join(
+                            f if isinstance(f, six.string_types) else f.db_field
+                            for f in subdoc._lookup_field(field_parts)
+                        )
+                        db_field_paths.append(subfield)
                         found = True
                         break
                     except LookUpError:
@@ -1748,7 +1756,8 @@ class BaseQuerySet(object):
 
                 if not found:
                     raise err
-        return ret
+
+        return db_field_paths
 
     def _get_order_by(self, keys):
         """Given a list of MongoEngine-style sort keys, return a list
@@ -1799,59 +1808,25 @@ class BaseQuerySet(object):
 
         return tuple(data)
 
-    def _get_as_pymongo(self, row):
-        # Extract which fields paths we should follow if .fields(...) was
-        # used. If not, handle all fields.
-        if not getattr(self, '__as_pymongo_fields', None):
-            self.__as_pymongo_fields = []
+    def _get_as_pymongo(self, doc):
+        """Clean up a PyMongo doc, removing fields that were only fetched
+        for the sake of MongoEngine's implementation, and return it.
+        """
+        # Always remove _cls as a MongoEngine's implementation detail.
+        if '_cls' in doc:
+            del doc['_cls']
 
-            for field in self._loaded_fields.fields - set(['_cls']):
-                self.__as_pymongo_fields.append(field)
-                while '.' in field:
-                    field, _ = field.rsplit('.', 1)
-                    self.__as_pymongo_fields.append(field)
+        # If the _id was not included in a .only or was excluded in a .exclude,
+        # remove it from the doc (we always fetch it so that we can properly
+        # construct documents).
+        fields = self._loaded_fields
+        if fields and '_id' in doc and (
+            (fields.value == QueryFieldList.ONLY and '_id' not in fields.fields) or
+            (fields.value == QueryFieldList.EXCLUDE and '_id' in fields.fields)
+        ):
+            del doc['_id']
 
-        all_fields = not self.__as_pymongo_fields
-
-        def clean(data, path=None):
-            path = path or ''
-
-            if isinstance(data, dict):
-                new_data = {}
-                for key, value in data.iteritems():
-                    new_path = '%s.%s' % (path, key) if path else key
-
-                    if all_fields:
-                        include_field = True
-                    elif self._loaded_fields.value == QueryFieldList.ONLY:
-                        include_field = new_path in self.__as_pymongo_fields
-                    else:
-                        include_field = new_path not in self.__as_pymongo_fields
-
-                    if include_field:
-                        new_data[key] = clean(value, path=new_path)
-                data = new_data
-            elif isinstance(data, list):
-                data = [clean(d, path=path) for d in data]
-            else:
-                if self._as_pymongo_coerce:
-                    # If we need to coerce types, we need to determine the
-                    # type of this field and use the corresponding
-                    # .to_python(...)
-                    EmbeddedDocumentField = _import_class('EmbeddedDocumentField')
-
-                    obj = self._document
-                    for chunk in path.split('.'):
-                        obj = getattr(obj, chunk, None)
-                        if obj is None:
-                            break
-                        elif isinstance(obj, EmbeddedDocumentField):
-                            obj = obj.document_type
-                    if obj and data is not None:
-                        data = obj.to_python(data)
-            return data
-
-        return clean(row)
+        return doc
 
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where

mongoengine/queryset/queryset.py

@@ -1,3 +1,5 @@
+import six
+
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
                                        NULLIFY, PULL)
@@ -112,7 +114,7 @@ class QuerySet(BaseQuerySet):
         # Pull in ITER_CHUNK_SIZE docs from the database and store them in
         # the result cache.
         try:
-            for _ in xrange(ITER_CHUNK_SIZE):
+            for _ in six.moves.range(ITER_CHUNK_SIZE):
                 self._result_cache.append(self.next())
         except StopIteration:
             # Getting this exception means there are no more docs in the
@@ -166,7 +168,7 @@ class QuerySetNoCache(BaseQuerySet):
             return '.. queryset mid-iteration ..'
 
         data = []
-        for _ in xrange(REPR_OUTPUT_SIZE + 1):
+        for _ in six.moves.range(REPR_OUTPUT_SIZE + 1):
             try:
                 data.append(self.next())
             except StopIteration:

mongoengine/queryset/transform.py

@@ -284,7 +284,9 @@ def update(_doc_cls=None, **update):
             if isinstance(field, GeoJsonBaseField):
                 value = field.to_mongo(value)
 
-            if op in (None, 'set', 'push', 'pull'):
+            if op == 'push' and isinstance(value, (list, tuple, set)):
+                value = [field.prepare_query_value(op, v) for v in value]
+            elif op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
@@ -333,10 +335,22 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
+        elif op == 'push':
+            if parts[-1].isdigit():
+                key = parts[0]
+                position = int(parts[-1])
+                # $position expects an iterable. If pushing a single value,
+                # wrap it in a list.
+                if not isinstance(value, (set, tuple, list)):
+                    value = [value]
+                value = {key: {'$each': value, '$position': position}}
+            elif isinstance(value, list):
+                value = {key: {'$each': value}}
+            else:
+                value = {key: value}
         else:
             value = {key: value}
         key = '$' + op
-
         if key not in mongo_update:
             mongo_update[key] = value
         elif key in mongo_update and isinstance(mongo_update[key], dict):

tests/document/instance.py

@@ -22,6 +22,8 @@ from mongoengine.queryset import NULLIFY, Q
 from mongoengine.context_managers import switch_db, query_counter
 from mongoengine import signals
 
+from tests.utils import needs_mongodb_v26
+
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
                                '../fields/mongoengine.png')
 
@@ -826,6 +828,22 @@ class InstanceTest(unittest.TestCase):
 
         self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
+    @needs_mongodb_v26
+    def test_modify_with_positional_push(self):
+        class BlogPost(Document):
+            tags = ListField(StringField())
+
+        post = BlogPost.objects.create(tags=['python'])
+        self.assertEqual(post.tags, ['python'])
+        post.modify(push__tags__0=['code', 'mongo'])
+        self.assertEqual(post.tags, ['code', 'mongo', 'python'])
+
+        # Assert same order of the list items is maintained in the db
+        self.assertEqual(
+            BlogPost._get_collection().find_one({'_id': post.pk})['tags'],
+            ['code', 'mongo', 'python']
+        )
+
     def test_save(self):
         """Ensure that a document may be saved in the database."""
 
@@ -3149,6 +3167,22 @@ class InstanceTest(unittest.TestCase):
 
         person.update(set__height=2.0)
 
+    @needs_mongodb_v26
+    def test_push_with_position(self):
+        """Ensure that push with position works properly for an instance."""
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        blog = BlogPost()
+        blog.slug = "ABC"
+        blog.tags = ["python"]
+        blog.save()
+
+        blog.update(push__tags__0=["mongodb", "code"])
+        blog.reload()
+        self.assertEqual(blog.tags, ['mongodb', 'code', 'python'])
+
 
 if __name__ == '__main__':
     unittest.main()

tests/fields/fields.py

@@ -35,6 +35,28 @@ __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
 
 class FieldTest(MongoDBTestCase):
 
+    def test_datetime_from_empty_string(self):
+        """
+        Ensure an exception is raised when trying to
+        cast an empty string to datetime.
+        """
+        class MyDoc(Document):
+            dt = DateTimeField()
+
+        md = MyDoc(dt='')
+        self.assertRaises(ValidationError, md.save)
+
+    def test_datetime_from_whitespace_string(self):
+        """
+        Ensure an exception is raised when trying to
+        cast a whitespace-only string to datetime.
+        """
+        class MyDoc(Document):
+            dt = DateTimeField()
+
+        md = MyDoc(dt='   ')
+        self.assertRaises(ValidationError, md.save)
+
     def test_default_values_nothing_set(self):
         """Ensure that default field values are used when creating
         a document.

tests/queryset/field_list.py

@@ -197,14 +197,18 @@ class OnlyExcludeAllTest(unittest.TestCase):
             title = StringField()
             text = StringField()
 
+        class VariousData(EmbeddedDocument):
+            some = BooleanField()
+
         class BlogPost(Document):
             content = StringField()
             author = EmbeddedDocumentField(User)
             comments = ListField(EmbeddedDocumentField(Comment))
+            various = MapField(field=EmbeddedDocumentField(VariousData))
 
         BlogPost.drop_collection()
 
-        post = BlogPost(content='Had a good coffee today...')
+        post = BlogPost(content='Had a good coffee today...', various={'test_dynamic':{'some': True}})
         post.author = User(name='Test User')
         post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')]
         post.save()
@@ -215,6 +219,9 @@ class OnlyExcludeAllTest(unittest.TestCase):
         self.assertEqual(obj.author.name, 'Test User')
         self.assertEqual(obj.comments, [])
 
+        obj = BlogPost.objects.only('various.test_dynamic.some').get()
+        self.assertEqual(obj.various["test_dynamic"].some, True)
+
         obj = BlogPost.objects.only('content', 'comments.title',).get()
         self.assertEqual(obj.content, 'Had a good coffee today...')
         self.assertEqual(obj.author, None)

tests/queryset/geo.py

@@ -510,6 +510,24 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
+    def test_aspymongo_with_only(self):
+        """Ensure as_pymongo works with only"""
+        class Place(Document):
+            location = PointField()
+
+        Place.drop_collection()
+        p = Place(location=[24.946861267089844, 60.16311983618494])
+        p.save()
+        qs = Place.objects().only('location')
+        self.assertDictEqual(
+            qs.as_pymongo()[0]['location'],
+            {u'type': u'Point',
+             u'coordinates': [
+                24.946861267089844,
+                60.16311983618494]
+            }
+        )
+
     def test_2dsphere_point_sets_correctly(self):
         class Location(Document):
             loc = PointField()

tests/queryset/modify.py

@@ -1,6 +1,8 @@
 import unittest
 
-from mongoengine import connect, Document, IntField
+from mongoengine import connect, Document, IntField, StringField, ListField
+
+from tests.utils import needs_mongodb_v26
 
 __all__ = ("FindAndModifyTest",)
 
@@ -94,6 +96,37 @@ class FindAndModifyTest(unittest.TestCase):
         self.assertEqual(old_doc.to_mongo(), {"_id": 1})
         self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
+    @needs_mongodb_v26
+    def test_modify_with_push(self):
+        class BlogPost(Document):
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        blog = BlogPost.objects.create()
+
+        # Push a new tag via modify with new=False (default).
+        BlogPost(id=blog.id).modify(push__tags='code')
+        self.assertEqual(blog.tags, [])
+        blog.reload()
+        self.assertEqual(blog.tags, ['code'])
+
+        # Push a new tag via modify with new=True.
+        blog = BlogPost.objects(id=blog.id).modify(push__tags='java', new=True)
+        self.assertEqual(blog.tags, ['code', 'java'])
+
+        # Push a new tag with a positional argument.
+        blog = BlogPost.objects(id=blog.id).modify(
+            push__tags__0='python',
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'code', 'java'])
+
+        # Push multiple new tags with a positional argument.
+        blog = BlogPost.objects(id=blog.id).modify(
+            push__tags__1=['go', 'rust'],
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'go', 'rust', 'code', 'java'])
+
 
 if __name__ == '__main__':
     unittest.main()

tests/queryset/queryset.py

@@ -1903,6 +1903,32 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    @needs_mongodb_v26
+    def test_update_push_with_position(self):
+        """Ensure that the 'push' update with position works properly.
+        """
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        post = BlogPost.objects.create(slug="test")
+
+        BlogPost.objects.filter(id=post.id).update(push__tags="code")
+        BlogPost.objects.filter(id=post.id).update(push__tags__0=["mongodb", "python"])
+        post.reload()
+        self.assertEqual(post.tags, ['mongodb', 'python', 'code'])
+
+        BlogPost.objects.filter(id=post.id).update(set__tags__2="java")
+        post.reload()
+        self.assertEqual(post.tags, ['mongodb', 'python', 'java'])
+
+        #test push with singular value
+        BlogPost.objects.filter(id=post.id).update(push__tags__0='scala')
+        post.reload()
+        self.assertEqual(post.tags, ['scala', 'mongodb', 'python', 'java'])
+
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.
         """
@@ -4047,6 +4073,35 @@ class QuerySetTest(unittest.TestCase):
         plist = list(Person.objects.scalar('name', 'state'))
         self.assertEqual(plist, [(u'Wilson JR', s1)])
 
+    def test_generic_reference_field_with_only_and_as_pymongo(self):
+        class TestPerson(Document):
+            name = StringField()
+
+        class TestActivity(Document):
+            name = StringField()
+            owner = GenericReferenceField()
+
+        TestPerson.drop_collection()
+        TestActivity.drop_collection()
+
+        person = TestPerson(name='owner')
+        person.save()
+
+        a1 = TestActivity(name='a1', owner=person)
+        a1.save()
+
+        activity = TestActivity.objects(owner=person).scalar('id', 'owner').no_dereference().first()
+        self.assertEqual(activity[0], a1.pk)
+        self.assertEqual(activity[1]['_ref'], DBRef('test_person', person.pk))
+
+        activity = TestActivity.objects(owner=person).only('id', 'owner')[0]
+        self.assertEqual(activity.pk, a1.pk)
+        self.assertEqual(activity.owner, person)
+
+        activity = TestActivity.objects(owner=person).only('id', 'owner').as_pymongo().first()
+        self.assertEqual(activity['_id'], a1.pk)
+        self.assertTrue(activity['owner']['_ref'], DBRef('test_person', person.pk))
+
     def test_scalar_db_field(self):
 
         class TestDoc(Document):
@@ -4392,21 +4447,44 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(doc_objects, Doc.objects.from_json(json_data))
 
     def test_as_pymongo(self):
-
         from decimal import Decimal
 
+        class LastLogin(EmbeddedDocument):
+            location = StringField()
+            ip = StringField()
+
         class User(Document):
             id = ObjectIdField('_id')
             name = StringField()
             age = IntField()
             price = DecimalField()
+            last_login = EmbeddedDocumentField(LastLogin)
 
         User.drop_collection()
-        User(name="Bob Dole", age=89, price=Decimal('1.11')).save()
-        User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
+
+        User.objects.create(name="Bob Dole", age=89, price=Decimal('1.11'))
+        User.objects.create(
+            name="Barack Obama",
+            age=51,
+            price=Decimal('2.22'),
+            last_login=LastLogin(
+                location='White House',
+                ip='104.107.108.116'
+            )
+        )
+
+        results = User.objects.as_pymongo()
+        self.assertEqual(
+            set(results[0].keys()),
+            set(['_id', 'name', 'age', 'price'])
+        )
+        self.assertEqual(
+            set(results[1].keys()),
+            set(['_id', 'name', 'age', 'price', 'last_login'])
+        )
 
         results = User.objects.only('id', 'name').as_pymongo()
-        self.assertEqual(sorted(results[0].keys()), sorted(['_id', 'name']))
+        self.assertEqual(set(results[0].keys()), set(['_id', 'name']))
 
         users = User.objects.only('name', 'price').as_pymongo()
         results = list(users)
@@ -4417,16 +4495,20 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(results[1]['name'], 'Barack Obama')
         self.assertEqual(results[1]['price'], 2.22)
 
-        # Test coerce_types
-        users = User.objects.only(
-            'name', 'price').as_pymongo(coerce_types=True)
+        users = User.objects.only('name', 'last_login').as_pymongo()
         results = list(users)
         self.assertTrue(isinstance(results[0], dict))
         self.assertTrue(isinstance(results[1], dict))
-        self.assertEqual(results[0]['name'], 'Bob Dole')
-        self.assertEqual(results[0]['price'], Decimal('1.11'))
-        self.assertEqual(results[1]['name'], 'Barack Obama')
-        self.assertEqual(results[1]['price'], Decimal('2.22'))
+        self.assertEqual(results[0], {
+            'name': 'Bob Dole'
+        })
+        self.assertEqual(results[1], {
+            'name': 'Barack Obama',
+            'last_login': {
+                'location': 'White House',
+                'ip': '104.107.108.116'
+            }
+        })
 
     def test_as_pymongo_json_limit_fields(self):
 
@@ -4590,7 +4672,6 @@ class QuerySetTest(unittest.TestCase):
 
     def test_no_cache(self):
         """Ensure you can add meta data to file"""
-
         class Noddy(Document):
             fields = DictField()
 
@@ -4608,15 +4689,19 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(len(list(docs)), 100)
 
+        # Can't directly get a length of a no-cache queryset.
         with self.assertRaises(TypeError):
             len(docs)
 
+        # Another iteration over the queryset should result in another db op.
         with query_counter() as q:
-            self.assertEqual(q, 0)
             list(docs)
             self.assertEqual(q, 1)
+
+        # ... and another one to double-check.
+        with query_counter() as q:
             list(docs)
-            self.assertEqual(q, 2)
+            self.assertEqual(q, 1)
 
     def test_nested_queryset_iterator(self):
         # Try iterating the same queryset twice, nested.

tests/test_datastructures.py

@@ -1,6 +1,6 @@
 import unittest
 
-from mongoengine.base.datastructures import StrictDict, SemiStrictDict
+from mongoengine.base.datastructures import StrictDict
 
 
 class TestStrictDict(unittest.TestCase):
@@ -76,44 +76,5 @@ class TestStrictDict(unittest.TestCase):
         assert dict(**d) == {'a': 1, 'b': 2}
 
 
-class TestSemiSrictDict(TestStrictDict):
-    def strict_dict_class(self, *args, **kwargs):
-        return SemiStrictDict.create(*args, **kwargs)
-
-    def test_init_fails_on_nonexisting_attrs(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_raises_on_nonexisting_attr(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_getattr_nonexisting_attr_succeeds(self):
-        d = self.dtype()
-        d.x = 1
-        self.assertEqual(d.x, 1)
-
-    def test_init_succeeds_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2))
-
-    def test_iter_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d), ['a', 'b', 'c', 'x'])
-
-    def test_iteritems_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)])
-
-    def tets_cmp_with_strict_dicts(self):
-        d = self.dtype(a=1, b=1, c=1)
-        dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1)
-        self.assertEqual(d, dd)
-
-    def test_cmp_with_strict_dict_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2)
-        self.assertEqual(d, dd)
-
 if __name__ == '__main__':
     unittest.main()