v0.14.3 version bump

Add missing dunder method  - __ne__ to the class GridFSProxy class

.install_mongodb_on_travis.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+sudo apt-get remove mongodb-org-server
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 
 if [ "$MONGODB" = "2.4" ]; then
@@ -13,7 +14,7 @@ elif [ "$MONGODB" = "2.6" ]; then
     sudo apt-get install mongodb-org-server=2.6.12
     # service should be started automatically
 elif [ "$MONGODB" = "3.0" ]; then
-    echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
     sudo apt-get update
     sudo apt-get install mongodb-org-server=3.0.14
     # service should be started automatically
@@ -21,3 +22,6 @@ else
     echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
     exit 1
 fi;
+
+mkdir db
+1>db/logs mongod --dbpath=db &

.travis.yml

@@ -15,6 +15,7 @@ language: python
 python:
 - 2.7
 - 3.5
+- 3.6
 - pypy
 
 env:
@@ -39,9 +40,15 @@ matrix:
     env: MONGODB=2.4 PYMONGO=3.0
   - python: 3.5
     env: MONGODB=3.0 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=2.4 PYMONGO=3.0
+  - python: 3.6
+    env: MONGODB=3.0 PYMONGO=3.0
 
 before_install:
 - bash .install_mongodb_on_travis.sh
+- sleep 15  # https://docs.travis-ci.com/user/database-setup/#MongoDB-does-not-immediately-accept-connections
+- mongo --eval 'db.version();'
 
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
@@ -90,7 +97,7 @@ deploy:
   distributions: "sdist bdist_wheel"
 
   # only deploy on tagged commits (aka GitHub releases) and only for the
-  # parent repo's builds running Python 2.7 along with dev PyMongo (we run
+  # parent repo's builds running Python 2.7 along with PyMongo v3.0 (we run
   # Travis against many different Python and PyMongo versions and we don't
   # want the deploy to occur multiple times).
   on:

AUTHORS

@@ -243,3 +243,5 @@ that much better:
  * Victor Varvaryuk
  * Stanislav Kaledin (https://github.com/sallyruthstruik)
  * Dmitry Yantsen (https://github.com/mrTable)
+ * Renjianxin (https://github.com/Davidrjx)
+ * Erdenezul Batmunkh (https://github.com/erdenezul)

docs/changelog.rst

@@ -6,6 +6,14 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 
+Changes in 0.14.1
+=================
+- Removed SemiStrictDict and started using a regular dict for `BaseDocument._data` #1630
+- Added support for the `$position` param in the `$push` operator #1566
+- Fixed `DateTimeField` interpreting an empty string as today #1533
+- Added a missing `__ne__` method to the `GridFSProxy` class #1632
+- Fixed `BaseQuerySet._fields_to_db_fields` #1553
+
 Changes in 0.14.0
 =================
 - BREAKING CHANGE: Removed the `coerce_types` param from `QuerySet.as_pymongo` #1549

docs/guide/querying.rst

@@ -565,6 +565,15 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`::
     >>> post.tags
     ['database', 'mongodb']
 
+From MongoDB version 2.6, push operator supports $position value which allows
+to push values with index.
+    >>> post = BlogPost(title="Test", tags=["mongo"])
+    >>> post.save()
+    >>> post.update(push__tags__0=["database", "code"])
+    >>> post.reload()
+    >>> post.tags
+    ['database', 'code', 'mongo']
+
 .. note::
     Currently only top level lists are handled, future versions of mongodb /
     pymongo plan to support nested positional operators.  See `The $ positional

mongoengine/__init__.py

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
            list(signals.__all__) + list(errors.__all__))
 
 
-VERSION = (0, 14, 0)
+VERSION = (0, 14, 3)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -127,7 +127,7 @@ class BaseList(list):
         return value
 
     def __iter__(self):
-        for i in xrange(self.__len__()):
+        for i in six.moves.range(self.__len__()):
             yield self[i]
 
     def __setitem__(self, key, value, *args, **kwargs):
@@ -445,42 +445,3 @@ class StrictDict(object):
 
             cls._classes[allowed_keys] = SpecificStrictDict
         return cls._classes[allowed_keys]
-
-
-class SemiStrictDict(StrictDict):
-    __slots__ = ('_extras', )
-    _classes = {}
-
-    def __getattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__getattr__(attr)
-        except AttributeError:
-            try:
-                return self.__getattribute__('_extras')[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __setattr__(self, attr, value):
-        try:
-            super(SemiStrictDict, self).__setattr__(attr, value)
-        except AttributeError:
-            try:
-                self._extras[attr] = value
-            except AttributeError:
-                self._extras = {attr: value}
-
-    def __delattr__(self, attr):
-        try:
-            super(SemiStrictDict, self).__delattr__(attr)
-        except AttributeError:
-            try:
-                del self._extras[attr]
-            except KeyError as e:
-                raise AttributeError(e)
-
-    def __iter__(self):
-        try:
-            extras_iter = iter(self.__getattribute__('_extras'))
-        except AttributeError:
-            extras_iter = ()
-        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)

mongoengine/base/document.py

@@ -13,13 +13,13 @@ from mongoengine import signals
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
-                                             SemiStrictDict, StrictDict)
+                                             StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
 from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
                                 LookUpError, OperationError, ValidationError)
 
-__all__ = ('BaseDocument',)
+__all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
 NON_FIELD_ERRORS = '__all__'
 
@@ -79,8 +79,7 @@ class BaseDocument(object):
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
         else:
-            self._data = SemiStrictDict.create(
-                allowed_keys=self._fields_ordered)()
+            self._data = {}
 
         self._dynamic_fields = SON()
 

mongoengine/connection.py

@@ -146,13 +146,14 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         raise MongoEngineConnectionError(msg)
 
     def _clean_settings(settings_dict):
-        irrelevant_fields = set([
-            'name', 'username', 'password', 'authentication_source',
-            'authentication_mechanism'
-        ])
+        # set literal more efficient than calling set function
+        irrelevant_fields_set = {
+            'name', 'username', 'password',
+            'authentication_source', 'authentication_mechanism'
+        }
         return {
             k: v for k, v in settings_dict.items()
-            if k not in irrelevant_fields
+            if k not in irrelevant_fields_set
         }
 
     # Retrieve a copy of the connection settings associated with the requested

mongoengine/document.py

@@ -320,7 +320,7 @@ class Document(BaseDocument):
         :param save_condition: only perform save if matching record in db
             satisfies condition(s) (e.g. version number).
             Raises :class:`OperationError` if the conditions are not satisfied
-        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+        :param signal_kwargs: (optional) kwargs dictionary to be passed to
             the signal calls.
 
         .. versionchanged:: 0.5

mongoengine/fields.py

@@ -483,6 +483,10 @@ class DateTimeField(BaseField):
         if not isinstance(value, six.string_types):
             return None
 
+        value = value.strip()
+        if not value:
+            return None
+
         # Attempt to parse a datetime:
         if dateutil:
             try:
@@ -1461,6 +1465,9 @@ class GridFSProxy(object):
         else:
             return False
 
+    def __ne__(self, other):
+        return not self == other
+
     @property
     def fs(self):
         if not self._fs:

mongoengine/queryset/base.py

@@ -1722,25 +1722,33 @@ class BaseQuerySet(object):
         return frequencies
 
     def _fields_to_dbfields(self, fields):
-        """Translate fields paths to its db equivalents"""
-        ret = []
+        """Translate fields' paths to their db equivalents."""
         subclasses = []
-        document = self._document
-        if document._meta['allow_inheritance']:
+        if self._document._meta['allow_inheritance']:
             subclasses = [get_document(x)
-                          for x in document._subclasses][1:]
+                          for x in self._document._subclasses][1:]
+
+        db_field_paths = []
         for field in fields:
+            field_parts = field.split('.')
             try:
-                field = '.'.join(f.db_field for f in
-                                 document._lookup_field(field.split('.')))
-                ret.append(field)
+                field = '.'.join(
+                    f if isinstance(f, six.string_types) else f.db_field
+                    for f in self._document._lookup_field(field_parts)
+                )
+                db_field_paths.append(field)
             except LookUpError as err:
                 found = False
+
+                # If a field path wasn't found on the main document, go
+                # through its subclasses and see if it exists on any of them.
                 for subdoc in subclasses:
                     try:
-                        subfield = '.'.join(f.db_field for f in
-                                            subdoc._lookup_field(field.split('.')))
-                        ret.append(subfield)
+                        subfield = '.'.join(
+                            f if isinstance(f, six.string_types) else f.db_field
+                            for f in subdoc._lookup_field(field_parts)
+                        )
+                        db_field_paths.append(subfield)
                         found = True
                         break
                     except LookUpError:
@@ -1748,7 +1756,8 @@ class BaseQuerySet(object):
 
                 if not found:
                     raise err
-        return ret
+
+        return db_field_paths
 
     def _get_order_by(self, keys):
         """Given a list of MongoEngine-style sort keys, return a list

mongoengine/queryset/queryset.py

@@ -1,3 +1,5 @@
+import six
+
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
                                        NULLIFY, PULL)
@@ -112,7 +114,7 @@ class QuerySet(BaseQuerySet):
         # Pull in ITER_CHUNK_SIZE docs from the database and store them in
         # the result cache.
         try:
-            for _ in xrange(ITER_CHUNK_SIZE):
+            for _ in six.moves.range(ITER_CHUNK_SIZE):
                 self._result_cache.append(self.next())
         except StopIteration:
             # Getting this exception means there are no more docs in the
@@ -166,7 +168,7 @@ class QuerySetNoCache(BaseQuerySet):
             return '.. queryset mid-iteration ..'
 
         data = []
-        for _ in xrange(REPR_OUTPUT_SIZE + 1):
+        for _ in six.moves.range(REPR_OUTPUT_SIZE + 1):
             try:
                 data.append(self.next())
             except StopIteration:

mongoengine/queryset/transform.py

@@ -284,7 +284,9 @@ def update(_doc_cls=None, **update):
             if isinstance(field, GeoJsonBaseField):
                 value = field.to_mongo(value)
 
-            if op in (None, 'set', 'push', 'pull'):
+            if op == 'push' and isinstance(value, (list, tuple, set)):
+                value = [field.prepare_query_value(op, v) for v in value]
+            elif op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
@@ -333,10 +335,22 @@ def update(_doc_cls=None, **update):
                 value = {key: value}
         elif op == 'addToSet' and isinstance(value, list):
             value = {key: {'$each': value}}
+        elif op == 'push':
+            if parts[-1].isdigit():
+                key = parts[0]
+                position = int(parts[-1])
+                # $position expects an iterable. If pushing a single value,
+                # wrap it in a list.
+                if not isinstance(value, (set, tuple, list)):
+                    value = [value]
+                value = {key: {'$each': value, '$position': position}}
+            elif isinstance(value, list):
+                value = {key: {'$each': value}}
+            else:
+                value = {key: value}
         else:
             value = {key: value}
         key = '$' + op
-
         if key not in mongo_update:
             mongo_update[key] = value
         elif key in mongo_update and isinstance(mongo_update[key], dict):

setup.py

@@ -70,9 +70,9 @@ setup(
     name='mongoengine',
     version=VERSION,
     author='Harry Marr',
-    author_email='harry.marr@{nospam}gmail.com',
-    maintainer="Ross Lawley",
-    maintainer_email="ross.lawley@{nospam}gmail.com",
+    author_email='harry.marr@gmail.com',
+    maintainer="Stefan Wojcik",
+    maintainer_email="wojcikstefan@gmail.com",
     url='http://mongoengine.org/',
     download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
     license='MIT',

tests/document/instance.py

@@ -22,6 +22,8 @@ from mongoengine.queryset import NULLIFY, Q
 from mongoengine.context_managers import switch_db, query_counter
 from mongoengine import signals
 
+from tests.utils import needs_mongodb_v26
+
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
                                '../fields/mongoengine.png')
 
@@ -826,6 +828,22 @@ class InstanceTest(unittest.TestCase):
 
         self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
 
+    @needs_mongodb_v26
+    def test_modify_with_positional_push(self):
+        class BlogPost(Document):
+            tags = ListField(StringField())
+
+        post = BlogPost.objects.create(tags=['python'])
+        self.assertEqual(post.tags, ['python'])
+        post.modify(push__tags__0=['code', 'mongo'])
+        self.assertEqual(post.tags, ['code', 'mongo', 'python'])
+
+        # Assert same order of the list items is maintained in the db
+        self.assertEqual(
+            BlogPost._get_collection().find_one({'_id': post.pk})['tags'],
+            ['code', 'mongo', 'python']
+        )
+
     def test_save(self):
         """Ensure that a document may be saved in the database."""
 
@@ -3149,6 +3167,22 @@ class InstanceTest(unittest.TestCase):
 
         person.update(set__height=2.0)
 
+    @needs_mongodb_v26
+    def test_push_with_position(self):
+        """Ensure that push with position works properly for an instance."""
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        blog = BlogPost()
+        blog.slug = "ABC"
+        blog.tags = ["python"]
+        blog.save()
+
+        blog.update(push__tags__0=["mongodb", "code"])
+        blog.reload()
+        self.assertEqual(blog.tags, ['mongodb', 'code', 'python'])
+
 
 if __name__ == '__main__':
     unittest.main()

tests/fields/fields.py

@@ -35,6 +35,28 @@ __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
 
 class FieldTest(MongoDBTestCase):
 
+    def test_datetime_from_empty_string(self):
+        """
+        Ensure an exception is raised when trying to
+        cast an empty string to datetime.
+        """
+        class MyDoc(Document):
+            dt = DateTimeField()
+
+        md = MyDoc(dt='')
+        self.assertRaises(ValidationError, md.save)
+
+    def test_datetime_from_whitespace_string(self):
+        """
+        Ensure an exception is raised when trying to
+        cast a whitespace-only string to datetime.
+        """
+        class MyDoc(Document):
+            dt = DateTimeField()
+
+        md = MyDoc(dt='   ')
+        self.assertRaises(ValidationError, md.save)
+
     def test_default_values_nothing_set(self):
         """Ensure that default field values are used when creating
         a document.

tests/queryset/field_list.py

@@ -197,14 +197,18 @@ class OnlyExcludeAllTest(unittest.TestCase):
             title = StringField()
             text = StringField()
 
+        class VariousData(EmbeddedDocument):
+            some = BooleanField()
+
         class BlogPost(Document):
             content = StringField()
             author = EmbeddedDocumentField(User)
             comments = ListField(EmbeddedDocumentField(Comment))
+            various = MapField(field=EmbeddedDocumentField(VariousData))
 
         BlogPost.drop_collection()
 
-        post = BlogPost(content='Had a good coffee today...')
+        post = BlogPost(content='Had a good coffee today...', various={'test_dynamic':{'some': True}})
         post.author = User(name='Test User')
         post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')]
         post.save()
@@ -215,6 +219,9 @@ class OnlyExcludeAllTest(unittest.TestCase):
         self.assertEqual(obj.author.name, 'Test User')
         self.assertEqual(obj.comments, [])
 
+        obj = BlogPost.objects.only('various.test_dynamic.some').get()
+        self.assertEqual(obj.various["test_dynamic"].some, True)
+
         obj = BlogPost.objects.only('content', 'comments.title',).get()
         self.assertEqual(obj.content, 'Had a good coffee today...')
         self.assertEqual(obj.author, None)

tests/queryset/geo.py

@@ -510,6 +510,24 @@ class GeoQueriesTest(MongoDBTestCase):
         roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count()
         self.assertEqual(1, roads)
 
+    def test_aspymongo_with_only(self):
+        """Ensure as_pymongo works with only"""
+        class Place(Document):
+            location = PointField()
+
+        Place.drop_collection()
+        p = Place(location=[24.946861267089844, 60.16311983618494])
+        p.save()
+        qs = Place.objects().only('location')
+        self.assertDictEqual(
+            qs.as_pymongo()[0]['location'],
+            {u'type': u'Point',
+             u'coordinates': [
+                24.946861267089844,
+                60.16311983618494]
+            }
+        )
+
     def test_2dsphere_point_sets_correctly(self):
         class Location(Document):
             loc = PointField()

tests/queryset/modify.py

@@ -1,6 +1,8 @@
 import unittest
 
-from mongoengine import connect, Document, IntField
+from mongoengine import connect, Document, IntField, StringField, ListField
+
+from tests.utils import needs_mongodb_v26
 
 __all__ = ("FindAndModifyTest",)
 
@@ -94,6 +96,37 @@ class FindAndModifyTest(unittest.TestCase):
         self.assertEqual(old_doc.to_mongo(), {"_id": 1})
         self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
 
+    @needs_mongodb_v26
+    def test_modify_with_push(self):
+        class BlogPost(Document):
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        blog = BlogPost.objects.create()
+
+        # Push a new tag via modify with new=False (default).
+        BlogPost(id=blog.id).modify(push__tags='code')
+        self.assertEqual(blog.tags, [])
+        blog.reload()
+        self.assertEqual(blog.tags, ['code'])
+
+        # Push a new tag via modify with new=True.
+        blog = BlogPost.objects(id=blog.id).modify(push__tags='java', new=True)
+        self.assertEqual(blog.tags, ['code', 'java'])
+
+        # Push a new tag with a positional argument.
+        blog = BlogPost.objects(id=blog.id).modify(
+            push__tags__0='python',
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'code', 'java'])
+
+        # Push multiple new tags with a positional argument.
+        blog = BlogPost.objects(id=blog.id).modify(
+            push__tags__1=['go', 'rust'],
+            new=True)
+        self.assertEqual(blog.tags, ['python', 'go', 'rust', 'code', 'java'])
+
 
 if __name__ == '__main__':
     unittest.main()

tests/queryset/queryset.py

@@ -1903,6 +1903,32 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    @needs_mongodb_v26
+    def test_update_push_with_position(self):
+        """Ensure that the 'push' update with position works properly.
+        """
+        class BlogPost(Document):
+            slug = StringField()
+            tags = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        post = BlogPost.objects.create(slug="test")
+
+        BlogPost.objects.filter(id=post.id).update(push__tags="code")
+        BlogPost.objects.filter(id=post.id).update(push__tags__0=["mongodb", "python"])
+        post.reload()
+        self.assertEqual(post.tags, ['mongodb', 'python', 'code'])
+
+        BlogPost.objects.filter(id=post.id).update(set__tags__2="java")
+        post.reload()
+        self.assertEqual(post.tags, ['mongodb', 'python', 'java'])
+
+        #test push with singular value
+        BlogPost.objects.filter(id=post.id).update(push__tags__0='scala')
+        post.reload()
+        self.assertEqual(post.tags, ['scala', 'mongodb', 'python', 'java'])
+
     def test_update_push_and_pull_add_to_set(self):
         """Ensure that the 'pull' update operation works correctly.
         """

tests/test_datastructures.py

@@ -1,6 +1,6 @@
 import unittest
 
-from mongoengine.base.datastructures import StrictDict, SemiStrictDict
+from mongoengine.base.datastructures import StrictDict
 
 
 class TestStrictDict(unittest.TestCase):
@@ -76,44 +76,5 @@ class TestStrictDict(unittest.TestCase):
         assert dict(**d) == {'a': 1, 'b': 2}
 
 
-class TestSemiSrictDict(TestStrictDict):
-    def strict_dict_class(self, *args, **kwargs):
-        return SemiStrictDict.create(*args, **kwargs)
-
-    def test_init_fails_on_nonexisting_attrs(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_raises_on_nonexisting_attr(self):
-        # disable irrelevant test
-        pass
-
-    def test_setattr_getattr_nonexisting_attr_succeeds(self):
-        d = self.dtype()
-        d.x = 1
-        self.assertEqual(d.x, 1)
-
-    def test_init_succeeds_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2))
-
-    def test_iter_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d), ['a', 'b', 'c', 'x'])
-
-    def test_iteritems_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)])
-
-    def tets_cmp_with_strict_dicts(self):
-        d = self.dtype(a=1, b=1, c=1)
-        dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1)
-        self.assertEqual(d, dd)
-
-    def test_cmp_with_strict_dict_with_nonexisting_attrs(self):
-        d = self.dtype(a=1, b=1, c=1, x=2)
-        dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2)
-        self.assertEqual(d, dd)
-
 if __name__ == '__main__':
     unittest.main()