Merge pull request #1903 from bagerard/mongodb_32

Fixed tests to allow support of MongoDB 3.2 + Drop Mongo2.4 from CI

.install_mongodb_on_travis.sh

@@ -18,6 +18,12 @@ elif [ "$MONGODB" = "3.0" ]; then
     sudo apt-get update
     sudo apt-get install mongodb-org-server=3.0.14
     # service should be started automatically
+elif [ "$MONGODB" = "3.2" ]; then
+    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv EA312927
+    echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
+    sudo apt-get update
+    sudo apt-get install mongodb-org-server=3.2.20
+    # service should be started automatically
 else
     echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
     exit 1

.travis.yml

@@ -27,17 +27,17 @@ matrix:
 
   include:
   - python: 2.7
-    env: MONGODB=2.4 PYMONGO=3.5
+    env: MONGODB=3.0 PYMONGO=3.5
   - python: 2.7
-    env: MONGODB=3.0 PYMONGO=3.x
+    env: MONGODB=3.2 PYMONGO=3.x
   - python: 3.5
-    env: MONGODB=2.4 PYMONGO=3.5
+    env: MONGODB=3.0 PYMONGO=3.5
   - python: 3.5
-    env: MONGODB=3.0 PYMONGO=3.x
+    env: MONGODB=3.2 PYMONGO=3.x
   - python: 3.6
-    env: MONGODB=2.4 PYMONGO=3.5
+    env: MONGODB=3.0 PYMONGO=3.5
   - python: 3.6
-    env: MONGODB=3.0 PYMONGO=3.x
+    env: MONGODB=3.2 PYMONGO=3.x
 
 before_install:
 - bash .install_mongodb_on_travis.sh

README.rst

@@ -26,10 +26,10 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Supported MongoDB Versions
 ==========================
-MongoEngine is currently tested against MongoDB v2.4, v2.6, and v3.0. Future
+MongoEngine is currently tested against MongoDB v2.6, v3.0 and v3.2. Future
 versions should be supported as well, but aren't actively tested at the moment.
 Make sure to open an issue or submit a pull request if you experience any
-problems with MongoDB v3.2+.
+problems with MongoDB v3.4+.
 
 Installation
 ============

mongoengine/context_managers.py

@@ -182,8 +182,10 @@ class query_counter(object):
         self._ignored_query = {
             'ns':
                 {'$ne': '%s.system.indexes' % self.db.name},
-            'op':
+            'op':                       # MONGODB < 3.2
-                {'$ne': 'killcursors'}
+                {'$ne': 'killcursors'},
+            'command.killCursors':      # MONGODB >= 3.2
+                {'$exists': False}
         }
 
     def _turn_on_profiling(self):

tests/document/indexes.py

@@ -1,15 +1,14 @@
 # -*- coding: utf-8 -*-
 import unittest
-import sys
+from datetime import datetime
 
 from nose.plugins.skip import SkipTest
-from datetime import datetime
+from pymongo.errors import OperationFailure
 import pymongo
 
 from mongoengine import *
 from mongoengine.connection import get_db
-
+from tests.utils import get_mongodb_version, needs_mongodb_v26, MONGODB_32, MONGODB_3
-from tests.utils import get_mongodb_version, needs_mongodb_v26
 
 __all__ = ("IndexesTest", )
 
@@ -19,6 +18,7 @@ class IndexesTest(unittest.TestCase):
     def setUp(self):
         self.connection = connect(db='mongoenginetest')
         self.db = get_db()
+        self.mongodb_version = get_mongodb_version()
 
         class Person(Document):
             name = StringField()
@@ -491,7 +491,7 @@ class IndexesTest(unittest.TestCase):
         obj = Test(a=1)
         obj.save()
 
-        IS_MONGODB_3 = get_mongodb_version()[0] >= 3
+        IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3
 
         # Need to be explicit about covered indexes as mongoDB doesn't know if
         # the documents returned might have more keys in that here.
@@ -541,19 +541,24 @@ class IndexesTest(unittest.TestCase):
                                  [('categories', 1), ('_id', 1)])
 
     def test_hint(self):
+        MONGO_VER = self.mongodb_version
 
+        TAGS_INDEX_NAME = 'tags_1'
         class BlogPost(Document):
             tags = ListField(StringField())
             meta = {
                 'indexes': [
-                    'tags',
+                    {
+                        'fields': ['tags'],
+                        'name': TAGS_INDEX_NAME
+                    }
                 ],
             }
 
         BlogPost.drop_collection()
 
-        for i in range(0, 10):
+        for i in range(10):
-            tags = [("tag %i" % n) for n in range(0, i % 2)]
+            tags = [("tag %i" % n) for n in range(i % 2)]
             BlogPost(tags=tags).save()
 
         self.assertEqual(BlogPost.objects.count(), 10)
@@ -563,18 +568,18 @@ class IndexesTest(unittest.TestCase):
         if pymongo.version != '3.0':
             self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10)
 
+        if MONGO_VER == MONGODB_32:
+            # Mongo32 throws an error if an index exists (i.e `tags` in our case)
+            # and you use hint on an index name that does not exist
+            with self.assertRaises(OperationFailure):
+                BlogPost.objects.hint([('ZZ', 1)]).count()
+        else:
             self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10)
 
-        if pymongo.version >= '2.8':
+        self.assertEqual(BlogPost.objects.hint(TAGS_INDEX_NAME ).count(), 10)
-            self.assertEqual(BlogPost.objects.hint('tags').count(), 10)
-        else:
-            def invalid_index():
-                BlogPost.objects.hint('tags').next()
-            self.assertRaises(TypeError, invalid_index)
 
-        def invalid_index_2():
+        with self.assertRaises(Exception):
-            return BlogPost.objects.hint(('tags', 1)).next()
+            BlogPost.objects.hint(('tags', 1)).next()
-        self.assertRaises(Exception, invalid_index_2)
 
     def test_unique(self):
         """Ensure that uniqueness constraints are applied to fields.

tests/queryset/queryset.py

@@ -21,8 +21,7 @@ from mongoengine.python_support import IS_PYMONGO_3
 from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned,
                                   QuerySet, QuerySetManager, queryset_manager)
 
-from tests.utils import needs_mongodb_v26, skip_pymongo3
+from tests.utils import needs_mongodb_v26, skip_pymongo3, get_mongodb_version, MONGODB_32
-
 
 __all__ = ("QuerySetTest",)
 
@@ -30,10 +29,8 @@ __all__ = ("QuerySetTest",)
 class db_ops_tracker(query_counter):
 
     def get_ops(self):
-        ignore_query = {
+        ignore_query = dict(self._ignored_query)
-            'ns': {'$ne': '%s.system.indexes' % self.db.name},
+        ignore_query['command.count'] = {'$ne': 'system.profile'}   # Ignore the query issued by query_counter
-            'command.count': {'$ne': 'system.profile'}
-        }
         return list(self.db.system.profile.find(ignore_query))
 
 
@@ -56,6 +53,8 @@ class QuerySetTest(unittest.TestCase):
         self.PersonMeta = PersonMeta
         self.Person = Person
 
+        self.mongodb_version = get_mongodb_version()
+
     def test_initialisation(self):
         """Ensure that a QuerySet is correctly initialised by QuerySetManager.
         """
@@ -813,8 +812,8 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(record.embed.field, 2)
 
     def test_bulk_insert(self):
-        """Ensure that bulk insert works
+        """Ensure that bulk insert works"""
-        """
+        MONGO_VER = self.mongodb_version
 
         class Comment(EmbeddedDocument):
             name = StringField()
@@ -832,35 +831,41 @@ class QuerySetTest(unittest.TestCase):
         # get MongoDB version info
         connection = get_connection()
         info = connection.test.command('buildInfo')
-        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
 
         # Recreates the collection
         self.assertEqual(0, Blog.objects.count())
 
+        comment1 = Comment(name='testa')
+        comment2 = Comment(name='testb')
+        post1 = Post(comments=[comment1, comment2])
+        post2 = Post(comments=[comment2, comment2])
+        blogs = [Blog(title="post %s" % i, posts=[post1, post2])
+                 for i in range(99)]
+
+        # Check bulk insert using load_bulk=False
         with query_counter() as q:
             self.assertEqual(q, 0)
-
-            comment1 = Comment(name='testa')
-            comment2 = Comment(name='testb')
-            post1 = Post(comments=[comment1, comment2])
-            post2 = Post(comments=[comment2, comment2])
-
-            blogs = []
-            for i in range(1, 100):
-                blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
-
             Blog.objects.insert(blogs, load_bulk=False)
-            # profiling logs each doc now in the bulk op
+
-            self.assertEqual(q, 99)
+            if MONGO_VER == MONGODB_32:
+                self.assertEqual(q, 1)      # 1 entry containing the list of inserts
+            else:
+                self.assertEqual(q, len(blogs))    # 1 entry per doc inserted
+
+        self.assertEqual(Blog.objects.count(), len(blogs))
 
         Blog.drop_collection()
         Blog.ensure_indexes()
 
+        # Check bulk insert using load_bulk=True
         with query_counter() as q:
             self.assertEqual(q, 0)
-
             Blog.objects.insert(blogs)
-            self.assertEqual(q, 100) # 99 for insert 1 for fetch
+
+            if MONGO_VER == MONGODB_32:
+                self.assertEqual(q, 2)              # 1 for insert 1 for fetch
+            else:
+                self.assertEqual(q, len(blogs)+1)    # + 1 to fetch all docs
 
         Blog.drop_collection()
 
@@ -1262,6 +1267,9 @@ class QuerySetTest(unittest.TestCase):
         """Ensure that the default ordering can be cleared by calling
         order_by() w/o any arguments.
         """
+        MONGO_VER = self.mongodb_version
+        ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
+
         class BlogPost(Document):
             title = StringField()
             published_date = DateTimeField()
@@ -1277,7 +1285,7 @@ class QuerySetTest(unittest.TestCase):
             BlogPost.objects.filter(title='whatever').first()
             self.assertEqual(len(q.get_ops()), 1)
             self.assertEqual(
-                q.get_ops()[0]['query']['$orderby'],
+                q.get_ops()[0]['query'][ORDER_BY_KEY],
                 {'published_date': -1}
             )
 
@@ -1285,14 +1293,14 @@ class QuerySetTest(unittest.TestCase):
         with db_ops_tracker() as q:
             BlogPost.objects.filter(title='whatever').order_by().first()
             self.assertEqual(len(q.get_ops()), 1)
-            self.assertNotIn('$orderby', q.get_ops()[0]['query'])
+            self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
 
         # calling an explicit order_by should use a specified sort
         with db_ops_tracker() as q:
             BlogPost.objects.filter(title='whatever').order_by('published_date').first()
             self.assertEqual(len(q.get_ops()), 1)
             self.assertEqual(
-                q.get_ops()[0]['query']['$orderby'],
+                q.get_ops()[0]['query'][ORDER_BY_KEY],
                 {'published_date': 1}
             )
 
@@ -1301,11 +1309,14 @@ class QuerySetTest(unittest.TestCase):
             qs = BlogPost.objects.filter(title='whatever').order_by('published_date')
             qs.order_by().first()
             self.assertEqual(len(q.get_ops()), 1)
-            self.assertNotIn('$orderby', q.get_ops()[0]['query'])
+            self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
 
     def test_no_ordering_for_get(self):
         """ Ensure that Doc.objects.get doesn't use any ordering.
         """
+        MONGO_VER = self.mongodb_version
+        ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
+
         class BlogPost(Document):
             title = StringField()
             published_date = DateTimeField()
@@ -1320,13 +1331,13 @@ class QuerySetTest(unittest.TestCase):
         with db_ops_tracker() as q:
             BlogPost.objects.get(title='whatever')
             self.assertEqual(len(q.get_ops()), 1)
-            self.assertNotIn('$orderby', q.get_ops()[0]['query'])
+            self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
 
         # Ordering should be ignored for .get even if we set it explicitly
         with db_ops_tracker() as q:
             BlogPost.objects.order_by('-title').get(title='whatever')
             self.assertEqual(len(q.get_ops()), 1)
-            self.assertNotIn('$orderby', q.get_ops()[0]['query'])
+            self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
 
     def test_find_embedded(self):
         """Ensure that an embedded document is properly returned from
@@ -2450,7 +2461,11 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(names, ['User A', 'User B', 'User C'])
 
     def test_comment(self):
-        """Make sure adding a comment to the query works."""
+        """Make sure adding a comment to the query gets added to the query"""
+        MONGO_VER = self.mongodb_version
+        QUERY_KEY = 'filter' if MONGO_VER == MONGODB_32 else '$query'
+        COMMENT_KEY = 'comment' if MONGO_VER == MONGODB_32 else '$comment'
+
         class User(Document):
             age = IntField()
 
@@ -2466,8 +2481,8 @@ class QuerySetTest(unittest.TestCase):
             ops = q.get_ops()
             self.assertEqual(len(ops), 2)
             for op in ops:
-                self.assertEqual(op['query']['$query'], {'age': {'$gte': 18}})
+                self.assertEqual(op['query'][QUERY_KEY], {'age': {'$gte': 18}})
-                self.assertEqual(op['query']['$comment'], 'looking for an adult')
+                self.assertEqual(op['query'][COMMENT_KEY], 'looking for an adult')
 
     def test_map_reduce(self):
         """Ensure map/reduce is both mapping and reducing.
@@ -4825,27 +4840,18 @@ class QuerySetTest(unittest.TestCase):
         for i in range(100):
             Person(name="No: %s" % i).save()
 
-            with query_counter() as q:
+        with query_counter() as q:
-                try:
+            self.assertEqual(q, 0)
-                    self.assertEqual(q, 0)
+            people = Person.objects.no_cache()
-                    people = Person.objects.no_cache()
 
-                    [x for x in people]
+            [x for x in people]
-                    self.assertEqual(q, 1)
+            self.assertEqual(q, 1)
 
-                    list(people)
+            list(people)
-                    self.assertEqual(q, 2)
+            self.assertEqual(q, 2)
-
-                    people.count()
-                    self.assertEqual(q, 3)
-                except AssertionError as exc:
-                    db = get_db()
-                    msg = ''
-                    for q in list(db.system.profile.find())[-50:]:
-                        msg += str([q['ts'], q['ns'], q.get('query'), q['op']])+'\n'
-                        msg += str(q)
-                    raise AssertionError(str(exc) + '\n'+msg)
 
+            people.count()
+            self.assertEqual(q, 3)
 
     def test_cache_not_cloned(self):
 
@@ -5119,35 +5125,39 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(op['nreturned'], 1)
 
     def test_bool_with_ordering(self):
+        MONGO_VER = self.mongodb_version
+        ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
 
         class Person(Document):
             name = StringField()
 
         Person.drop_collection()
+
         Person(name="Test").save()
 
+        # Check that bool(queryset) does not uses the orderby
         qs = Person.objects.order_by('name')
-
         with query_counter() as q:
 
-            if qs:
+            if bool(qs):
                 pass
 
             op = q.db.system.profile.find({"ns":
                                            {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
-            self.assertNotIn('$orderby', op['query'],
+            self.assertNotIn(ORDER_BY_KEY, op['query'])
-                             'BaseQuerySet cannot use orderby in if stmt')
 
+        # Check that normal query uses orderby
+        qs2 = Person.objects.order_by('name')
         with query_counter() as p:
 
-            for x in qs:
+            for x in qs2:
                 pass
 
             op = p.db.system.profile.find({"ns":
                                            {"$ne": "%s.system.indexes" % q.db.name}})[0]
 
-            self.assertIn('$orderby', op['query'], 'BaseQuerySet cannot remove orderby in for loop')
+            self.assertIn(ORDER_BY_KEY, op['query'])
 
     def test_bool_with_ordering_from_meta_dict(self):
 

tests/test_context_managers.py

@@ -299,7 +299,6 @@ class ContextManagersTest(unittest.TestCase):
 
             cursor.close()              # issues a `killcursors` query that is ignored by the context
             self.assertEqual(q, 1)
-
             _ = db.system.indexes.find_one()    # queries on db.system.indexes are ignored as well
             self.assertEqual(q, 1)
 

tests/utils.py

@@ -10,6 +10,13 @@ from mongoengine.python_support import IS_PYMONGO_3
 MONGO_TEST_DB = 'mongoenginetest'   # standard name for the test database
 
 
+# Constant that can be used to compare the version retrieved with
+# get_mongodb_version()
+MONGODB_26 = (2, 6)
+MONGODB_3 = (3,0)
+MONGODB_32 = (3, 2)
+
+
 class MongoDBTestCase(unittest.TestCase):
     """Base class for tests that need a mongodb connection
     It ensures that the db is clean at the beginning and dropped at the end automatically
@@ -27,24 +34,26 @@ class MongoDBTestCase(unittest.TestCase):
 
 
 def get_mongodb_version():
-    """Return the version tuple of the MongoDB server that the default
+    """Return the version of the connected mongoDB (first 2 digits)
-    connection is connected to.
+
+    :return: tuple(int, int)
     """
-    return tuple(get_connection().server_info()['versionArray'])
+    version_list = get_connection().server_info()['versionArray'][:2]     # e.g: (3, 2)
+    return tuple(version_list)
 
 
-def _decorated_with_ver_requirement(func, ver_tuple):
+def _decorated_with_ver_requirement(func, version):
     """Return a given function decorated with the version requirement
     for a particular MongoDB version tuple.
+
+    :param version: The version required (tuple(int, int))
     """
     def _inner(*args, **kwargs):
-        mongodb_ver = get_mongodb_version()
+        MONGODB_V = get_mongodb_version()
-        if mongodb_ver >= ver_tuple:
+        if MONGODB_V >= version:
             return func(*args, **kwargs)
 
-        raise SkipTest('Needs MongoDB v{}+'.format(
+        raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in version)))
-            '.'.join([str(v) for v in ver_tuple])
-        ))
 
     _inner.__name__ = func.__name__
     _inner.__doc__ = func.__doc__
@@ -56,14 +65,14 @@ def needs_mongodb_v26(func):
     """Raise a SkipTest exception if we're working with MongoDB version
     lower than v2.6.
     """
-    return _decorated_with_ver_requirement(func, (2, 6))
+    return _decorated_with_ver_requirement(func, MONGODB_26)
 
 
 def needs_mongodb_v3(func):
     """Raise a SkipTest exception if we're working with MongoDB version
     lower than v3.0.
     """
-    return _decorated_with_ver_requirement(func, (3, 0))
+    return _decorated_with_ver_requirement(func, MONGODB_3)
 
 
 def skip_pymongo3(f):