Merge pull request #2009 from bagerard/mongodb_34

Add support for Mongodb 34 (tests/travis)

.install_mongodb_on_travis.sh

@@ -19,8 +19,14 @@ elif [ "$MONGODB" = "3.2" ]; then
     sudo apt-get update
     sudo apt-get install mongodb-org-server=3.2.20
     # service should be started automatically
+elif [ "$MONGODB" = "3.4" ]; then
+    sudo apt-key adv --keyserver keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
+    echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list
+    sudo apt-get update
+    sudo apt-get install mongodb-org-server=3.4.17
+    # service should be started automatically
 else
-    echo "Invalid MongoDB version, expected 2.6, 3.0, or 3.2"
+    echo "Invalid MongoDB version, expected 2.6, 3.0, 3.2 or 3.4."
     exit 1
 fi;
 

.travis.yml

@@ -6,6 +6,7 @@
 #   v3.5, v3.6, PyPy, and PyMongo v3.x.
 # * MongoDB v3.0 & v3.2 are tested against Python v2.7, v3.5 & v3.6
 #   and Pymongo v3.5 & v3.x
+# * MongoDB v3.4 is tested against v3.6 and Pymongo v3.x
 # Reminder: Update README.rst if you change MongoDB versions we test.
 
 language: python
@@ -26,16 +27,14 @@ matrix:
   include:
   - python: 2.7
     env: MONGODB=3.0 PYMONGO=3.5
-  - python: 2.7
-    env: MONGODB=3.2 PYMONGO=3.x
-  - python: 3.5
-    env: MONGODB=3.0 PYMONGO=3.5
   - python: 3.5
     env: MONGODB=3.2 PYMONGO=3.x
   - python: 3.6
     env: MONGODB=3.0 PYMONGO=3.5
   - python: 3.6
     env: MONGODB=3.2 PYMONGO=3.x
+  - python: 3.6
+    env: MONGODB=3.4 PYMONGO=3.x
 
 before_install:
 - bash .install_mongodb_on_travis.sh

README.rst

@@ -26,7 +26,7 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Supported MongoDB Versions
 ==========================
-MongoEngine is currently tested against MongoDB v2.6, v3.0 and v3.2. Future
+MongoEngine is currently tested against MongoDB v2.6, v3.0, v3.2 and v3.4. Future
 versions should be supported as well, but aren't actively tested at the moment.
 Make sure to open an issue or submit a pull request if you experience any
 problems with MongoDB v3.4+.

docs/changelog.rst

@@ -10,6 +10,7 @@ Development
 - Document a BREAKING CHANGE introduced in 0.15.3 and not reported at that time (#1995)
 - Fix InvalidStringData error when using modify on a BinaryField #1127
 - DEPRECATION: `EmbeddedDocument.save` & `.reload` are marked as deprecated and will be removed in a next version of mongoengine #1552
+- Fix test suite and CI to support MongoDB 3.4 #1445
 
 =================
 Changes in 0.16.3

mongoengine/mongodb_support.py

@@ -0,0 +1,21 @@
+"""
+Helper functions, constants, and types to aid with MongoDB version support
+"""
+from mongoengine.connection import get_connection
+
+
+# Constant that can be used to compare the version retrieved with
+# get_mongodb_version()
+MONGODB_34 = (3, 4)
+MONGODB_32 = (3, 2)
+MONGODB_3 = (3, 0)
+MONGODB_26 = (2, 6)
+
+
+def get_mongodb_version():
+    """Return the version of the connected mongoDB (first 2 digits)
+
+    :return: tuple(int, int)
+    """
+    version_list = get_connection().server_info()['versionArray'][:2]     # e.g: (3, 2)
+    return tuple(version_list)

tests/document/indexes.py

@@ -9,7 +9,8 @@ from six import iteritems
 
 from mongoengine import *
 from mongoengine.connection import get_db
-from tests.utils import get_mongodb_version, requires_mongodb_gte_26, MONGODB_32, MONGODB_3
+from mongoengine.mongodb_support import get_mongodb_version, MONGODB_32, MONGODB_3
+from tests.utils import requires_mongodb_gte_26, requires_mongodb_lte_32, requires_mongodb_gte_34
 
 __all__ = ("IndexesTest", )
 
@@ -477,6 +478,7 @@ class IndexesTest(unittest.TestCase):
     def test_covered_index(self):
         """Ensure that covered indexes can be used
         """
+        IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3
 
         class Test(Document):
             a = IntField()
@@ -492,8 +494,6 @@ class IndexesTest(unittest.TestCase):
         obj = Test(a=1)
         obj.save()
 
-        IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3
-
         # Need to be explicit about covered indexes as mongoDB doesn't know if
         # the documents returned might have more keys in that here.
         query_plan = Test.objects(id=obj.id).exclude('a').explain()
@@ -569,7 +569,7 @@ class IndexesTest(unittest.TestCase):
         if pymongo.version != '3.0':
             self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10)
 
-        if MONGO_VER == MONGODB_32:
+        if MONGO_VER >= MONGODB_32:
             # Mongo32 throws an error if an index exists (i.e `tags` in our case)
             # and you use hint on an index name that does not exist
             with self.assertRaises(OperationFailure):
@@ -601,6 +601,32 @@ class IndexesTest(unittest.TestCase):
         # Ensure backwards compatibilty for errors
         self.assertRaises(OperationError, post2.save)
 
+    @requires_mongodb_gte_34
+    def test_primary_key_unique_not_working_under_mongo_34(self):
+        """Relates to  #1445"""
+        class Blog(Document):
+            id = StringField(primary_key=True, unique=True)
+
+        Blog.drop_collection()
+
+        with self.assertRaises(OperationFailure) as ctx_err:
+            Blog(id='garbage').save()
+        try:
+            self.assertIn("The field 'unique' is not valid for an _id index specification", str(ctx_err.exception))
+        except AssertionError:
+            # error is slightly different on python 3.6
+            self.assertIn("The field 'background' is not valid for an _id index specification", str(ctx_err.exception))
+
+    @requires_mongodb_lte_32
+    def test_primary_key_unique_working_under_mongo_32(self):
+        """Relates to  #1445"""
+        class Blog(Document):
+            id = StringField(primary_key=True, unique=True)
+
+        Blog.drop_collection()
+
+        Blog(id='garbage').save()
+
     def test_unique_with(self):
         """Ensure that unique_with constraints are applied to fields.
         """
@@ -760,7 +786,7 @@ class IndexesTest(unittest.TestCase):
         You won't create a duplicate but you will update an existing document.
         """
         class User(Document):
-            name = StringField(primary_key=True, unique=True)
+            name = StringField(primary_key=True)
             password = StringField()
 
         User.drop_collection()

tests/fields/fields.py

@@ -2113,7 +2113,7 @@ class FieldTest(MongoDBTestCase):
             field_1 = StringField(db_field='f')
 
         class Doc(Document):
-            my_id = IntField(required=True, unique=True, primary_key=True)
+            my_id = IntField(primary_key=True)
             embed_me = DynamicField(db_field='e')
             field_x = StringField(db_field='x')
 
@@ -2135,7 +2135,7 @@ class FieldTest(MongoDBTestCase):
             field_1 = StringField(db_field='f')
 
         class Doc(Document):
-            my_id = IntField(required=True, unique=True, primary_key=True)
+            my_id = IntField(primary_key=True)
             embed_me = DynamicField(db_field='e')
             field_x = StringField(db_field='x')
 

tests/queryset/queryset.py

@@ -18,11 +18,12 @@ from mongoengine import *
 from mongoengine.connection import get_connection, get_db
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.errors import InvalidQueryError
+from mongoengine.mongodb_support import get_mongodb_version, MONGODB_32
 from mongoengine.python_support import IS_PYMONGO_3
 from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned,
                                   QuerySet, QuerySetManager, queryset_manager)
 
-from tests.utils import requires_mongodb_gte_26, skip_pymongo3, get_mongodb_version, MONGODB_32
+from tests.utils import requires_mongodb_gte_26, skip_pymongo3
 
 __all__ = ("QuerySetTest",)
 
@@ -852,7 +853,7 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 0)
             Blog.objects.insert(blogs, load_bulk=False)
 
-            if MONGO_VER == MONGODB_32:
+            if MONGO_VER >= MONGODB_32:
                 self.assertEqual(q, 1)      # 1 entry containing the list of inserts
             else:
                 self.assertEqual(q, len(blogs))     # 1 entry per doc inserted
@@ -869,7 +870,7 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 0)
             Blog.objects.insert(blogs)
 
-            if MONGO_VER == MONGODB_32:
+            if MONGO_VER >= MONGODB_32:
                 self.assertEqual(q, 2)              # 1 for insert 1 for fetch
             else:
                 self.assertEqual(q, len(blogs)+1)       # + 1 to fetch all docs
@@ -1204,7 +1205,7 @@ class QuerySetTest(unittest.TestCase):
         """Ensure filters can be chained together.
         """
         class Blog(Document):
-            id = StringField(unique=True, primary_key=True)
+            id = StringField(primary_key=True)
 
         class BlogPost(Document):
             blog = ReferenceField(Blog)
@@ -1316,7 +1317,7 @@ class QuerySetTest(unittest.TestCase):
         order_by() w/o any arguments.
         """
         MONGO_VER = self.mongodb_version
-        ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
+        ORDER_BY_KEY = 'sort' if MONGO_VER >= MONGODB_32 else '$orderby'
 
         class BlogPost(Document):
             title = StringField()
@@ -2524,8 +2525,8 @@ class QuerySetTest(unittest.TestCase):
     def test_comment(self):
         """Make sure adding a comment to the query gets added to the query"""
         MONGO_VER = self.mongodb_version
-        QUERY_KEY = 'filter' if MONGO_VER == MONGODB_32 else '$query'
+        QUERY_KEY = 'filter' if MONGO_VER >= MONGODB_32 else '$query'
-        COMMENT_KEY = 'comment' if MONGO_VER == MONGODB_32 else '$comment'
+        COMMENT_KEY = 'comment' if MONGO_VER >= MONGODB_32 else '$comment'
 
         class User(Document):
             age = IntField()
@@ -3349,7 +3350,7 @@ class QuerySetTest(unittest.TestCase):
             meta = {'indexes': [
                 {'fields': ['$title', "$content"],
                  'default_language': 'portuguese',
-                 'weight': {'title': 10, 'content': 2}
+                 'weights': {'title': 10, 'content': 2}
                  }
             ]}
 
@@ -5131,7 +5132,7 @@ class QuerySetTest(unittest.TestCase):
     def test_query_reference_to_custom_pk_doc(self):
 
         class A(Document):
-            id = StringField(unique=True, primary_key=True)
+            id = StringField(primary_key=True)
 
         class B(Document):
             a = ReferenceField(A)
@@ -5236,7 +5237,7 @@ class QuerySetTest(unittest.TestCase):
 
     def test_bool_with_ordering(self):
         MONGO_VER = self.mongodb_version
-        ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
+        ORDER_BY_KEY = 'sort' if MONGO_VER >= MONGODB_32 else '$orderby'
 
         class Person(Document):
             name = StringField()

tests/utils.py

@@ -1,22 +1,17 @@
+import operator
 import unittest
 
 from nose.plugins.skip import SkipTest
 
 from mongoengine import connect
-from mongoengine.connection import get_db, get_connection
+from mongoengine.connection import get_db
+from mongoengine.mongodb_support import get_mongodb_version, MONGODB_26, MONGODB_3, MONGODB_32, MONGODB_34
 from mongoengine.python_support import IS_PYMONGO_3
 
 
 MONGO_TEST_DB = 'mongoenginetest'   # standard name for the test database
 
 
-# Constant that can be used to compare the version retrieved with
-# get_mongodb_version()
-MONGODB_26 = (2, 6)
-MONGODB_3 = (3, 0)
-MONGODB_32 = (3, 2)
-
-
 class MongoDBTestCase(unittest.TestCase):
     """Base class for tests that need a mongodb connection
     It ensures that the db is clean at the beginning and dropped at the end automatically
@@ -38,46 +33,51 @@ def get_as_pymongo(doc):
     return doc.__class__.objects.as_pymongo().get(id=doc.id)
 
 
-def get_mongodb_version():
+def _decorated_with_ver_requirement(func, mongo_version_req, oper):
-    """Return the version of the connected mongoDB (first 2 digits)
-
-    :return: tuple(int, int)
-    """
-    version_list = get_connection().server_info()['versionArray'][:2]     # e.g: (3, 2)
-    return tuple(version_list)
-
-
-def _decorated_with_ver_requirement(func, version):
     """Return a given function decorated with the version requirement
     for a particular MongoDB version tuple.
 
-    :param version: The version required (tuple(int, int))
+    :param mongo_version_req: The mongodb version requirement (tuple(int, int))
+    :param oper: The operator to apply (e.g: operator.ge)
     """
     def _inner(*args, **kwargs):
-        MONGODB_V = get_mongodb_version()
+        mongodb_v = get_mongodb_version()
-        if MONGODB_V >= version:
+        if oper(mongodb_v, mongo_version_req):
             return func(*args, **kwargs)
 
-        raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in version)))
+        raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in mongo_version_req)))
 
     _inner.__name__ = func.__name__
     _inner.__doc__ = func.__doc__
-
     return _inner
 
 
+def requires_mongodb_gte_34(func):
+    """Raise a SkipTest exception if we're working with MongoDB version
+    lower than v3.4
+    """
+    return _decorated_with_ver_requirement(func, MONGODB_34, oper=operator.ge)
+
+
+def requires_mongodb_lte_32(func):
+    """Raise a SkipTest exception if we're working with MongoDB version
+    greater than v3.2.
+    """
+    return _decorated_with_ver_requirement(func, MONGODB_32, oper=operator.le)
+
+
 def requires_mongodb_gte_26(func):
     """Raise a SkipTest exception if we're working with MongoDB version
     lower than v2.6.
     """
-    return _decorated_with_ver_requirement(func, MONGODB_26)
+    return _decorated_with_ver_requirement(func, MONGODB_26, oper=operator.ge)
 
 
 def requires_mongodb_gte_3(func):
     """Raise a SkipTest exception if we're working with MongoDB version
     lower than v3.0.
     """
-    return _decorated_with_ver_requirement(func, MONGODB_3)
+    return _decorated_with_ver_requirement(func, MONGODB_3, oper=operator.ge)
 
 
 def skip_pymongo3(f):