From dca837b843272385e7be80c7d6c1c037dc38393d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 2 Oct 2018 21:26:14 +0200 Subject: [PATCH] Add suport for Mongo 3.4 (travis, fix tests) --- .install_mongodb_on_travis.sh | 8 +++++- .travis.yml | 9 +++---- README.rst | 4 +-- docs/changelog.rst | 1 + mongoengine/mongodb_support.py | 21 ++++++++++++++++ tests/document/indexes.py | 26 +++++++++++++++---- tests/fields/fields.py | 4 +-- tests/queryset/queryset.py | 25 +++++++++--------- tests/utils.py | 46 +++++++++++++++++----------------- 9 files changed, 94 insertions(+), 50 deletions(-) create mode 100644 mongoengine/mongodb_support.py diff --git a/.install_mongodb_on_travis.sh b/.install_mongodb_on_travis.sh index 057ccf74..6ac2e364 100644 --- a/.install_mongodb_on_travis.sh +++ b/.install_mongodb_on_travis.sh @@ -19,8 +19,14 @@ elif [ "$MONGODB" = "3.2" ]; then sudo apt-get update sudo apt-get install mongodb-org-server=3.2.20 # service should be started automatically +elif [ "$MONGODB" = "3.4" ]; then + sudo apt-key adv --keyserver keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 + echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list + sudo apt-get update + sudo apt-get install mongodb-org-server=3.4.17 + # service should be started automatically else - echo "Invalid MongoDB version, expected 2.6, 3.0, or 3.2" + echo "Invalid MongoDB version, expected 2.6, 3.0, 3.2 or 3.4." exit 1 fi; diff --git a/.travis.yml b/.travis.yml index 4f77f4e0..a73391aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,9 @@ # combinations: # * MongoDB v2.6 is currently the "main" version tested against Python v2.7, # v3.5, v3.6, PyPy, and PyMongo v3.x. -# * MongoDB v3.0 & v3.2 are tested against Python v2.7, v3.5 & v3.6 +# * MongoDB v3.0, v3.2 are tested against Python v2.7, v3.5 & v3.6 # and Pymongo v3.5 & v3.x +# * MongoDB v3.4 is tested against v3.6 and Pymongo v3.x # Reminder: Update README.rst if you change MongoDB versions we test. language: python @@ -26,16 +27,14 @@ matrix: include: - python: 2.7 env: MONGODB=3.0 PYMONGO=3.5 - - python: 2.7 - env: MONGODB=3.2 PYMONGO=3.x - - python: 3.5 - env: MONGODB=3.0 PYMONGO=3.5 - python: 3.5 env: MONGODB=3.2 PYMONGO=3.x - python: 3.6 env: MONGODB=3.0 PYMONGO=3.5 - python: 3.6 env: MONGODB=3.2 PYMONGO=3.x + - python: 3.6 + env: MONGODB=3.4 PYMONGO=3.x before_install: - bash .install_mongodb_on_travis.sh diff --git a/README.rst b/README.rst index 4e186a85..f0309170 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ an `API reference `_. Supported MongoDB Versions ========================== -MongoEngine is currently tested against MongoDB v2.6, v3.0 and v3.2. Future +MongoEngine is currently tested against MongoDB v2.6, v3.0, v3.2 and v3.4. Future versions should be supported as well, but aren't actively tested at the moment. Make sure to open an issue or submit a pull request if you experience any problems with MongoDB v3.4+. @@ -36,7 +36,7 @@ Installation We recommend the use of `virtualenv `_ and of `pip `_. You can then use ``pip install -U mongoengine``. You may also have `setuptools `_ -and thus you can use ``easy_install -U mongoengine``. Another option is +and thus you can use ``easy_install -U mongoengine``. Another option is `pipenv `_. You can then use ``pipenv install mongoengine`` to both create the virtual environment and install the package. Otherwise, you can download the source from `GitHub `_ and diff --git a/docs/changelog.rst b/docs/changelog.rst index ae734448..c87fd1e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Development - Document a BREAKING CHANGE introduced in 0.15.3 and not reported at that time (#1995) - Fix InvalidStringData error when using modify on a BinaryField #1127 - DEPRECATION: `EmbeddedDocument.save` & `.reload` are marked as deprecated and will be removed in a next version of mongoengine #1552 +- Fix test suite and CI to support MongoDB 3.4 #1445 ================= Changes in 0.16.3 diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py new file mode 100644 index 00000000..b5f3bdc8 --- /dev/null +++ b/mongoengine/mongodb_support.py @@ -0,0 +1,21 @@ +""" +Helper functions, constants, and types to aid with MongoDB v3.x support +""" +from mongoengine.connection import get_connection + + +# Constant that can be used to compare the version retrieved with +# get_mongodb_version() +MONGODB_34 = (3, 4) +MONGODB_32 = (3, 2) +MONGODB_3 = (3, 0) +MONGODB_26 = (2, 6) + + +def get_mongodb_version(): + """Return the version of the connected mongoDB (first 2 digits) + + :return: tuple(int, int) + """ + version_list = get_connection().server_info()['versionArray'][:2] # e.g: (3, 2) + return tuple(version_list) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 57f48587..a21b45bc 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -9,7 +9,8 @@ from six import iteritems from mongoengine import * from mongoengine.connection import get_db -from tests.utils import get_mongodb_version, requires_mongodb_gte_26, MONGODB_32, MONGODB_3 +from mongoengine.mongodb_support import get_mongodb_version, MONGODB_32, MONGODB_3 +from tests.utils import requires_mongodb_gte_26, requires_mongodb_lte_32, requires_mongodb_gte_34 __all__ = ("IndexesTest", ) @@ -477,6 +478,7 @@ class IndexesTest(unittest.TestCase): def test_covered_index(self): """Ensure that covered indexes can be used """ + IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3 class Test(Document): a = IntField() @@ -492,8 +494,6 @@ class IndexesTest(unittest.TestCase): obj = Test(a=1) obj.save() - IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3 - # Need to be explicit about covered indexes as mongoDB doesn't know if # the documents returned might have more keys in that here. query_plan = Test.objects(id=obj.id).exclude('a').explain() @@ -569,7 +569,7 @@ class IndexesTest(unittest.TestCase): if pymongo.version != '3.0': self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) - if MONGO_VER == MONGODB_32: + if MONGO_VER >= MONGODB_32: # Mongo32 throws an error if an index exists (i.e `tags` in our case) # and you use hint on an index name that does not exist with self.assertRaises(OperationFailure): @@ -601,6 +601,22 @@ class IndexesTest(unittest.TestCase): # Ensure backwards compatibilty for errors self.assertRaises(OperationError, post2.save) + @requires_mongodb_gte_34 + def test_primary_key_unique_not_working_under_mongo_34(self): + class Blog(Document): + id = StringField(primary_key=True, unique=True) + + with self.assertRaises(OperationFailure) as ctx_err: + Blog(id='garbage').save() + self.assertIn("The field 'unique' is not valid for an _id index specification", str(ctx_err.exception)) + + @requires_mongodb_lte_32 + def test_primary_key_unique_working_under_mongo_32(self): + class Blog(Document): + id = StringField(primary_key=True, unique=True) + + Blog(id='garbage').save() + def test_unique_with(self): """Ensure that unique_with constraints are applied to fields. """ @@ -760,7 +776,7 @@ class IndexesTest(unittest.TestCase): You won't create a duplicate but you will update an existing document. """ class User(Document): - name = StringField(primary_key=True, unique=True) + name = StringField(primary_key=True) password = StringField() User.drop_collection() diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 128936bf..2c4ac3ac 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2113,7 +2113,7 @@ class FieldTest(MongoDBTestCase): field_1 = StringField(db_field='f') class Doc(Document): - my_id = IntField(required=True, unique=True, primary_key=True) + my_id = IntField(primary_key=True) embed_me = DynamicField(db_field='e') field_x = StringField(db_field='x') @@ -2135,7 +2135,7 @@ class FieldTest(MongoDBTestCase): field_1 = StringField(db_field='f') class Doc(Document): - my_id = IntField(required=True, unique=True, primary_key=True) + my_id = IntField(primary_key=True) embed_me = DynamicField(db_field='e') field_x = StringField(db_field='x') diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3d8e8960..05a7ca75 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -18,11 +18,12 @@ from mongoengine import * from mongoengine.connection import get_connection, get_db from mongoengine.context_managers import query_counter, switch_db from mongoengine.errors import InvalidQueryError +from mongoengine.mongodb_support import get_mongodb_version, MONGODB_32 from mongoengine.python_support import IS_PYMONGO_3 from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned, QuerySet, QuerySetManager, queryset_manager) -from tests.utils import requires_mongodb_gte_26, skip_pymongo3, get_mongodb_version, MONGODB_32 +from tests.utils import requires_mongodb_gte_26, skip_pymongo3 __all__ = ("QuerySetTest",) @@ -852,8 +853,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) Blog.objects.insert(blogs, load_bulk=False) - if MONGO_VER == MONGODB_32: - self.assertEqual(q, 1) # 1 entry containing the list of inserts + if MONGO_VER >= MONGODB_32: + self.assertEqual(q, 1) # 1 entry containing the list of inserts else: self.assertEqual(q, len(blogs)) # 1 entry per doc inserted @@ -869,8 +870,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) Blog.objects.insert(blogs) - if MONGO_VER == MONGODB_32: - self.assertEqual(q, 2) # 1 for insert 1 for fetch + if MONGO_VER >= MONGODB_32: + self.assertEqual(q, 2) # 1 for insert 1 for fetch else: self.assertEqual(q, len(blogs)+1) # + 1 to fetch all docs @@ -1204,7 +1205,7 @@ class QuerySetTest(unittest.TestCase): """Ensure filters can be chained together. """ class Blog(Document): - id = StringField(unique=True, primary_key=True) + id = StringField(primary_key=True) class BlogPost(Document): blog = ReferenceField(Blog) @@ -1316,7 +1317,7 @@ class QuerySetTest(unittest.TestCase): order_by() w/o any arguments. """ MONGO_VER = self.mongodb_version - ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby' + ORDER_BY_KEY = 'sort' if MONGO_VER >= MONGODB_32 else '$orderby' class BlogPost(Document): title = StringField() @@ -2524,8 +2525,8 @@ class QuerySetTest(unittest.TestCase): def test_comment(self): """Make sure adding a comment to the query gets added to the query""" MONGO_VER = self.mongodb_version - QUERY_KEY = 'filter' if MONGO_VER == MONGODB_32 else '$query' - COMMENT_KEY = 'comment' if MONGO_VER == MONGODB_32 else '$comment' + QUERY_KEY = 'filter' if MONGO_VER >= MONGODB_32 else '$query' + COMMENT_KEY = 'comment' if MONGO_VER >= MONGODB_32 else '$comment' class User(Document): age = IntField() @@ -3349,7 +3350,7 @@ class QuerySetTest(unittest.TestCase): meta = {'indexes': [ {'fields': ['$title', "$content"], 'default_language': 'portuguese', - 'weight': {'title': 10, 'content': 2} + 'weights': {'title': 10, 'content': 2} } ]} @@ -5131,7 +5132,7 @@ class QuerySetTest(unittest.TestCase): def test_query_reference_to_custom_pk_doc(self): class A(Document): - id = StringField(unique=True, primary_key=True) + id = StringField(primary_key=True) class B(Document): a = ReferenceField(A) @@ -5236,7 +5237,7 @@ class QuerySetTest(unittest.TestCase): def test_bool_with_ordering(self): MONGO_VER = self.mongodb_version - ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby' + ORDER_BY_KEY = 'sort' if MONGO_VER >= MONGODB_32 else '$orderby' class Person(Document): name = StringField() diff --git a/tests/utils.py b/tests/utils.py index 19936a54..e0380dbc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,22 +1,17 @@ +import operator import unittest from nose.plugins.skip import SkipTest from mongoengine import connect -from mongoengine.connection import get_db, get_connection +from mongoengine.connection import get_db +from mongoengine.mongodb_support import get_mongodb_version, MONGODB_26, MONGODB_3, MONGODB_32, MONGODB_34 from mongoengine.python_support import IS_PYMONGO_3 MONGO_TEST_DB = 'mongoenginetest' # standard name for the test database -# Constant that can be used to compare the version retrieved with -# get_mongodb_version() -MONGODB_26 = (2, 6) -MONGODB_3 = (3, 0) -MONGODB_32 = (3, 2) - - class MongoDBTestCase(unittest.TestCase): """Base class for tests that need a mongodb connection It ensures that the db is clean at the beginning and dropped at the end automatically @@ -38,34 +33,39 @@ def get_as_pymongo(doc): return doc.__class__.objects.as_pymongo().get(id=doc.id) -def get_mongodb_version(): - """Return the version of the connected mongoDB (first 2 digits) - - :return: tuple(int, int) - """ - version_list = get_connection().server_info()['versionArray'][:2] # e.g: (3, 2) - return tuple(version_list) - - -def _decorated_with_ver_requirement(func, version): +def _decorated_with_ver_requirement(func, mongo_version_req, oper=operator.ge): """Return a given function decorated with the version requirement for a particular MongoDB version tuple. - :param version: The version required (tuple(int, int)) + :param mongo_version_req: The mongodb version requirement (tuple(int, int)) + :param oper: The operator to apply """ def _inner(*args, **kwargs): - MONGODB_V = get_mongodb_version() - if MONGODB_V >= version: + mongodb_v = get_mongodb_version() + if oper(mongodb_v, mongo_version_req): return func(*args, **kwargs) - raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in version))) + raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in mongo_version_req))) _inner.__name__ = func.__name__ _inner.__doc__ = func.__doc__ - return _inner +def requires_mongodb_gte_34(func): + """Raise a SkipTest exception if we're working with MongoDB version + lower than v3.4 + """ + return _decorated_with_ver_requirement(func, MONGODB_34) + + +def requires_mongodb_lte_32(func): + """Raise a SkipTest exception if we're working with MongoDB version + greater than v3.2. + """ + return _decorated_with_ver_requirement(func, MONGODB_32, oper=operator.le) + + def requires_mongodb_gte_26(func): """Raise a SkipTest exception if we're working with MongoDB version lower than v2.6.