diff --git a/.install_mongodb_on_travis.sh b/.install_mongodb_on_travis.sh index f2018411..a4ff085b 100644 --- a/.install_mongodb_on_travis.sh +++ b/.install_mongodb_on_travis.sh @@ -18,6 +18,12 @@ elif [ "$MONGODB" = "3.0" ]; then sudo apt-get update sudo apt-get install mongodb-org-server=3.0.14 # service should be started automatically +elif [ "$MONGODB" = "3.2" ]; then + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv EA312927 + echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list + sudo apt-get update + sudo apt-get install mongodb-org-server=3.2.20 + # service should be started automatically else echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0." exit 1 diff --git a/.travis.yml b/.travis.yml index 381f7385..f9c7db1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,17 +27,17 @@ matrix: include: - python: 2.7 - env: MONGODB=2.4 PYMONGO=3.5 + env: MONGODB=3.0 PYMONGO=3.5 - python: 2.7 - env: MONGODB=3.0 PYMONGO=3.x + env: MONGODB=3.2 PYMONGO=3.x - python: 3.5 - env: MONGODB=2.4 PYMONGO=3.5 + env: MONGODB=3.0 PYMONGO=3.5 - python: 3.5 - env: MONGODB=3.0 PYMONGO=3.x + env: MONGODB=3.2 PYMONGO=3.x - python: 3.6 - env: MONGODB=2.4 PYMONGO=3.5 + env: MONGODB=3.0 PYMONGO=3.5 - python: 3.6 - env: MONGODB=3.0 PYMONGO=3.x + env: MONGODB=3.2 PYMONGO=3.x before_install: - bash .install_mongodb_on_travis.sh diff --git a/README.rst b/README.rst index e1e2aef6..17cb3e33 100644 --- a/README.rst +++ b/README.rst @@ -26,10 +26,10 @@ an `API reference `_. Supported MongoDB Versions ========================== -MongoEngine is currently tested against MongoDB v2.4, v2.6, and v3.0. Future +MongoEngine is currently tested against MongoDB v2.6, v3.0 and v3.2. Future versions should be supported as well, but aren't actively tested at the moment. Make sure to open an issue or submit a pull request if you experience any -problems with MongoDB v3.2+. +problems with MongoDB v3.4+. Installation ============ diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index ee1f5e01..c26b0a79 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -182,8 +182,10 @@ class query_counter(object): self._ignored_query = { 'ns': {'$ne': '%s.system.indexes' % self.db.name}, - 'op': - {'$ne': 'killcursors'} + 'op': # MONGODB < 3.2 + {'$ne': 'killcursors'}, + 'command.killCursors': # MONGODB >= 3.2 + {'$exists': False} } def _turn_on_profiling(self): diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 1cbb4ec3..b3129fc8 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- import unittest -import sys +from datetime import datetime from nose.plugins.skip import SkipTest -from datetime import datetime +from pymongo.errors import OperationFailure import pymongo from mongoengine import * from mongoengine.connection import get_db - -from tests.utils import get_mongodb_version, needs_mongodb_v26 +from tests.utils import get_mongodb_version, needs_mongodb_v26, MONGODB_32, MONGODB_3 __all__ = ("IndexesTest", ) @@ -19,6 +18,7 @@ class IndexesTest(unittest.TestCase): def setUp(self): self.connection = connect(db='mongoenginetest') self.db = get_db() + self.mongodb_version = get_mongodb_version() class Person(Document): name = StringField() @@ -491,7 +491,7 @@ class IndexesTest(unittest.TestCase): obj = Test(a=1) obj.save() - IS_MONGODB_3 = get_mongodb_version()[0] >= 3 + IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3 # Need to be explicit about covered indexes as mongoDB doesn't know if # the documents returned might have more keys in that here. @@ -541,19 +541,24 @@ class IndexesTest(unittest.TestCase): [('categories', 1), ('_id', 1)]) def test_hint(self): + MONGO_VER = self.mongodb_version + TAGS_INDEX_NAME = 'tags_1' class BlogPost(Document): tags = ListField(StringField()) meta = { 'indexes': [ - 'tags', + { + 'fields': ['tags'], + 'name': TAGS_INDEX_NAME + } ], } BlogPost.drop_collection() - for i in range(0, 10): - tags = [("tag %i" % n) for n in range(0, i % 2)] + for i in range(10): + tags = [("tag %i" % n) for n in range(i % 2)] BlogPost(tags=tags).save() self.assertEqual(BlogPost.objects.count(), 10) @@ -563,18 +568,18 @@ class IndexesTest(unittest.TestCase): if pymongo.version != '3.0': self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) + if MONGO_VER == MONGODB_32: + # Mongo32 throws an error if an index exists (i.e `tags` in our case) + # and you use hint on an index name that does not exist + with self.assertRaises(OperationFailure): + BlogPost.objects.hint([('ZZ', 1)]).count() + else: self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) - if pymongo.version >= '2.8': - self.assertEqual(BlogPost.objects.hint('tags').count(), 10) - else: - def invalid_index(): - BlogPost.objects.hint('tags').next() - self.assertRaises(TypeError, invalid_index) + self.assertEqual(BlogPost.objects.hint(TAGS_INDEX_NAME ).count(), 10) - def invalid_index_2(): - return BlogPost.objects.hint(('tags', 1)).next() - self.assertRaises(Exception, invalid_index_2) + with self.assertRaises(Exception): + BlogPost.objects.hint(('tags', 1)).next() def test_unique(self): """Ensure that uniqueness constraints are applied to fields. diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 35ebe24d..050f65d9 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -21,8 +21,7 @@ from mongoengine.python_support import IS_PYMONGO_3 from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned, QuerySet, QuerySetManager, queryset_manager) -from tests.utils import needs_mongodb_v26, skip_pymongo3 - +from tests.utils import needs_mongodb_v26, skip_pymongo3, get_mongodb_version, MONGODB_32 __all__ = ("QuerySetTest",) @@ -30,10 +29,8 @@ __all__ = ("QuerySetTest",) class db_ops_tracker(query_counter): def get_ops(self): - ignore_query = { - 'ns': {'$ne': '%s.system.indexes' % self.db.name}, - 'command.count': {'$ne': 'system.profile'} - } + ignore_query = dict(self._ignored_query) + ignore_query['command.count'] = {'$ne': 'system.profile'} # Ignore the query issued by query_counter return list(self.db.system.profile.find(ignore_query)) @@ -56,6 +53,8 @@ class QuerySetTest(unittest.TestCase): self.PersonMeta = PersonMeta self.Person = Person + self.mongodb_version = get_mongodb_version() + def test_initialisation(self): """Ensure that a QuerySet is correctly initialised by QuerySetManager. """ @@ -813,8 +812,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(record.embed.field, 2) def test_bulk_insert(self): - """Ensure that bulk insert works - """ + """Ensure that bulk insert works""" + MONGO_VER = self.mongodb_version class Comment(EmbeddedDocument): name = StringField() @@ -832,35 +831,41 @@ class QuerySetTest(unittest.TestCase): # get MongoDB version info connection = get_connection() info = connection.test.command('buildInfo') - mongodb_version = tuple([int(i) for i in info['version'].split('.')]) # Recreates the collection self.assertEqual(0, Blog.objects.count()) + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blogs = [Blog(title="post %s" % i, posts=[post1, post2]) + for i in range(99)] + + # Check bulk insert using load_bulk=False with query_counter() as q: self.assertEqual(q, 0) - - comment1 = Comment(name='testa') - comment2 = Comment(name='testb') - post1 = Post(comments=[comment1, comment2]) - post2 = Post(comments=[comment2, comment2]) - - blogs = [] - for i in range(1, 100): - blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) - Blog.objects.insert(blogs, load_bulk=False) - # profiling logs each doc now in the bulk op - self.assertEqual(q, 99) + + if MONGO_VER == MONGODB_32: + self.assertEqual(q, 1) # 1 entry containing the list of inserts + else: + self.assertEqual(q, len(blogs)) # 1 entry per doc inserted + + self.assertEqual(Blog.objects.count(), len(blogs)) Blog.drop_collection() Blog.ensure_indexes() + # Check bulk insert using load_bulk=True with query_counter() as q: self.assertEqual(q, 0) - Blog.objects.insert(blogs) - self.assertEqual(q, 100) # 99 for insert 1 for fetch + + if MONGO_VER == MONGODB_32: + self.assertEqual(q, 2) # 1 for insert 1 for fetch + else: + self.assertEqual(q, len(blogs)+1) # + 1 to fetch all docs Blog.drop_collection() @@ -1262,6 +1267,9 @@ class QuerySetTest(unittest.TestCase): """Ensure that the default ordering can be cleared by calling order_by() w/o any arguments. """ + MONGO_VER = self.mongodb_version + ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby' + class BlogPost(Document): title = StringField() published_date = DateTimeField() @@ -1277,7 +1285,7 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects.filter(title='whatever').first() self.assertEqual(len(q.get_ops()), 1) self.assertEqual( - q.get_ops()[0]['query']['$orderby'], + q.get_ops()[0]['query'][ORDER_BY_KEY], {'published_date': -1} ) @@ -1285,14 +1293,14 @@ class QuerySetTest(unittest.TestCase): with db_ops_tracker() as q: BlogPost.objects.filter(title='whatever').order_by().first() self.assertEqual(len(q.get_ops()), 1) - self.assertNotIn('$orderby', q.get_ops()[0]['query']) + self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query']) # calling an explicit order_by should use a specified sort with db_ops_tracker() as q: BlogPost.objects.filter(title='whatever').order_by('published_date').first() self.assertEqual(len(q.get_ops()), 1) self.assertEqual( - q.get_ops()[0]['query']['$orderby'], + q.get_ops()[0]['query'][ORDER_BY_KEY], {'published_date': 1} ) @@ -1301,11 +1309,14 @@ class QuerySetTest(unittest.TestCase): qs = BlogPost.objects.filter(title='whatever').order_by('published_date') qs.order_by().first() self.assertEqual(len(q.get_ops()), 1) - self.assertNotIn('$orderby', q.get_ops()[0]['query']) + self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query']) def test_no_ordering_for_get(self): """ Ensure that Doc.objects.get doesn't use any ordering. """ + MONGO_VER = self.mongodb_version + ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby' + class BlogPost(Document): title = StringField() published_date = DateTimeField() @@ -1320,13 +1331,13 @@ class QuerySetTest(unittest.TestCase): with db_ops_tracker() as q: BlogPost.objects.get(title='whatever') self.assertEqual(len(q.get_ops()), 1) - self.assertNotIn('$orderby', q.get_ops()[0]['query']) + self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query']) # Ordering should be ignored for .get even if we set it explicitly with db_ops_tracker() as q: BlogPost.objects.order_by('-title').get(title='whatever') self.assertEqual(len(q.get_ops()), 1) - self.assertNotIn('$orderby', q.get_ops()[0]['query']) + self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query']) def test_find_embedded(self): """Ensure that an embedded document is properly returned from @@ -2450,7 +2461,11 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(names, ['User A', 'User B', 'User C']) def test_comment(self): - """Make sure adding a comment to the query works.""" + """Make sure adding a comment to the query gets added to the query""" + MONGO_VER = self.mongodb_version + QUERY_KEY = 'filter' if MONGO_VER == MONGODB_32 else '$query' + COMMENT_KEY = 'comment' if MONGO_VER == MONGODB_32 else '$comment' + class User(Document): age = IntField() @@ -2466,8 +2481,8 @@ class QuerySetTest(unittest.TestCase): ops = q.get_ops() self.assertEqual(len(ops), 2) for op in ops: - self.assertEqual(op['query']['$query'], {'age': {'$gte': 18}}) - self.assertEqual(op['query']['$comment'], 'looking for an adult') + self.assertEqual(op['query'][QUERY_KEY], {'age': {'$gte': 18}}) + self.assertEqual(op['query'][COMMENT_KEY], 'looking for an adult') def test_map_reduce(self): """Ensure map/reduce is both mapping and reducing. @@ -4817,27 +4832,18 @@ class QuerySetTest(unittest.TestCase): for i in range(100): Person(name="No: %s" % i).save() - with query_counter() as q: - try: - self.assertEqual(q, 0) - people = Person.objects.no_cache() + with query_counter() as q: + self.assertEqual(q, 0) + people = Person.objects.no_cache() - [x for x in people] - self.assertEqual(q, 1) + [x for x in people] + self.assertEqual(q, 1) - list(people) - self.assertEqual(q, 2) - - people.count() - self.assertEqual(q, 3) - except AssertionError as exc: - db = get_db() - msg = '' - for q in list(db.system.profile.find())[-50:]: - msg += str([q['ts'], q['ns'], q.get('query'), q['op']])+'\n' - msg += str(q) - raise AssertionError(str(exc) + '\n'+msg) + list(people) + self.assertEqual(q, 2) + people.count() + self.assertEqual(q, 3) def test_cache_not_cloned(self): @@ -5111,35 +5117,39 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(op['nreturned'], 1) def test_bool_with_ordering(self): + MONGO_VER = self.mongodb_version + ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby' class Person(Document): name = StringField() Person.drop_collection() + Person(name="Test").save() + # Check that bool(queryset) does not uses the orderby qs = Person.objects.order_by('name') - with query_counter() as q: - if qs: + if bool(qs): pass op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertNotIn('$orderby', op['query'], - 'BaseQuerySet cannot use orderby in if stmt') + self.assertNotIn(ORDER_BY_KEY, op['query']) + # Check that normal query uses orderby + qs2 = Person.objects.order_by('name') with query_counter() as p: - for x in qs: + for x in qs2: pass op = p.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertIn('$orderby', op['query'], 'BaseQuerySet cannot remove orderby in for loop') + self.assertIn(ORDER_BY_KEY, op['query']) def test_bool_with_ordering_from_meta_dict(self): diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index df5e5212..8fb7bc78 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -299,7 +299,6 @@ class ContextManagersTest(unittest.TestCase): cursor.close() # issues a `killcursors` query that is ignored by the context self.assertEqual(q, 1) - _ = db.system.indexes.find_one() # queries on db.system.indexes are ignored as well self.assertEqual(q, 1) diff --git a/tests/utils.py b/tests/utils.py index acd318c5..39d3681e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,6 +10,13 @@ from mongoengine.python_support import IS_PYMONGO_3 MONGO_TEST_DB = 'mongoenginetest' # standard name for the test database +# Constant that can be used to compare the version retrieved with +# get_mongodb_version() +MONGODB_26 = (2, 6) +MONGODB_3 = (3,0) +MONGODB_32 = (3, 2) + + class MongoDBTestCase(unittest.TestCase): """Base class for tests that need a mongodb connection It ensures that the db is clean at the beginning and dropped at the end automatically @@ -27,24 +34,26 @@ class MongoDBTestCase(unittest.TestCase): def get_mongodb_version(): - """Return the version tuple of the MongoDB server that the default - connection is connected to. + """Return the version of the connected mongoDB (first 2 digits) + + :return: tuple(int, int) """ - return tuple(get_connection().server_info()['versionArray']) + version_list = get_connection().server_info()['versionArray'][:2] # e.g: (3, 2) + return tuple(version_list) -def _decorated_with_ver_requirement(func, ver_tuple): +def _decorated_with_ver_requirement(func, version): """Return a given function decorated with the version requirement for a particular MongoDB version tuple. + + :param version: The version required (tuple(int, int)) """ def _inner(*args, **kwargs): - mongodb_ver = get_mongodb_version() - if mongodb_ver >= ver_tuple: + MONGODB_V = get_mongodb_version() + if MONGODB_V >= version: return func(*args, **kwargs) - raise SkipTest('Needs MongoDB v{}+'.format( - '.'.join([str(v) for v in ver_tuple]) - )) + raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in version))) _inner.__name__ = func.__name__ _inner.__doc__ = func.__doc__ @@ -56,14 +65,14 @@ def needs_mongodb_v26(func): """Raise a SkipTest exception if we're working with MongoDB version lower than v2.6. """ - return _decorated_with_ver_requirement(func, (2, 6)) + return _decorated_with_ver_requirement(func, MONGODB_26) def needs_mongodb_v3(func): """Raise a SkipTest exception if we're working with MongoDB version lower than v3.0. """ - return _decorated_with_ver_requirement(func, (3, 0)) + return _decorated_with_ver_requirement(func, MONGODB_3) def skip_pymongo3(f):