pub/mongoengine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Fixed tests to allow support of MongoDB 3.2

Browse Source 
- Replaced MongoDB 2.4 tests in CI by MongoDB 3.2
This commit is contained in:
Bastien Gérard 2018-09-28 22:39:08 +02:00
parent 1887f5b7e7
commit 96f0919633
8 changed files with 126 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.install_mongodb_on_travis.sh
View File

@ -18,6 +18,12 @@ elif [ "$MONGODB" = "3.0" ]; then
sudo apt-get update
sudo apt-get install mongodb-org-server=3.0.14
# service should be started automatically
elif [ "$MONGODB" = "3.2" ]; then
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv EA312927
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
sudo apt-get update
sudo apt-get install mongodb-org-server=3.2.20
# service should be started automatically
else
echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
exit 1

12
.travis.yml
View File

@ -27,17 +27,17 @@ matrix:
include:
- python: 2.7
env: MONGODB=2.4 PYMONGO=3.5
env: MONGODB=3.0 PYMONGO=3.5
- python: 2.7
env: MONGODB=3.0 PYMONGO=3.x
env: MONGODB=3.2 PYMONGO=3.x
- python: 3.5
env: MONGODB=2.4 PYMONGO=3.5
env: MONGODB=3.0 PYMONGO=3.5
- python: 3.5
env: MONGODB=3.0 PYMONGO=3.x
env: MONGODB=3.2 PYMONGO=3.x
- python: 3.6
env: MONGODB=2.4 PYMONGO=3.5
env: MONGODB=3.0 PYMONGO=3.5
- python: 3.6
env: MONGODB=3.0 PYMONGO=3.x
env: MONGODB=3.2 PYMONGO=3.x
before_install:
- bash .install_mongodb_on_travis.sh

4
README.rst
View File

@ -26,10 +26,10 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
Supported MongoDB Versions
==========================
MongoEngine is currently tested against MongoDB v2.4, v2.6, and v3.0. Future
MongoEngine is currently tested against MongoDB v2.6, v3.0 and v3.2. Future
versions should be supported as well, but aren't actively tested at the moment.
Make sure to open an issue or submit a pull request if you experience any
problems with MongoDB v3.2+.
problems with MongoDB v3.4+.
Installation
============

6
mongoengine/context_managers.py
View File

@ -182,8 +182,10 @@ class query_counter(object):
self._ignored_query = {
'ns':
{'$ne': '%s.system.indexes' % self.db.name},
'op':
{'$ne': 'killcursors'}
'op': # MONGODB < 3.2
{'$ne': 'killcursors'},
'command.killCursors': # MONGODB >= 3.2
{'$exists': False}
}
def _turn_on_profiling(self):

39
tests/document/indexes.py
View File

@ -1,15 +1,14 @@
# -*- coding: utf-8 -*-
import unittest
import sys
from datetime import datetime
from nose.plugins.skip import SkipTest
from datetime import datetime
from pymongo.errors import OperationFailure
import pymongo
from mongoengine import *
from mongoengine.connection import get_db
from tests.utils import get_mongodb_version, needs_mongodb_v26
from tests.utils import get_mongodb_version, needs_mongodb_v26, MONGODB_32, MONGODB_3
__all__ = ("IndexesTest", )
@ -19,6 +18,7 @@ class IndexesTest(unittest.TestCase):
def setUp(self):
self.connection = connect(db='mongoenginetest')
self.db = get_db()
self.mongodb_version = get_mongodb_version()
class Person(Document):
name = StringField()
@ -491,7 +491,7 @@ class IndexesTest(unittest.TestCase):
obj = Test(a=1)
obj.save()
IS_MONGODB_3 = get_mongodb_version()[0] >= 3
IS_MONGODB_3 = get_mongodb_version() >= MONGODB_3
# Need to be explicit about covered indexes as mongoDB doesn't know if
# the documents returned might have more keys in that here.
@ -541,19 +541,24 @@ class IndexesTest(unittest.TestCase):
[('categories', 1), ('_id', 1)])
def test_hint(self):
MONGO_VER = self.mongodb_version
TAGS_INDEX_NAME = 'tags_1'
class BlogPost(Document):
tags = ListField(StringField())
meta = {
'indexes': [
'tags',
{
'fields': ['tags'],
'name': TAGS_INDEX_NAME
}
],
}
BlogPost.drop_collection()
for i in range(0, 10):
tags = [("tag %i" % n) for n in range(0, i % 2)]
for i in range(10):
tags = [("tag %i" % n) for n in range(i % 2)]
BlogPost(tags=tags).save()
self.assertEqual(BlogPost.objects.count(), 10)
@ -563,18 +568,18 @@ class IndexesTest(unittest.TestCase):
if pymongo.version != '3.0':
self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10)
if MONGO_VER == MONGODB_32:
# Mongo32 throws an error if an index exists (i.e `tags` in our case)
# and you use hint on an index name that does not exist
with self.assertRaises(OperationFailure):
BlogPost.objects.hint([('ZZ', 1)]).count()
else:
self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10)
if pymongo.version >= '2.8':
self.assertEqual(BlogPost.objects.hint('tags').count(), 10)
else:
def invalid_index():
BlogPost.objects.hint('tags').next()
self.assertRaises(TypeError, invalid_index)
self.assertEqual(BlogPost.objects.hint(TAGS_INDEX_NAME ).count(), 10)
def invalid_index_2():
return BlogPost.objects.hint(('tags', 1)).next()
self.assertRaises(Exception, invalid_index_2)
with self.assertRaises(Exception):
BlogPost.objects.hint(('tags', 1)).next()
def test_unique(self):
"""Ensure that uniqueness constraints are applied to fields.

122
tests/queryset/queryset.py
View File

@ -21,8 +21,7 @@ from mongoengine.python_support import IS_PYMONGO_3
from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned,
QuerySet, QuerySetManager, queryset_manager)
from tests.utils import needs_mongodb_v26, skip_pymongo3
from tests.utils import needs_mongodb_v26, skip_pymongo3, get_mongodb_version, MONGODB_32
__all__ = ("QuerySetTest",)
@ -30,10 +29,8 @@ __all__ = ("QuerySetTest",)
class db_ops_tracker(query_counter):
def get_ops(self):
ignore_query = {
'ns': {'$ne': '%s.system.indexes' % self.db.name},
'command.count': {'$ne': 'system.profile'}
}
ignore_query = dict(self._ignored_query)
ignore_query['command.count'] = {'$ne': 'system.profile'} # Ignore the query issued by query_counter
return list(self.db.system.profile.find(ignore_query))
@ -56,6 +53,8 @@ class QuerySetTest(unittest.TestCase):
self.PersonMeta = PersonMeta
self.Person = Person
self.mongodb_version = get_mongodb_version()
def test_initialisation(self):
"""Ensure that a QuerySet is correctly initialised by QuerySetManager.
"""
@ -813,8 +812,8 @@ class QuerySetTest(unittest.TestCase):
self.assertEqual(record.embed.field, 2)
def test_bulk_insert(self):
"""Ensure that bulk insert works
"""
"""Ensure that bulk insert works"""
MONGO_VER = self.mongodb_version
class Comment(EmbeddedDocument):
name = StringField()
@ -832,35 +831,41 @@ class QuerySetTest(unittest.TestCase):
# get MongoDB version info
connection = get_connection()
info = connection.test.command('buildInfo')
mongodb_version = tuple([int(i) for i in info['version'].split('.')])
# Recreates the collection
self.assertEqual(0, Blog.objects.count())
comment1 = Comment(name='testa')
comment2 = Comment(name='testb')
post1 = Post(comments=[comment1, comment2])
post2 = Post(comments=[comment2, comment2])
blogs = [Blog(title="post %s" % i, posts=[post1, post2])
for i in range(99)]
# Check bulk insert using load_bulk=False
with query_counter() as q:
self.assertEqual(q, 0)
comment1 = Comment(name='testa')
comment2 = Comment(name='testb')
post1 = Post(comments=[comment1, comment2])
post2 = Post(comments=[comment2, comment2])
blogs = []
for i in range(1, 100):
blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
Blog.objects.insert(blogs, load_bulk=False)
# profiling logs each doc now in the bulk op
self.assertEqual(q, 99)
if MONGO_VER == MONGODB_32:
self.assertEqual(q, 1) # 1 entry containing the list of inserts
else:
self.assertEqual(q, len(blogs)) # 1 entry per doc inserted
self.assertEqual(Blog.objects.count(), len(blogs))
Blog.drop_collection()
Blog.ensure_indexes()
# Check bulk insert using load_bulk=True
with query_counter() as q:
self.assertEqual(q, 0)
Blog.objects.insert(blogs)
self.assertEqual(q, 100) # 99 for insert 1 for fetch
if MONGO_VER == MONGODB_32:
self.assertEqual(q, 2) # 1 for insert 1 for fetch
else:
self.assertEqual(q, len(blogs)+1) # + 1 to fetch all docs
Blog.drop_collection()
@ -1262,6 +1267,9 @@ class QuerySetTest(unittest.TestCase):
"""Ensure that the default ordering can be cleared by calling
order_by() w/o any arguments.
"""
MONGO_VER = self.mongodb_version
ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
class BlogPost(Document):
title = StringField()
published_date = DateTimeField()
@ -1277,7 +1285,7 @@ class QuerySetTest(unittest.TestCase):
BlogPost.objects.filter(title='whatever').first()
self.assertEqual(len(q.get_ops()), 1)
self.assertEqual(
q.get_ops()[0]['query']['$orderby'],
q.get_ops()[0]['query'][ORDER_BY_KEY],
{'published_date': -1}
)
@ -1285,14 +1293,14 @@ class QuerySetTest(unittest.TestCase):
with db_ops_tracker() as q:
BlogPost.objects.filter(title='whatever').order_by().first()
self.assertEqual(len(q.get_ops()), 1)
self.assertNotIn('$orderby', q.get_ops()[0]['query'])
self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
# calling an explicit order_by should use a specified sort
with db_ops_tracker() as q:
BlogPost.objects.filter(title='whatever').order_by('published_date').first()
self.assertEqual(len(q.get_ops()), 1)
self.assertEqual(
q.get_ops()[0]['query']['$orderby'],
q.get_ops()[0]['query'][ORDER_BY_KEY],
{'published_date': 1}
)
@ -1301,11 +1309,14 @@ class QuerySetTest(unittest.TestCase):
qs = BlogPost.objects.filter(title='whatever').order_by('published_date')
qs.order_by().first()
self.assertEqual(len(q.get_ops()), 1)
self.assertNotIn('$orderby', q.get_ops()[0]['query'])
self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
def test_no_ordering_for_get(self):
""" Ensure that Doc.objects.get doesn't use any ordering.
"""
MONGO_VER = self.mongodb_version
ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
class BlogPost(Document):
title = StringField()
published_date = DateTimeField()
@ -1320,13 +1331,13 @@ class QuerySetTest(unittest.TestCase):
with db_ops_tracker() as q:
BlogPost.objects.get(title='whatever')
self.assertEqual(len(q.get_ops()), 1)
self.assertNotIn('$orderby', q.get_ops()[0]['query'])
self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
# Ordering should be ignored for .get even if we set it explicitly
with db_ops_tracker() as q:
BlogPost.objects.order_by('-title').get(title='whatever')
self.assertEqual(len(q.get_ops()), 1)
self.assertNotIn('$orderby', q.get_ops()[0]['query'])
self.assertNotIn(ORDER_BY_KEY, q.get_ops()[0]['query'])
def test_find_embedded(self):
"""Ensure that an embedded document is properly returned from
@ -2450,7 +2461,11 @@ class QuerySetTest(unittest.TestCase):
self.assertEqual(names, ['User A', 'User B', 'User C'])
def test_comment(self):
"""Make sure adding a comment to the query works."""
"""Make sure adding a comment to the query gets added to the query"""
MONGO_VER = self.mongodb_version
QUERY_KEY = 'filter' if MONGO_VER == MONGODB_32 else '$query'
COMMENT_KEY = 'comment' if MONGO_VER == MONGODB_32 else '$comment'
class User(Document):
age = IntField()
@ -2466,8 +2481,8 @@ class QuerySetTest(unittest.TestCase):
ops = q.get_ops()
self.assertEqual(len(ops), 2)
for op in ops:
self.assertEqual(op['query']['$query'], {'age': {'$gte': 18}})
self.assertEqual(op['query']['$comment'], 'looking for an adult')
self.assertEqual(op['query'][QUERY_KEY], {'age': {'$gte': 18}})
self.assertEqual(op['query'][COMMENT_KEY], 'looking for an adult')
def test_map_reduce(self):
"""Ensure map/reduce is both mapping and reducing.
@ -4817,27 +4832,18 @@ class QuerySetTest(unittest.TestCase):
for i in range(100):
Person(name="No: %s" % i).save()
with query_counter() as q:
try:
self.assertEqual(q, 0)
people = Person.objects.no_cache()
with query_counter() as q:
self.assertEqual(q, 0)
people = Person.objects.no_cache()
[x for x in people]
self.assertEqual(q, 1)
[x for x in people]
self.assertEqual(q, 1)
list(people)
self.assertEqual(q, 2)
people.count()
self.assertEqual(q, 3)
except AssertionError as exc:
db = get_db()
msg = ''
for q in list(db.system.profile.find())[-50:]:
msg += str([q['ts'], q['ns'], q.get('query'), q['op']])+'\n'
msg += str(q)
raise AssertionError(str(exc) + '\n'+msg)
list(people)
self.assertEqual(q, 2)
people.count()
self.assertEqual(q, 3)
def test_cache_not_cloned(self):
@ -5111,35 +5117,39 @@ class QuerySetTest(unittest.TestCase):
self.assertEqual(op['nreturned'], 1)
def test_bool_with_ordering(self):
MONGO_VER = self.mongodb_version
ORDER_BY_KEY = 'sort' if MONGO_VER == MONGODB_32 else '$orderby'
class Person(Document):
name = StringField()
Person.drop_collection()
Person(name="Test").save()
# Check that bool(queryset) does not uses the orderby
qs = Person.objects.order_by('name')
with query_counter() as q:
if qs:
if bool(qs):
pass
op = q.db.system.profile.find({"ns":
{"$ne": "%s.system.indexes" % q.db.name}})[0]
self.assertNotIn('$orderby', op['query'],
'BaseQuerySet cannot use orderby in if stmt')
self.assertNotIn(ORDER_BY_KEY, op['query'])
# Check that normal query uses orderby
qs2 = Person.objects.order_by('name')
with query_counter() as p:
for x in qs:
for x in qs2:
pass
op = p.db.system.profile.find({"ns":
{"$ne": "%s.system.indexes" % q.db.name}})[0]
self.assertIn('$orderby', op['query'], 'BaseQuerySet cannot remove orderby in for loop')
self.assertIn(ORDER_BY_KEY, op['query'])
def test_bool_with_ordering_from_meta_dict(self):

1
tests/test_context_managers.py
View File

@ -299,7 +299,6 @@ class ContextManagersTest(unittest.TestCase):
cursor.close() # issues a `killcursors` query that is ignored by the context
self.assertEqual(q, 1)
_ = db.system.indexes.find_one() # queries on db.system.indexes are ignored as well
self.assertEqual(q, 1)

31
tests/utils.py
View File

@ -10,6 +10,13 @@ from mongoengine.python_support import IS_PYMONGO_3
MONGO_TEST_DB = 'mongoenginetest' # standard name for the test database
# Constant that can be used to compare the version retrieved with
# get_mongodb_version()
MONGODB_26 = (2, 6)
MONGODB_3 = (3,0)
MONGODB_32 = (3, 2)
class MongoDBTestCase(unittest.TestCase):
"""Base class for tests that need a mongodb connection
It ensures that the db is clean at the beginning and dropped at the end automatically
@ -27,24 +34,26 @@ class MongoDBTestCase(unittest.TestCase):
def get_mongodb_version():
"""Return the version tuple of the MongoDB server that the default
connection is connected to.
"""Return the version of the connected mongoDB (first 2 digits)
:return: tuple(int, int)
"""
return tuple(get_connection().server_info()['versionArray'])
version_list = get_connection().server_info()['versionArray'][:2] # e.g: (3, 2)
return tuple(version_list)
def _decorated_with_ver_requirement(func, ver_tuple):
def _decorated_with_ver_requirement(func, version):
"""Return a given function decorated with the version requirement
for a particular MongoDB version tuple.
:param version: The version required (tuple(int, int))
"""
def _inner(*args, **kwargs):
mongodb_ver = get_mongodb_version()
if mongodb_ver >= ver_tuple:
MONGODB_V = get_mongodb_version()
if MONGODB_V >= version:
return func(*args, **kwargs)
raise SkipTest('Needs MongoDB v{}+'.format(
'.'.join([str(v) for v in ver_tuple])
))
raise SkipTest('Needs MongoDB v{}+'.format('.'.join(str(n) for n in version)))
_inner.__name__ = func.__name__
_inner.__doc__ = func.__doc__
@ -56,14 +65,14 @@ def needs_mongodb_v26(func):
"""Raise a SkipTest exception if we're working with MongoDB version
lower than v2.6.
"""
return _decorated_with_ver_requirement(func, (2, 6))
return _decorated_with_ver_requirement(func, MONGODB_26)
def needs_mongodb_v3(func):
"""Raise a SkipTest exception if we're working with MongoDB version
lower than v3.0.
"""
return _decorated_with_ver_requirement(func, (3, 0))
return _decorated_with_ver_requirement(func, MONGODB_3)
def skip_pymongo3(f):