pub/mongoengine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:MongoEngine/mongoengine into v0.18.0_fix

Browse Source
This commit is contained in:
Bastien Gérard 2019-06-16 10:52:44 +02:00
commit baabf97acd
8 changed files with 313 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.install_mongodb_on_travis.sh
View File

@ -1,24 +0,0 @@
#!/bin/bash
sudo apt-get remove mongodb-org-server
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
if [ "$MONGODB" = "3.4" ]; then
sudo apt-key adv --keyserver keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list
sudo apt-get update
sudo apt-get install mongodb-org-server=3.4.17
# service should be started automatically
elif [ "$MONGODB" = "3.6" ]; then
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
sudo apt-get update
sudo apt-get install mongodb-org-server=3.6.12
# service should be started automatically
else
echo "Invalid MongoDB version, expected 2.6, 3.0, 3.2, 3.4 or 3.6."
exit 1
fi;
mkdir db
1>db/logs mongod --dbpath=db &

53
.travis.yml
View File

@ -16,52 +16,59 @@
#
# Reminder: Update README.rst if you change MongoDB versions we test.
language: python
language: python
python:
- 2.7
- 3.5
- 3.6
- pypy
dist: xenial
env:
- MONGODB=3.4 PYMONGO=3.x
global:
- MONGODB_3_4=3.4.17
- MONGODB_3_6=3.6.12
matrix:
- MONGODB=${MONGODB_3_4} PYMONGO=3.x
matrix:
# Finish the build as soon as one job fails
fast_finish: true
include:
- python: 2.7
env: MONGODB=3.4 PYMONGO=3.4.x
env: MONGODB=${MONGODB_3_4} PYMONGO=3.4.x
- python: 3.6
env: MONGODB=3.6 PYMONGO=3.x
env: MONGODB=${MONGODB_3_6} PYMONGO=3.x
- python: 3.7
env: MONGODB=${MONGODB_3_6} PYMONGO=3.x
before_install:
- bash .install_mongodb_on_travis.sh
- sleep 20 # https://docs.travis-ci.com/user/database-setup/#mongodb-does-not-immediately-accept-connections
- mongo --eval 'db.version();'
install:
- sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev
python-tk
- travis_retry pip install --upgrade pip
- travis_retry pip install coveralls
- travis_retry pip install flake8 flake8-import-order
- travis_retry pip install "tox" # tox 3.11.0 has requirement virtualenv>=14.0.0
- travis_retry pip install "virtualenv" # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
- travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
# Install Mongo
- wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB}.tgz
- tar xzf mongodb-linux-x86_64-${MONGODB}.tgz
- ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --version
# Install python dependencies
- pip install --upgrade pip
- pip install coveralls
- pip install flake8 flake8-import-order
- pip install tox # tox 3.11.0 has requirement virtualenv>=14.0.0
- pip install virtualenv # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
# Install the tox venv
- tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
# Cache dependencies installed via pip
cache: pip
# Run flake8 for py27
before_script:
- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi
- mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data
- ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi # Run flake8 for py27
- mongo --eval 'db.version();' # Make sure mongo is awake
script:
- tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
- tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
# For now only submit coveralls for Python v2.7. Python v3.x currently shows
# 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible

207
benchmark.py
View File

@ -1,207 +0,0 @@
#!/usr/bin/env python
"""
Simple benchmark comparing PyMongo and MongoEngine.
Sample run on a mid 2015 MacBook Pro (commit b282511):
Benchmarking...
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - Pymongo
2.58979988098
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - Pymongo write_concern={"w": 0}
1.26657605171
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - MongoEngine
8.4351580143
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries without continual assign - MongoEngine
7.20191693306
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True
6.31104588509
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True
6.07083487511
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False
5.97704291344
----------------------------------------------------------------------------------------------------
Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False
5.9111430645
"""
import timeit
def main():
print("Benchmarking...")
setup = """
from pymongo import MongoClient
connection = MongoClient()
connection.drop_database('timeit_test')
"""
stmt = """
from pymongo import MongoClient
connection = MongoClient()
db = connection.timeit_test
noddy = db.noddy
for i in range(10000):
example = {'fields': {}}
for j in range(20):
example['fields']['key' + str(j)] = 'value ' + str(j)
noddy.save(example)
myNoddys = noddy.find()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - Pymongo""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
from pymongo import MongoClient
from pymongo.write_concern import WriteConcern
connection = MongoClient()
db = connection.get_database('timeit_test', write_concern=WriteConcern(w=0))
noddy = db.noddy
for i in range(10000):
example = {'fields': {}}
for j in range(20):
example['fields']["key"+str(j)] = "value "+str(j)
noddy.save(example)
myNoddys = noddy.find()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
setup = """
from pymongo import MongoClient
connection = MongoClient()
connection.drop_database('timeit_test')
connection.close()
from mongoengine import Document, DictField, connect
connect('timeit_test')
class Noddy(Document):
fields = DictField()
"""
stmt = """
for i in range(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save()
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - MongoEngine""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
for i in range(10000):
noddy = Noddy()
fields = {}
for j in range(20):
fields["key"+str(j)] = "value "+str(j)
noddy.fields = fields
noddy.save()
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries without continual assign - MongoEngine""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
for i in range(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(write_concern={"w": 0}, cascade=True)
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
for i in range(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(write_concern={"w": 0}, validate=False, cascade=True)
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
for i in range(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(validate=False, write_concern={"w": 0})
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
stmt = """
for i in range(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(force_insert=True, write_concern={"w": 0}, validate=False)
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print("-" * 100)
print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""")
t = timeit.Timer(stmt=stmt, setup=setup)
print(t.timeit(1))
if __name__ == "__main__":
main()

112
benchmarks/test_basic_doc_ops.py Normal file
View File

@ -0,0 +1,112 @@
import unittest
from timeit import repeat
import mongoengine
from mongoengine import (BooleanField, Document, EmbeddedDocument,
EmbeddedDocumentField, IntField, ListField,
StringField)
conn_settings = {
'db': 'mongoengine-benchmark-test',
}
mongoengine.connect(**conn_settings)
def timeit(f, n=10000):
return min(repeat(f, repeat=3, number=n)) / float(n)
def test_basic():
class Book(Document):
name = StringField()
pages = IntField()
tags = ListField(StringField())
is_published = BooleanField()
Book.drop_collection()
def init_book():
return Book(
name='Always be closing',
pages=100,
tags=['self-help', 'sales'],
is_published=True,
)
print 'Doc initialization: %.3fus' % (timeit(init_book, 1000) * 10**6)
b = init_book()
print 'Doc getattr: %.3fus' % (timeit(lambda: b.name, 10000) * 10**6)
print 'Doc setattr: %.3fus' % (
timeit(lambda: setattr(b, 'name', 'New name'), 10000) * 10**6
)
print 'Doc to mongo: %.3fus' % (timeit(b.to_mongo, 1000) * 10**6)
def save_book():
b._mark_as_changed('name')
b._mark_as_changed('tags')
b.save()
save_book()
son = b.to_mongo()
print 'Load from SON: %.3fus' % (
timeit(lambda: Book._from_son(son), 1000) * 10**6
)
print 'Save to database: %.3fus' % (timeit(save_book, 100) * 10**6)
print 'Load from database: %.3fus' % (
timeit(lambda: Book.objects[0], 100) * 10**6
)
def test_embedded():
class Contact(EmbeddedDocument):
name = StringField()
title = StringField()
address = StringField()
class Company(Document):
name = StringField()
contacts = ListField(EmbeddedDocumentField(Contact))
Company.drop_collection()
def init_company():
return Company(
name='MongoDB, Inc.',
contacts=[
Contact(
name='Contact %d' % x,
title='CEO',
address='Address %d' % x,
)
for x in range(1000)
]
)
def create_company():
c = init_company()
c.save()
c.delete()
print 'Save/delete big object to database: %.3fms' % (
timeit(create_company, 10) * 10**3
)
c = init_company().save()
print 'Serialize big object: %.3fms' % (
timeit(c.to_mongo, 100) * 10**3
)
print 'Load big object from database: %.3fms' % (
timeit(lambda: Company.objects[0], 100) * 10**3
)
if __name__ == '__main__':
test_basic()
test_embedded()

154
benchmarks/test_inserts.py Normal file
View File

@ -0,0 +1,154 @@
import timeit
def main():
setup = """
from pymongo import MongoClient
connection = MongoClient()
connection.drop_database('timeit_test')
"""
stmt = """
from pymongo import MongoClient
connection = MongoClient()
db = connection.timeit_test
noddy = db.noddy
for i in xrange(10000):
example = {'fields': {}}
for j in range(20):
example['fields']["key"+str(j)] = "value "+str(j)
noddy.insert_one(example)
myNoddys = noddy.find()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print """Creating 10000 dictionaries - PyMongo"""
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
stmt = """
from pymongo import MongoClient, WriteConcern
connection = MongoClient()
db = connection.timeit_test
noddy = db.noddy.with_options(write_concern=WriteConcern(w=0))
for i in xrange(10000):
example = {'fields': {}}
for j in range(20):
example['fields']["key"+str(j)] = "value "+str(j)
noddy.insert_one(example)
myNoddys = noddy.find()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print """Creating 10000 dictionaries - PyMongo write_concern={"w": 0}"""
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
setup = """
from pymongo import MongoClient
connection = MongoClient()
connection.drop_database('timeit_test')
connection.close()
from mongoengine import Document, DictField, connect
connect("timeit_test")
class Noddy(Document):
fields = DictField()
"""
stmt = """
for i in xrange(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save()
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print "Creating 10000 dictionaries - MongoEngine"
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
stmt = """
for i in xrange(10000):
noddy = Noddy()
fields = {}
for j in range(20):
fields["key"+str(j)] = "value "+str(j)
noddy.fields = fields
noddy.save()
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print "Creating 10000 dictionaries without continual assign - MongoEngine"
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
stmt = """
for i in xrange(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(write_concern={"w": 0})
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}"""
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
stmt = """
for i in xrange(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(write_concern={"w": 0}, validate=False)
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False"""
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
stmt = """
for i in xrange(10000):
noddy = Noddy()
for j in range(20):
noddy.fields["key"+str(j)] = "value "+str(j)
noddy.save(force_insert=True, write_concern={"w": 0}, validate=False)
myNoddys = Noddy.objects()
[n for n in myNoddys] # iterate
"""
print "-" * 100
print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False"""
t = timeit.Timer(stmt=stmt, setup=setup)
print '{}s'.format(t.timeit(1))
if __name__ == "__main__":
main()

2
docs/changelog.rst
View File

@ -1,9 +1,11 @@
=========
Changelog
=========
Development
===========
- Add Python 3.7 in travis #2058
- (Fill this out as you fix issues and develop your features).
Changes in 0.18.0

9
mongoengine/document.py
View File

@ -182,8 +182,13 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)):
@classmethod
def _get_collection(cls):
"""Return the corresponding PyMongo collection of this document.
Upon the first call, it will ensure that indexes gets created. The returned collection then gets cached
"""Return the PyMongo collection corresponding to this document.
Upon first call, this method:
1. Initializes a :class:`~pymongo.collection.Collection` corresponding
to this document.
2. Creates indexes defined in this document's :attr:`meta` dictionary.
This happens only if `auto_create_index` is True.
"""
if not hasattr(cls, '_collection') or cls._collection is None:
# Get the collection, either capped or regular.

16
tests/queryset/queryset.py
View File

@ -7,7 +7,6 @@ from decimal import Decimal
from bson import DBRef, ObjectId
import pymongo
from pymongo.errors import ConfigurationError
from pymongo.read_preferences import ReadPreference
from pymongo.results import UpdateResult
import six
@ -17,7 +16,7 @@ from mongoengine import *
from mongoengine.connection import get_connection, get_db
from mongoengine.context_managers import query_counter, switch_db
from mongoengine.errors import InvalidQueryError
from mongoengine.mongodb_support import get_mongodb_version, MONGODB_36
from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned,
QuerySet, QuerySetManager, queryset_manager)
@ -832,8 +831,6 @@ class QuerySetTest(unittest.TestCase):
def test_bulk_insert(self):
"""Ensure that bulk insert works"""
MONGO_VER = self.mongodb_version
class Comment(EmbeddedDocument):
name = StringField()
@ -847,10 +844,6 @@ class QuerySetTest(unittest.TestCase):
Blog.drop_collection()
# get MongoDB version info
connection = get_connection()
info = connection.test.command('buildInfo')
# Recreates the collection
self.assertEqual(0, Blog.objects.count())
@ -5386,6 +5379,13 @@ class QuerySetTest(unittest.TestCase):
{'_id': None, 'avg': 29, 'total': 2}
])
data = Person.objects().aggregate({'$match': {'name': 'Isabella Luanna'}})
self.assertEqual(list(data), [
{u'_id': p1.pk,
u'age': 16,
u'name': u'Isabella Luanna'}]
)
def test_queryset_aggregation_with_skip(self):
class Person(Document):
name = StringField()