diff --git a/.install_mongodb_on_travis.sh b/.install_mongodb_on_travis.sh deleted file mode 100644 index 0be02655..00000000 --- a/.install_mongodb_on_travis.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -sudo apt-get remove mongodb-org-server -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 - -if [ "$MONGODB" = "3.4" ]; then - sudo apt-key adv --keyserver keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 - echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list - sudo apt-get update - sudo apt-get install mongodb-org-server=3.4.17 - # service should be started automatically -elif [ "$MONGODB" = "3.6" ]; then - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 - echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list - sudo apt-get update - sudo apt-get install mongodb-org-server=3.6.12 - # service should be started automatically -else - echo "Invalid MongoDB version, expected 2.6, 3.0, 3.2, 3.4 or 3.6." - exit 1 -fi; - -mkdir db -1>db/logs mongod --dbpath=db & diff --git a/.travis.yml b/.travis.yml index b1fcba07..7bbeef8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,52 +16,59 @@ # # Reminder: Update README.rst if you change MongoDB versions we test. -language: python +language: python python: - 2.7 - 3.5 - 3.6 - pypy +dist: xenial + env: -- MONGODB=3.4 PYMONGO=3.x + global: + - MONGODB_3_4=3.4.17 + - MONGODB_3_6=3.6.12 + matrix: + - MONGODB=${MONGODB_3_4} PYMONGO=3.x matrix: + # Finish the build as soon as one job fails fast_finish: true include: - python: 2.7 - env: MONGODB=3.4 PYMONGO=3.4.x + env: MONGODB=${MONGODB_3_4} PYMONGO=3.4.x - python: 3.6 - env: MONGODB=3.6 PYMONGO=3.x + env: MONGODB=${MONGODB_3_6} PYMONGO=3.x + - python: 3.7 + env: MONGODB=${MONGODB_3_6} PYMONGO=3.x -before_install: -- bash .install_mongodb_on_travis.sh -- sleep 20 # https://docs.travis-ci.com/user/database-setup/#mongodb-does-not-immediately-accept-connections -- mongo --eval 'db.version();' install: -- sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev - libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev - python-tk -- travis_retry pip install --upgrade pip -- travis_retry pip install coveralls -- travis_retry pip install flake8 flake8-import-order -- travis_retry pip install "tox" # tox 3.11.0 has requirement virtualenv>=14.0.0 -- travis_retry pip install "virtualenv" # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32) -- travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test + # Install Mongo + - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB}.tgz + - tar xzf mongodb-linux-x86_64-${MONGODB}.tgz + - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --version + # Install python dependencies + - pip install --upgrade pip + - pip install coveralls + - pip install flake8 flake8-import-order + - pip install tox # tox 3.11.0 has requirement virtualenv>=14.0.0 + - pip install virtualenv # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32) + # Install the tox venv + - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test -# Cache dependencies installed via pip -cache: pip - -# Run flake8 for py27 before_script: -- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi + - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data + - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork + - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi # Run flake8 for py27 + - mongo --eval 'db.version();' # Make sure mongo is awake script: -- tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage + - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage # For now only submit coveralls for Python v2.7. Python v3.x currently shows # 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible diff --git a/benchmark.py b/benchmark.py deleted file mode 100644 index 8e93ee40..00000000 --- a/benchmark.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python - -""" -Simple benchmark comparing PyMongo and MongoEngine. - -Sample run on a mid 2015 MacBook Pro (commit b282511): - -Benchmarking... ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - Pymongo -2.58979988098 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - Pymongo write_concern={"w": 0} -1.26657605171 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - MongoEngine -8.4351580143 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries without continual assign - MongoEngine -7.20191693306 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True -6.31104588509 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True -6.07083487511 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False -5.97704291344 ----------------------------------------------------------------------------------------------------- -Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False -5.9111430645 -""" - -import timeit - - -def main(): - print("Benchmarking...") - - setup = """ -from pymongo import MongoClient -connection = MongoClient() -connection.drop_database('timeit_test') -""" - - stmt = """ -from pymongo import MongoClient -connection = MongoClient() - -db = connection.timeit_test -noddy = db.noddy - -for i in range(10000): - example = {'fields': {}} - for j in range(20): - example['fields']['key' + str(j)] = 'value ' + str(j) - - noddy.save(example) - -myNoddys = noddy.find() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - Pymongo""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -from pymongo import MongoClient -from pymongo.write_concern import WriteConcern -connection = MongoClient() - -db = connection.get_database('timeit_test', write_concern=WriteConcern(w=0)) -noddy = db.noddy - -for i in range(10000): - example = {'fields': {}} - for j in range(20): - example['fields']["key"+str(j)] = "value "+str(j) - - noddy.save(example) - -myNoddys = noddy.find() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - setup = """ -from pymongo import MongoClient -connection = MongoClient() -connection.drop_database('timeit_test') -connection.close() - -from mongoengine import Document, DictField, connect -connect('timeit_test') - -class Noddy(Document): - fields = DictField() -""" - - stmt = """ -for i in range(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save() - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - MongoEngine""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -for i in range(10000): - noddy = Noddy() - fields = {} - for j in range(20): - fields["key"+str(j)] = "value "+str(j) - noddy.fields = fields - noddy.save() - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries without continual assign - MongoEngine""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -for i in range(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(write_concern={"w": 0}, cascade=True) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -for i in range(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(write_concern={"w": 0}, validate=False, cascade=True) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -for i in range(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(validate=False, write_concern={"w": 0}) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - stmt = """ -for i in range(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(force_insert=True, write_concern={"w": 0}, validate=False) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print("-" * 100) - print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""") - t = timeit.Timer(stmt=stmt, setup=setup) - print(t.timeit(1)) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/test_basic_doc_ops.py b/benchmarks/test_basic_doc_ops.py new file mode 100644 index 00000000..52dcab3c --- /dev/null +++ b/benchmarks/test_basic_doc_ops.py @@ -0,0 +1,112 @@ +import unittest +from timeit import repeat + +import mongoengine +from mongoengine import (BooleanField, Document, EmbeddedDocument, + EmbeddedDocumentField, IntField, ListField, + StringField) + +conn_settings = { + 'db': 'mongoengine-benchmark-test', +} + +mongoengine.connect(**conn_settings) + + +def timeit(f, n=10000): + return min(repeat(f, repeat=3, number=n)) / float(n) + + +def test_basic(): + class Book(Document): + name = StringField() + pages = IntField() + tags = ListField(StringField()) + is_published = BooleanField() + + Book.drop_collection() + + def init_book(): + return Book( + name='Always be closing', + pages=100, + tags=['self-help', 'sales'], + is_published=True, + ) + + print 'Doc initialization: %.3fus' % (timeit(init_book, 1000) * 10**6) + + b = init_book() + print 'Doc getattr: %.3fus' % (timeit(lambda: b.name, 10000) * 10**6) + + print 'Doc setattr: %.3fus' % ( + timeit(lambda: setattr(b, 'name', 'New name'), 10000) * 10**6 + ) + + print 'Doc to mongo: %.3fus' % (timeit(b.to_mongo, 1000) * 10**6) + + def save_book(): + b._mark_as_changed('name') + b._mark_as_changed('tags') + b.save() + + save_book() + son = b.to_mongo() + + print 'Load from SON: %.3fus' % ( + timeit(lambda: Book._from_son(son), 1000) * 10**6 + ) + + print 'Save to database: %.3fus' % (timeit(save_book, 100) * 10**6) + + print 'Load from database: %.3fus' % ( + timeit(lambda: Book.objects[0], 100) * 10**6 + ) + + +def test_embedded(): + class Contact(EmbeddedDocument): + name = StringField() + title = StringField() + address = StringField() + + class Company(Document): + name = StringField() + contacts = ListField(EmbeddedDocumentField(Contact)) + + Company.drop_collection() + + def init_company(): + return Company( + name='MongoDB, Inc.', + contacts=[ + Contact( + name='Contact %d' % x, + title='CEO', + address='Address %d' % x, + ) + for x in range(1000) + ] + ) + + def create_company(): + c = init_company() + c.save() + c.delete() + + print 'Save/delete big object to database: %.3fms' % ( + timeit(create_company, 10) * 10**3 + ) + + c = init_company().save() + print 'Serialize big object: %.3fms' % ( + timeit(c.to_mongo, 100) * 10**3 + ) + print 'Load big object from database: %.3fms' % ( + timeit(lambda: Company.objects[0], 100) * 10**3 + ) + + +if __name__ == '__main__': + test_basic() + test_embedded() diff --git a/benchmarks/test_inserts.py b/benchmarks/test_inserts.py new file mode 100644 index 00000000..29410ea5 --- /dev/null +++ b/benchmarks/test_inserts.py @@ -0,0 +1,154 @@ +import timeit + + +def main(): + setup = """ +from pymongo import MongoClient +connection = MongoClient() +connection.drop_database('timeit_test') +""" + + stmt = """ +from pymongo import MongoClient +connection = MongoClient() + +db = connection.timeit_test +noddy = db.noddy + +for i in xrange(10000): + example = {'fields': {}} + for j in range(20): + example['fields']["key"+str(j)] = "value "+str(j) + + noddy.insert_one(example) + +myNoddys = noddy.find() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - PyMongo""" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + stmt = """ +from pymongo import MongoClient, WriteConcern +connection = MongoClient() + +db = connection.timeit_test +noddy = db.noddy.with_options(write_concern=WriteConcern(w=0)) + +for i in xrange(10000): + example = {'fields': {}} + for j in range(20): + example['fields']["key"+str(j)] = "value "+str(j) + + noddy.insert_one(example) + +myNoddys = noddy.find() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - PyMongo write_concern={"w": 0}""" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + setup = """ +from pymongo import MongoClient +connection = MongoClient() +connection.drop_database('timeit_test') +connection.close() + +from mongoengine import Document, DictField, connect +connect("timeit_test") + +class Noddy(Document): + fields = DictField() +""" + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save() + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print "Creating 10000 dictionaries - MongoEngine" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + fields = {} + for j in range(20): + fields["key"+str(j)] = "value "+str(j) + noddy.fields = fields + noddy.save() + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print "Creating 10000 dictionaries without continual assign - MongoEngine" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(write_concern={"w": 0}) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}""" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(write_concern={"w": 0}, validate=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(force_insert=True, write_concern={"w": 0}, validate=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print '{}s'.format(t.timeit(1)) + + +if __name__ == "__main__": + main() diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bc2b67e..cdbb430e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,9 +1,11 @@ + ========= Changelog ========= Development =========== +- Add Python 3.7 in travis #2058 - (Fill this out as you fix issues and develop your features). Changes in 0.18.0 diff --git a/mongoengine/document.py b/mongoengine/document.py index c338263d..cc35c440 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -182,8 +182,13 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): @classmethod def _get_collection(cls): - """Return the corresponding PyMongo collection of this document. - Upon the first call, it will ensure that indexes gets created. The returned collection then gets cached + """Return the PyMongo collection corresponding to this document. + + Upon first call, this method: + 1. Initializes a :class:`~pymongo.collection.Collection` corresponding + to this document. + 2. Creates indexes defined in this document's :attr:`meta` dictionary. + This happens only if `auto_create_index` is True. """ if not hasattr(cls, '_collection') or cls._collection is None: # Get the collection, either capped or regular. diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 1f09ca59..55f256d9 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -7,7 +7,6 @@ from decimal import Decimal from bson import DBRef, ObjectId import pymongo -from pymongo.errors import ConfigurationError from pymongo.read_preferences import ReadPreference from pymongo.results import UpdateResult import six @@ -17,7 +16,7 @@ from mongoengine import * from mongoengine.connection import get_connection, get_db from mongoengine.context_managers import query_counter, switch_db from mongoengine.errors import InvalidQueryError -from mongoengine.mongodb_support import get_mongodb_version, MONGODB_36 +from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned, QuerySet, QuerySetManager, queryset_manager) @@ -832,8 +831,6 @@ class QuerySetTest(unittest.TestCase): def test_bulk_insert(self): """Ensure that bulk insert works""" - MONGO_VER = self.mongodb_version - class Comment(EmbeddedDocument): name = StringField() @@ -847,10 +844,6 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() - # get MongoDB version info - connection = get_connection() - info = connection.test.command('buildInfo') - # Recreates the collection self.assertEqual(0, Blog.objects.count()) @@ -5386,6 +5379,13 @@ class QuerySetTest(unittest.TestCase): {'_id': None, 'avg': 29, 'total': 2} ]) + data = Person.objects().aggregate({'$match': {'name': 'Isabella Luanna'}}) + self.assertEqual(list(data), [ + {u'_id': p1.pk, + u'age': 16, + u'name': u'Isabella Luanna'}] + ) + def test_queryset_aggregation_with_skip(self): class Person(Document): name = StringField()