clean up field unit tests

.gitignore

@@ -17,3 +17,4 @@ tests/test_bugfix.py
 htmlcov/
 venv
 venv3
+scratchpad

.install_mongodb_on_travis.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
+
+if [ "$MONGODB" = "2.4" ]; then
+    echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    sudo apt-get update
+    sudo apt-get install mongodb-10gen=2.4.14
+    sudo service mongodb start
+elif [ "$MONGODB" = "2.6" ]; then
+    echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    sudo apt-get update
+    sudo apt-get install mongodb-org-server=2.6.12
+    # service should be started automatically
+elif [ "$MONGODB" = "3.0" ]; then
+    echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
+    sudo apt-get update
+    sudo apt-get install mongodb-org-server=3.0.14
+    # service should be started automatically
+else
+    echo "Invalid MongoDB version, expected 2.4, 2.6, or 3.0."
+    exit 1
+fi;

.travis.yml

@@ -1,28 +1,48 @@
+# For full coverage, we'd have to test all supported Python, MongoDB, and
+# PyMongo combinations. However, that would result in an overly long build
+# with a very large number of jobs, hence we only test a subset of all the
+# combinations:
+# * MongoDB v2.4 & v3.0 are only tested against Python v2.7 & v3.5.
+# * MongoDB v2.4 is tested against PyMongo v2.7 & v3.x.
+# * MongoDB v3.0 is tested against PyMongo v3.x.
+# * MongoDB v2.6 is currently the "main" version tested against Python v2.7,
+#   v3.5, PyPy & PyPy3, and PyMongo v2.7, v2.8 & v3.x.
+#
+# Reminder: Update README.rst if you change MongoDB versions we test.
+
 language: python
 
 python:
-- '2.7'
+- 2.7
-- '3.3'
+- 3.5
-- '3.4'
-- '3.5'
 - pypy
 - pypy3
 
 env:
-- PYMONGO=2.7
+- MONGODB=2.6 PYMONGO=2.7
-- PYMONGO=2.8
+- MONGODB=2.6 PYMONGO=2.8
-- PYMONGO=3.0
+- MONGODB=2.6 PYMONGO=3.0
-- PYMONGO=dev
 
 matrix:
+  # Finish the build as soon as one job fails
   fast_finish: true
 
+  include:
+  - python: 2.7
+    env: MONGODB=2.4 PYMONGO=2.7
+  - python: 2.7
+    env: MONGODB=2.4 PYMONGO=3.0
+  - python: 2.7
+    env: MONGODB=3.0 PYMONGO=3.0
+  - python: 3.5
+    env: MONGODB=2.4 PYMONGO=2.7
+  - python: 3.5
+    env: MONGODB=2.4 PYMONGO=3.0
+  - python: 3.5
+    env: MONGODB=3.0 PYMONGO=3.0
+
 before_install:
-- travis_retry sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
+- bash .install_mongodb_on_travis.sh
-- echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' |
-  sudo tee /etc/apt/sources.list.d/mongodb.list
-- travis_retry sudo apt-get update
-- travis_retry sudo apt-get install mongodb-org-server
 
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
@@ -30,14 +50,17 @@ install:
   python-tk
 - travis_retry pip install --upgrade pip
 - travis_retry pip install coveralls
-- travis_retry pip install flake8
+- travis_retry pip install flake8 flake8-import-order
 - travis_retry pip install tox>=1.9
 - travis_retry pip install "virtualenv<14.0.0"  # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
 - travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
 
+# Cache dependencies installed via pip
+cache: pip
+
 # Run flake8 for py27
 before_script:
-- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then tox -e flake8; fi
+- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi
 
 script:
 - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
@@ -45,22 +68,34 @@ script:
 # For now only submit coveralls for Python v2.7. Python v3.x currently shows
 # 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible
 # code in a separate dir and runs tests on that.
-after_script:
+after_success:
 - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; fi
 
 notifications:
   irc: irc.freenode.org#mongoengine
 
+# Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
 branches:
   only:
   - master
   - /^v.*$/
 
+# Whenever a new release is created via GitHub, publish it on PyPI.
 deploy:
   provider: pypi
   user: the_drow
   password:
     secure: QMyatmWBnC6ZN3XLW2+fTBDU4LQcp1m/LjR2/0uamyeUzWKdlOoh/Wx5elOgLwt/8N9ppdPeG83ose1jOz69l5G0MUMjv8n/RIcMFSpCT59tGYqn3kh55b0cIZXFT9ar+5cxlif6a5rS72IHm5li7QQyxexJIII6Uxp0kpvUmek=
+
+  # create a source distribution and a pure python wheel for faster installs
+  distributions: "sdist bdist_wheel"
+
+  # only deploy on tagged commits (aka GitHub releases) and only for the
+  # parent repo's builds running Python 2.7 along with dev PyMongo (we run
+  # Travis against many different Python and PyMongo versions and we don't
+  # want the deploy to occur multiple times).
   on:
     tags: true
     repo: MongoEngine/mongoengine
+    condition: "$PYMONGO = 3.0"
+    python: 2.7

CONTRIBUTING.rst

@@ -29,19 +29,20 @@ Style Guide
 -----------
 
 MongoEngine aims to follow `PEP8 <http://www.python.org/dev/peps/pep-0008/>`_
-including 4 space indents. When possible we try to stick to 79 character line limits.
+including 4 space indents. When possible we try to stick to 79 character line
-However, screens got bigger and an ORM has a strong focus on readability and
+limits. However, screens got bigger and an ORM has a strong focus on
-if it can help, we accept 119 as maximum line length, in a similar way as
+readability and if it can help, we accept 119 as maximum line length, in a
-`django does <https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/#python-style>`_
+similar way as `django does
+<https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/#python-style>`_
 
 Testing
 -------
 
 All tests are run on `Travis <http://travis-ci.org/MongoEngine/mongoengine>`_
-and any pull requests are automatically tested by Travis. Any pull requests
+and any pull requests are automatically tested. Any pull requests without
-without tests will take longer to be integrated and might be refused.
+tests will take longer to be integrated and might be refused.
 
-You may also submit a simple failing test as a PullRequest if you don't know
+You may also submit a simple failing test as a pull request if you don't know
 how to fix it, it will be easier for other people to work on it and it may get
 fixed faster.
 
@@ -49,13 +50,18 @@ General Guidelines
 ------------------
 
 - Avoid backward breaking changes if at all possible.
+- If you *have* to introduce a breaking change, make it very clear in your
+  pull request's description. Also, describe how users of this package
+  should adapt to the breaking change in docs/upgrade.rst.
 - Write inline documentation for new classes and methods.
 - Write tests and make sure they pass (make sure you have a mongod
   running on the default port, then execute ``python setup.py nosetests``
   from the cmd line to run the test suite).
-- Ensure tests pass on every Python and PyMongo versions.
+- Ensure tests pass on all supported Python, PyMongo, and MongoDB versions.
-  You can test on these versions locally by executing ``tox``
+  You can test various Python and PyMongo versions locally by executing
-- Add enhancements or problematic bug fixes to docs/changelog.rst
+  ``tox``. For different MongoDB versions, you can rely on our automated
+  Travis tests.
+- Add enhancements or problematic bug fixes to docs/changelog.rst.
 - Add yourself to AUTHORS :)
 
 Documentation
@@ -69,3 +75,6 @@ just make your changes to the inline documentation of the appropriate
 branch and submit a `pull request <https://help.github.com/articles/using-pull-requests>`_.
 You might also use the github `Edit <https://github.com/blog/844-forking-with-the-edit-button>`_
 button.
+
+If you want to test your documentation changes locally, you need to install
+the ``sphinx`` package.

README.rst

@@ -19,23 +19,31 @@ MongoEngine
 About
 =====
 MongoEngine is a Python Object-Document Mapper for working with MongoDB.
-Documentation available at https://mongoengine-odm.readthedocs.io - there is currently
+Documentation is available at https://mongoengine-odm.readthedocs.io - there
-a `tutorial <https://mongoengine-odm.readthedocs.io/tutorial.html>`_, a `user guide
+is currently a `tutorial <https://mongoengine-odm.readthedocs.io/tutorial.html>`_,
-<https://mongoengine-odm.readthedocs.io/guide/index.html>`_ and an `API reference
+a `user guide <https://mongoengine-odm.readthedocs.io/guide/index.html>`_, and
-<https://mongoengine-odm.readthedocs.io/apireference.html>`_.
+an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
+
+Supported MongoDB Versions
+==========================
+MongoEngine is currently tested against MongoDB v2.4, v2.6, and v3.0. Future
+versions should be supported as well, but aren't actively tested at the moment.
+Make sure to open an issue or submit a pull request if you experience any
+problems with MongoDB v3.2+.
 
 Installation
 ============
 We recommend the use of `virtualenv <https://virtualenv.pypa.io/>`_ and of
 `pip <https://pip.pypa.io/>`_. You can then use ``pip install -U mongoengine``.
-You may also have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_ and thus
+You may also have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
-you can use ``easy_install -U mongoengine``. Otherwise, you can download the
+and thus you can use ``easy_install -U mongoengine``. Otherwise, you can download the
 source from `GitHub <http://github.com/MongoEngine/mongoengine>`_ and run ``python
 setup.py install``.
 
 Dependencies
 ============
-All of the dependencies can easily be installed via `pip <https://pip.pypa.io/>`_. At the very least, you'll need these two packages to use MongoEngine:
+All of the dependencies can easily be installed via `pip <https://pip.pypa.io/>`_.
+At the very least, you'll need these two packages to use MongoEngine:
 
 - pymongo>=2.7.1
 - six>=1.10.0
@@ -48,10 +56,6 @@ If you need to use an ``ImageField`` or ``ImageGridFsProxy``:
 
 - Pillow>=2.0.0
 
-If you want to generate the documentation (e.g. to contribute to it):
-
-- sphinx
-
 Examples
 ========
 Some simple examples of what MongoEngine code looks like:
@@ -110,11 +114,11 @@ Some simple examples of what MongoEngine code looks like:
 Tests
 =====
 To run the test suite, ensure you are running a local instance of MongoDB on
-the standard port and have ``nose`` installed. Then, run: ``python setup.py nosetests``.
+the standard port and have ``nose`` installed. Then, run ``python setup.py nosetests``.
 
-To run the test suite on every supported Python version and every supported PyMongo version,
+To run the test suite on every supported Python and PyMongo version, you can
-you can use ``tox``.
+use ``tox``. You'll need to make sure you have each supported Python version
-tox and each supported Python version should be installed in your environment:
+installed in your environment and then:
 
 .. code-block:: shell
 
@@ -123,13 +127,16 @@ tox and each supported Python version should be installed in your environment:
     # Run the test suites
     $ tox
 
-If you wish to run one single or selected tests, use the nosetest convention. It will find the folder,
+If you wish to run a subset of tests, use the nosetests convention:
-eventually the file, go to the TestClass specified after the colon and eventually right to the single test.
-Also use the -s argument if you want to print out whatever or access pdb while testing.
 
 .. code-block:: shell
 
-    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest.test_cls_field -s
+    # Run all the tests in a particular test file
+    $ python setup.py nosetests --tests tests/fields/fields.py
+    # Run only particular test class in that file
+    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest
+    # Use the -s option if you want to print some debug statements or use pdb
+    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest -s
 
 Community
 =========

docs/changelog.rst

@@ -4,7 +4,10 @@ Changelog
 
 Development
 ===========
-- (Fill this out as you fix issues and develop you features).
+- (Fill this out as you fix issues and develop your features).
+- Fixed using sets in field choices #1481
+- POTENTIAL BREAKING CHANGE: Fixed limit/skip/hint/batch_size chaining #1476
+- POTENTIAL BREAKING CHANGE: Changed a public `QuerySet.clone_into` method to a private `QuerySet._clone_into` #1476
 - Fixed connecting to a replica set with PyMongo 2.x #1436
 - Fixed an obscure error message when filtering by `field__in=non_iterable`. #1237
 

docs/guide/connecting.rst

@@ -43,12 +43,17 @@ the :attr:`host` to
     ``admin`` username and ``qwerty`` password.
 
 Replica Sets
-===========
+============
 
-MongoEngine supports
+MongoEngine supports connecting to replica sets::
-:class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`. To use them,
+
-please use an URI style connection and provide the ``replicaSet`` name
+    from mongoengine import connect
-in the connection kwargs.
+
+    # Regular connect
+    connect('dbname', replicaset='rs-name')
+
+    # MongoDB URI-style connect
+    connect(host='mongodb://localhost/dbname?replicaSet=rs-name')
 
 Read preferences are supported through the connection or via individual
 queries by passing the read_preference ::
@@ -59,76 +64,74 @@ queries by passing the read_preference ::
 Multiple Databases
 ==================
 
-Multiple database support was added in MongoEngine 0.6. To use multiple
+To use multiple databases you can use :func:`~mongoengine.connect` and provide
-databases you can use :func:`~mongoengine.connect` and provide an `alias` name
+an `alias` name for the connection - if no `alias` is provided then "default"
-for the connection - if no `alias` is provided then "default" is used.
+is used.
 
 In the background this uses :func:`~mongoengine.register_connection` to
 store the data and you can register all aliases up front if required.
 
 Individual documents can also support multiple databases by providing a
-`db_alias` in their meta data.  This allows :class:`~pymongo.dbref.DBRef` objects
+`db_alias` in their meta data. This allows :class:`~pymongo.dbref.DBRef`
-to point across databases and collections.  Below is an example schema, using
+objects to point across databases and collections. Below is an example schema,
-3 different databases to store data::
+using 3 different databases to store data::
 
         class User(Document):
             name = StringField()
 
-            meta = {"db_alias": "user-db"}
+            meta = {'db_alias': 'user-db'}
 
         class Book(Document):
             name = StringField()
 
-            meta = {"db_alias": "book-db"}
+            meta = {'db_alias': 'book-db'}
 
         class AuthorBooks(Document):
             author = ReferenceField(User)
             book = ReferenceField(Book)
 
-            meta = {"db_alias": "users-books-db"}
+            meta = {'db_alias': 'users-books-db'}
 
 
 Context Managers
 ================
-Sometimes you may want to switch the database or collection to query against
+Sometimes you may want to switch the database or collection to query against.
-for a class.
 For example, archiving older data into a separate database for performance
 reasons or writing functions that dynamically choose collections to write
-document to.
+a document to.
 
 Switch Database
 ---------------
 The :class:`~mongoengine.context_managers.switch_db` context manager allows
 you to change the database alias for a given class allowing quick and easy
-access the same User document across databases::
+access to the same User document across databases::
 
     from mongoengine.context_managers import switch_db
 
     class User(Document):
         name = StringField()
 
-        meta = {"db_alias": "user-db"}
+        meta = {'db_alias': 'user-db'}
 
     with switch_db(User, 'archive-user-db') as User:
-        User(name="Ross").save()  # Saves the 'archive-user-db'
+        User(name='Ross').save()  # Saves the 'archive-user-db'
 
 
 Switch Collection
 -----------------
 The :class:`~mongoengine.context_managers.switch_collection` context manager
 allows you to change the collection for a given class allowing quick and easy
-access the same Group document across collection::
+access to the same Group document across collection::
 
         from mongoengine.context_managers import switch_collection
 
         class Group(Document):
             name = StringField()
 
-        Group(name="test").save()  # Saves in the default db
+        Group(name='test').save()  # Saves in the default db
 
         with switch_collection(Group, 'group2000') as Group:
-            Group(name="hello Group 2000 collection!").save()  # Saves in group2000 collection
+            Group(name='hello Group 2000 collection!').save()  # Saves in group2000 collection
-
 
 
 .. note:: Make sure any aliases have been registered with

docs/guide/defining-documents.rst

@@ -150,7 +150,7 @@ arguments can be set on all fields:
     .. note:: If set, this field is also accessible through the `pk` field.
 
 :attr:`choices` (Default: None)
-    An iterable (e.g. a list or tuple) of choices to which the value of this
+    An iterable (e.g. list, tuple or set) of choices to which the value of this
     field should be limited.
 
     Can be either be a nested tuples of value (stored in mongo) and a

docs/guide/installing.rst

@@ -2,13 +2,13 @@
 Installing MongoEngine
 ======================
 
-To use MongoEngine, you will need to download `MongoDB <http://mongodb.org/>`_
+To use MongoEngine, you will need to download `MongoDB <http://mongodb.com/>`_
 and ensure it is running in an accessible location. You will also need
 `PyMongo <http://api.mongodb.org/python>`_ to use MongoEngine, but if you
 install MongoEngine using setuptools, then the dependencies will be handled for
 you.
 
-MongoEngine is available on PyPI, so to use it you can use :program:`pip`:
+MongoEngine is available on PyPI, so you can use :program:`pip`:
 
 .. code-block:: console
 

docs/tutorial.rst

@@ -3,11 +3,10 @@ Tutorial
 ========
 
 This tutorial introduces **MongoEngine** by means of example --- we will walk
-through how to create a simple **Tumblelog** application. A Tumblelog is a type
+through how to create a simple **Tumblelog** application. A tumblelog is a
-of blog where posts are not constrained to being conventional text-based posts.
+blog that supports mixed media content, including text, images, links, video,
-As well as text-based entries, users may post images, links, videos, etc. For
+audio, etc. For simplicity's sake, we'll stick to text, image, and link
-simplicity's sake, we'll stick to text, image and link entries in our
+entries. As the purpose of this tutorial is to introduce MongoEngine, we'll
-application. As the purpose of this tutorial is to introduce MongoEngine, we'll
 focus on the data-modelling side of the application, leaving out a user
 interface.
 
@@ -16,14 +15,14 @@ Getting started
 
 Before we start, make sure that a copy of MongoDB is running in an accessible
 location --- running it locally will be easier, but if that is not an option
-then it may be run on a remote server. If you haven't installed mongoengine,
+then it may be run on a remote server. If you haven't installed MongoEngine,
 simply use pip to install it like so::
 
     $ pip install mongoengine
 
 Before we can start using MongoEngine, we need to tell it how to connect to our
 instance of :program:`mongod`. For this we use the :func:`~mongoengine.connect`
-function. If running locally the only argument we need to provide is the name
+function. If running locally, the only argument we need to provide is the name
 of the MongoDB database to use::
 
     from mongoengine import *
@@ -39,8 +38,8 @@ Defining our documents
 MongoDB is *schemaless*, which means that no schema is enforced by the database
 --- we may add and remove fields however we want and MongoDB won't complain.
 This makes life a lot easier in many regards, especially when there is a change
-to the data model. However, defining schemata for our documents can help to
+to the data model. However, defining schemas for our documents can help to iron
-iron out bugs involving incorrect types or missing fields, and also allow us to
+out bugs involving incorrect types or missing fields, and also allow us to
 define utility methods on our documents in the same way that traditional
 :abbr:`ORMs (Object-Relational Mappers)` do.
 
@@ -96,7 +95,7 @@ using* the new fields we need to support video posts. This fits with the
 Object-Oriented principle of *inheritance* nicely. We can think of
 :class:`Post` as a base class, and :class:`TextPost`, :class:`ImagePost` and
 :class:`LinkPost` as subclasses of :class:`Post`. In fact, MongoEngine supports
-this kind of modelling out of the box --- all you need do is turn on inheritance
+this kind of modeling out of the box --- all you need do is turn on inheritance
 by setting :attr:`allow_inheritance` to True in the :attr:`meta`::
 
     class Post(Document):
@@ -128,8 +127,8 @@ link table, we can just store a list of tags in each post. So, for both
 efficiency and simplicity's sake, we'll store the tags as strings directly
 within the post, rather than storing references to tags in a separate
 collection. Especially as tags are generally very short (often even shorter
-than a document's id), this denormalisation won't impact very strongly on the
+than a document's id), this denormalization won't impact the size of the
-size of our database. So let's take a look that the code our modified
+database very strongly. Let's take a look at the code of our modified
 :class:`Post` class::
 
     class Post(Document):
@@ -141,7 +140,7 @@ The :class:`~mongoengine.fields.ListField` object that is used to define a Post'
 takes a field object as its first argument --- this means that you can have
 lists of any type of field (including lists).
 
-.. note:: We don't need to modify the specialised post types as they all
+.. note:: We don't need to modify the specialized post types as they all
     inherit from :class:`Post`.
 
 Comments
@@ -149,7 +148,7 @@ Comments
 
 A comment is typically associated with *one* post. In a relational database, to
 display a post with its comments, we would have to retrieve the post from the
-database, then query the database again for the comments associated with the
+database and then query the database again for the comments associated with the
 post. This works, but there is no real reason to be storing the comments
 separately from their associated posts, other than to work around the
 relational model. Using MongoDB we can store the comments as a list of
@@ -219,8 +218,8 @@ Now that we've got our user in the database, let's add a couple of posts::
     post2.tags = ['mongoengine']
     post2.save()
 
-.. note:: If you change a field on a object that has already been saved, then
+.. note:: If you change a field on an object that has already been saved and
-    call :meth:`save` again, the document will be updated.
+    then call :meth:`save` again, the document will be updated.
 
 Accessing our data
 ==================
@@ -232,17 +231,17 @@ used to access the documents in the database collection associated with that
 class. So let's see how we can get our posts' titles::
 
     for post in Post.objects:
-        print post.title
+        print(post.title)
 
 Retrieving type-specific information
 ------------------------------------
 
-This will print the titles of our posts, one on each line. But What if we want
+This will print the titles of our posts, one on each line. But what if we want
 to access the type-specific data (link_url, content, etc.)? One way is simply
 to use the :attr:`objects` attribute of a subclass of :class:`Post`::
 
     for post in TextPost.objects:
-        print post.content
+        print(post.content)
 
 Using TextPost's :attr:`objects` attribute only returns documents that were
 created using :class:`TextPost`. Actually, there is a more general rule here:
@@ -259,16 +258,14 @@ instances of :class:`Post` --- they were instances of the subclass of
 practice::
 
     for post in Post.objects:
-        print post.title
+        print(post.title)
-        print '=' * len(post.title)
+        print('=' * len(post.title))
 
         if isinstance(post, TextPost):
-            print post.content
+            print(post.content)
 
         if isinstance(post, LinkPost):
-            print 'Link:', post.link_url
+            print('Link: {}'.format(post.link_url))
-
-        print
 
 This would print the title of each post, followed by the content if it was a
 text post, and "Link: <url>" if it was a link post.
@@ -283,7 +280,7 @@ your query.  Let's adjust our query so that only posts with the tag "mongodb"
 are returned::
 
     for post in Post.objects(tags='mongodb'):
-        print post.title
+        print(post.title)
 
 There are also methods available on :class:`~mongoengine.queryset.QuerySet`
 objects that allow different results to be returned, for example, calling
@@ -292,11 +289,11 @@ the first matched by the query you provide. Aggregation functions may also be
 used on :class:`~mongoengine.queryset.QuerySet` objects::
 
     num_posts = Post.objects(tags='mongodb').count()
-    print 'Found %d posts with tag "mongodb"' % num_posts
+    print('Found {} posts with tag "mongodb"'.format(num_posts))
 
-Learning more about mongoengine
+Learning more about MongoEngine
 -------------------------------
 
 If you got this far you've made a great start, so well done! The next step on
-your mongoengine journey is the `full user guide <guide/index.html>`_, where you
+your MongoEngine journey is the `full user guide <guide/index.html>`_, where
-can learn indepth about how to use mongoengine and mongodb.
+you can learn in-depth about how to use MongoEngine and MongoDB.

docs/upgrade.rst

@@ -2,6 +2,20 @@
 Upgrading
 #########
 
+Development
+***********
+(Fill this out whenever you introduce breaking changes to MongoEngine)
+
+This release includes various fixes for the `BaseQuerySet` methods and how they
+are chained together. Since version 0.10.1 applying limit/skip/hint/batch_size
+to an already-existing queryset wouldn't modify the underlying PyMongo cursor.
+This has been fixed now, so you'll need to make sure that your code didn't rely
+on the broken implementation.
+
+Additionally, a public `BaseQuerySet.clone_into` has been renamed to a private
+`_clone_into`. If you directly used that method in your code, you'll need to
+rename its occurrences.
+
 0.11.0
 ******
 This release includes a major rehaul of MongoEngine's code quality and

mongoengine/base/fields.py

@@ -193,7 +193,8 @@ class BaseField(object):
         EmbeddedDocument = _import_class('EmbeddedDocument')
 
         choice_list = self.choices
-        if isinstance(choice_list[0], (list, tuple)):
+        if isinstance(next(iter(choice_list)), (list, tuple)):
+            # next(iter) is useful for sets
             choice_list = [k for k, _ in choice_list]
 
         # Choices which are other types of Documents

mongoengine/connection.py

@@ -51,7 +51,9 @@ def register_connection(alias, name=None, host=None, port=None,
         MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
     :param is_mock: explicitly use mongomock for this connection
         (can also be done by using `mongomock://` as db host prefix)
-    :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+        for example maxpoolsize, tz_aware, etc. See the documentation
+        for pymongo's `MongoClient` for a full list.
 
     .. versionchanged:: 0.10.6 - added mongomock support
     """
@@ -244,6 +246,9 @@ def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs):
     Multiple databases are supported by using aliases. Provide a separate
     `alias` to connect to a different instance of :program:`mongod`.
 
+    See the docstring for `register_connection` for more details about all
+    supported kwargs.
+
     .. versionchanged:: 0.6 - added multiple database support.
     """
     if alias not in _connections:

mongoengine/fields.py

@@ -139,12 +139,12 @@ class URLField(StringField):
         # Check first if the scheme is valid
         scheme = value.split('://')[0].lower()
         if scheme not in self.schemes:
-            self.error('Invalid scheme {} in URL: {}'.format(scheme, value))
+            self.error(u'Invalid scheme {} in URL: {}'.format(scheme, value))
             return
 
         # Then check full URL
         if not self.url_regex.match(value):
-            self.error('Invalid URL: {}'.format(value))
+            self.error(u'Invalid URL: {}'.format(value))
             return
 
 

mongoengine/queryset/base.py

@@ -286,7 +286,7 @@ class BaseQuerySet(object):
 
         .. versionadded:: 0.4
         """
-        return self._document(**kwargs).save()
+        return self._document(**kwargs).save(force_insert=True)
 
     def first(self):
         """Retrieve the first object matching the query."""

tests/document/indexes.py

@@ -2,14 +2,14 @@
 import unittest
 import sys
 
-
-import pymongo
-
 from nose.plugins.skip import SkipTest
 from datetime import datetime
+import pymongo
 
 from mongoengine import *
-from mongoengine.connection import get_db, get_connection
+from mongoengine.connection import get_db
+
+from tests.utils import get_mongodb_version, needs_mongodb_v26
 
 __all__ = ("IndexesTest", )
 
@@ -412,7 +412,6 @@ class IndexesTest(unittest.TestCase):
         User.ensure_indexes()
         info = User.objects._collection.index_information()
         self.assertEqual(sorted(info.keys()), ['_cls_1_user_guid_1', '_id_'])
-        User.drop_collection()
 
     def test_embedded_document_index(self):
         """Tests settings an index on an embedded document
@@ -434,7 +433,6 @@ class IndexesTest(unittest.TestCase):
 
         info = BlogPost.objects._collection.index_information()
         self.assertEqual(sorted(info.keys()), ['_id_', 'date.yr_-1'])
-        BlogPost.drop_collection()
 
     def test_list_embedded_document_index(self):
         """Ensure list embedded documents can be indexed
@@ -461,7 +459,6 @@ class IndexesTest(unittest.TestCase):
         post1 = BlogPost(title="Embedded Indexes tests in place",
                          tags=[Tag(name="about"), Tag(name="time")])
         post1.save()
-        BlogPost.drop_collection()
 
     def test_recursive_embedded_objects_dont_break_indexes(self):
 
@@ -494,8 +491,7 @@ class IndexesTest(unittest.TestCase):
         obj = Test(a=1)
         obj.save()
 
-        connection = get_connection()
+        IS_MONGODB_3 = get_mongodb_version()[0] >= 3
-        IS_MONGODB_3 = connection.server_info()['versionArray'][0] >= 3
 
         # Need to be explicit about covered indexes as mongoDB doesn't know if
         # the documents returned might have more keys in that here.
@@ -623,8 +619,6 @@ class IndexesTest(unittest.TestCase):
         post3 = BlogPost(title='test3', date=Date(year=2010), slug='test')
         self.assertRaises(OperationError, post3.save)
 
-        BlogPost.drop_collection()
-
     def test_unique_embedded_document(self):
         """Ensure that uniqueness constraints are applied to fields on embedded documents.
         """
@@ -652,8 +646,6 @@ class IndexesTest(unittest.TestCase):
                          sub=SubDocument(year=2010, slug='test'))
         self.assertRaises(NotUniqueError, post3.save)
 
-        BlogPost.drop_collection()
-
     def test_unique_embedded_document_in_list(self):
         """
         Ensure that the uniqueness constraints are applied to fields in
@@ -684,8 +676,6 @@ class IndexesTest(unittest.TestCase):
 
         self.assertRaises(NotUniqueError, post2.save)
 
-        BlogPost.drop_collection()
-
     def test_unique_with_embedded_document_and_embedded_unique(self):
         """Ensure that uniqueness constraints are applied to fields on
         embedded documents.  And work with unique_with as well.
@@ -719,8 +709,6 @@ class IndexesTest(unittest.TestCase):
                          sub=SubDocument(year=2009, slug='test-1'))
         self.assertRaises(NotUniqueError, post3.save)
 
-        BlogPost.drop_collection()
-
     def test_ttl_indexes(self):
 
         class Log(Document):
@@ -733,14 +721,6 @@ class IndexesTest(unittest.TestCase):
 
         Log.drop_collection()
 
-        if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3:
-            raise SkipTest('pymongo needs to be 2.3 or higher for this test')
-
-        connection = get_connection()
-        version_array = connection.server_info()['versionArray']
-        if version_array[0] < 2 and version_array[1] < 2:
-            raise SkipTest('MongoDB needs to be 2.2 or higher for this test')
-
         # Indexes are lazy so use list() to perform query
         list(Log.objects)
         info = Log.objects._collection.index_information()
@@ -768,13 +748,11 @@ class IndexesTest(unittest.TestCase):
             raise AssertionError("We saved a dupe!")
         except NotUniqueError:
             pass
-        Customer.drop_collection()
 
     def test_unique_and_primary(self):
         """If you set a field as primary, then unexpected behaviour can occur.
         You won't create a duplicate but you will update an existing document.
         """
-
         class User(Document):
             name = StringField(primary_key=True, unique=True)
             password = StringField()
@@ -790,8 +768,23 @@ class IndexesTest(unittest.TestCase):
         self.assertEqual(User.objects.count(), 1)
         self.assertEqual(User.objects.get().password, 'secret2')
 
+    def test_unique_and_primary_create(self):
+        """Create a new record with a duplicate primary key
+        throws an exception
+        """
+        class User(Document):
+            name = StringField(primary_key=True)
+            password = StringField()
+
         User.drop_collection()
 
+        User.objects.create(name='huangz', password='secret')
+        with self.assertRaises(NotUniqueError):
+            User.objects.create(name='huangz', password='secret2')
+
+        self.assertEqual(User.objects.count(), 1)
+        self.assertEqual(User.objects.get().password, 'secret')
+
     def test_index_with_pk(self):
         """Ensure you can use `pk` as part of a query"""
 
@@ -874,8 +867,8 @@ class IndexesTest(unittest.TestCase):
                          info['provider_ids.foo_1_provider_ids.bar_1']['key'])
         self.assertTrue(info['provider_ids.foo_1_provider_ids.bar_1']['sparse'])
 
+    @needs_mongodb_v26
     def test_text_indexes(self):
-
         class Book(Document):
             title = DictField()
             meta = {

tests/fields/file_tests.py

@@ -18,15 +18,13 @@ try:
 except ImportError:
     HAS_PIL = False
 
+from tests.utils import MongoDBTestCase
+
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png')
 TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), 'mongodb_leaf.png')
 
 
-class FileTest(unittest.TestCase):
+class FileTest(MongoDBTestCase):
-
-    def setUp(self):
-        connect(db='mongoenginetest')
-        self.db = get_db()
 
     def tearDown(self):
         self.db.drop_collection('fs.files')

tests/queryset/geo.py

@@ -1,105 +1,139 @@
-from datetime import datetime, timedelta
+import datetime
 import unittest
 
-from pymongo.errors import OperationFailure
 from mongoengine import *
-from mongoengine.connection import get_connection
+
-from nose.plugins.skip import SkipTest
+from tests.utils import MongoDBTestCase, needs_mongodb_v3
 
 
 __all__ = ("GeoQueriesTest",)
 
 
-class GeoQueriesTest(unittest.TestCase):
+class GeoQueriesTest(MongoDBTestCase):
 
-    def setUp(self):
+    def _create_event_data(self, point_field_class=GeoPointField):
-        connect(db='mongoenginetest')
+        """Create some sample data re-used in many of the tests below."""
-
-    def test_geospatial_operators(self):
-        """Ensure that geospatial queries are working.
-        """
         class Event(Document):
             title = StringField()
             date = DateTimeField()
-            location = GeoPointField()
+            location = point_field_class()
 
             def __unicode__(self):
                 return self.title
 
+        self.Event = Event
+
         Event.drop_collection()
 
-        event1 = Event(title="Coltrane Motion @ Double Door",
+        event1 = Event.objects.create(
-                       date=datetime.now() - timedelta(days=1),
+            title="Coltrane Motion @ Double Door",
-                       location=[-87.677137, 41.909889]).save()
+            date=datetime.datetime.now() - datetime.timedelta(days=1),
-        event2 = Event(title="Coltrane Motion @ Bottom of the Hill",
+            location=[-87.677137, 41.909889])
-                       date=datetime.now() - timedelta(days=10),
+        event2 = Event.objects.create(
-                       location=[-122.4194155, 37.7749295]).save()
+            title="Coltrane Motion @ Bottom of the Hill",
-        event3 = Event(title="Coltrane Motion @ Empty Bottle",
+            date=datetime.datetime.now() - datetime.timedelta(days=10),
-                       date=datetime.now(),
+            location=[-122.4194155, 37.7749295])
-                       location=[-87.686638, 41.900474]).save()
+        event3 = Event.objects.create(
+            title="Coltrane Motion @ Empty Bottle",
+            date=datetime.datetime.now(),
+            location=[-87.686638, 41.900474])
+
+        return event1, event2, event3
+
+    def test_near(self):
+        """Make sure the "near" operator works."""
+        event1, event2, event3 = self._create_event_data()
 
         # find all events "near" pitchfork office, chicago.
         # note that "near" will show the san francisco event, too,
         # although it sorts to last.
-        events = Event.objects(location__near=[-87.67892, 41.9120459])
+        events = self.Event.objects(location__near=[-87.67892, 41.9120459])
         self.assertEqual(events.count(), 3)
         self.assertEqual(list(events), [event1, event3, event2])
 
+        # ensure ordering is respected by "near"
+        events = self.Event.objects(location__near=[-87.67892, 41.9120459])
+        events = events.order_by("-date")
+        self.assertEqual(events.count(), 3)
+        self.assertEqual(list(events), [event3, event1, event2])
+
+    def test_near_and_max_distance(self):
+        """Ensure the "max_distance" operator works alongside the "near"
+        operator.
+        """
+        event1, event2, event3 = self._create_event_data()
+
+        # find events within 10 degrees of san francisco
+        point = [-122.415579, 37.7566023]
+        events = self.Event.objects(location__near=point,
+                                    location__max_distance=10)
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event2)
+
+    # $minDistance was added in MongoDB v2.6, but continued being buggy
+    # until v3.0; skip for older versions
+    @needs_mongodb_v3
+    def test_near_and_min_distance(self):
+        """Ensure the "min_distance" operator works alongside the "near"
+        operator.
+        """
+        event1, event2, event3 = self._create_event_data()
+
+        # find events at least 10 degrees away of san francisco
+        point = [-122.415579, 37.7566023]
+        events = self.Event.objects(location__near=point,
+                                    location__min_distance=10)
+        self.assertEqual(events.count(), 2)
+
+    def test_within_distance(self):
+        """Make sure the "within_distance" operator works."""
+        event1, event2, event3 = self._create_event_data()
+
         # find events within 5 degrees of pitchfork office, chicago
         point_and_distance = [[-87.67892, 41.9120459], 5]
-        events = Event.objects(location__within_distance=point_and_distance)
+        events = self.Event.objects(
+            location__within_distance=point_and_distance)
         self.assertEqual(events.count(), 2)
         events = list(events)
         self.assertTrue(event2 not in events)
         self.assertTrue(event1 in events)
         self.assertTrue(event3 in events)
 
-        # ensure ordering is respected by "near"
-        events = Event.objects(location__near=[-87.67892, 41.9120459])
-        events = events.order_by("-date")
-        self.assertEqual(events.count(), 3)
-        self.assertEqual(list(events), [event3, event1, event2])
-
-        # find events within 10 degrees of san francisco
-        point = [-122.415579, 37.7566023]
-        events = Event.objects(location__near=point, location__max_distance=10)
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0], event2)
-
-        # find events at least 10 degrees away of san francisco
-        point = [-122.415579, 37.7566023]
-        events = Event.objects(location__near=point, location__min_distance=10)
-        # The following real test passes on MongoDB 3 but minDistance seems
-        # buggy on older MongoDB versions
-        if get_connection().server_info()['versionArray'][0] > 2:
-            self.assertEqual(events.count(), 2)
-        else:
-            self.assertTrue(events.count() >= 2)
-
         # find events within 10 degrees of san francisco
         point_and_distance = [[-122.415579, 37.7566023], 10]
-        events = Event.objects(location__within_distance=point_and_distance)
+        events = self.Event.objects(
+            location__within_distance=point_and_distance)
         self.assertEqual(events.count(), 1)
         self.assertEqual(events[0], event2)
 
         # find events within 1 degree of greenpoint, broolyn, nyc, ny
         point_and_distance = [[-73.9509714, 40.7237134], 1]
-        events = Event.objects(location__within_distance=point_and_distance)
+        events = self.Event.objects(
+            location__within_distance=point_and_distance)
         self.assertEqual(events.count(), 0)
 
         # ensure ordering is respected by "within_distance"
         point_and_distance = [[-87.67892, 41.9120459], 10]
-        events = Event.objects(location__within_distance=point_and_distance)
+        events = self.Event.objects(
+            location__within_distance=point_and_distance)
         events = events.order_by("-date")
         self.assertEqual(events.count(), 2)
         self.assertEqual(events[0], event3)
 
+    def test_within_box(self):
+        """Ensure the "within_box" operator works."""
+        event1, event2, event3 = self._create_event_data()
+
         # check that within_box works
         box = [(-125.0, 35.0), (-100.0, 40.0)]
-        events = Event.objects(location__within_box=box)
+        events = self.Event.objects(location__within_box=box)
         self.assertEqual(events.count(), 1)
         self.assertEqual(events[0].id, event2.id)
 
+    def test_within_polygon(self):
+        """Ensure the "within_polygon" operator works."""
+        event1, event2, event3 = self._create_event_data()
+
         polygon = [
             (-87.694445, 41.912114),
             (-87.69084, 41.919395),
@@ -107,7 +141,7 @@ class GeoQueriesTest(unittest.TestCase):
             (-87.654276, 41.911731),
             (-87.656164, 41.898061),
         ]
-        events = Event.objects(location__within_polygon=polygon)
+        events = self.Event.objects(location__within_polygon=polygon)
         self.assertEqual(events.count(), 1)
         self.assertEqual(events[0].id, event1.id)
 
@@ -116,13 +150,151 @@ class GeoQueriesTest(unittest.TestCase):
             (-1.225891, 52.792797),
             (-4.40094, 53.389881)
         ]
-        events = Event.objects(location__within_polygon=polygon2)
+        events = self.Event.objects(location__within_polygon=polygon2)
         self.assertEqual(events.count(), 0)
 
-    def test_geo_spatial_embedded(self):
+    def test_2dsphere_near(self):
+        """Make sure the "near" operator works with a PointField, which
+        corresponds to a 2dsphere index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
 
+        # find all events "near" pitchfork office, chicago.
+        # note that "near" will show the san francisco event, too,
+        # although it sorts to last.
+        events = self.Event.objects(location__near=[-87.67892, 41.9120459])
+        self.assertEqual(events.count(), 3)
+        self.assertEqual(list(events), [event1, event3, event2])
+
+        # ensure ordering is respected by "near"
+        events = self.Event.objects(location__near=[-87.67892, 41.9120459])
+        events = events.order_by("-date")
+        self.assertEqual(events.count(), 3)
+        self.assertEqual(list(events), [event3, event1, event2])
+
+    def test_2dsphere_near_and_max_distance(self):
+        """Ensure the "max_distance" operator works alongside the "near"
+        operator with a 2dsphere index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
+
+        # find events within 10km of san francisco
+        point = [-122.415579, 37.7566023]
+        events = self.Event.objects(location__near=point,
+                                    location__max_distance=10000)
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event2)
+
+        # find events within 1km of greenpoint, broolyn, nyc, ny
+        events = self.Event.objects(location__near=[-73.9509714, 40.7237134],
+                                    location__max_distance=1000)
+        self.assertEqual(events.count(), 0)
+
+        # ensure ordering is respected by "near"
+        events = self.Event.objects(
+            location__near=[-87.67892, 41.9120459],
+            location__max_distance=10000
+        ).order_by("-date")
+        self.assertEqual(events.count(), 2)
+        self.assertEqual(events[0], event3)
+
+    def test_2dsphere_geo_within_box(self):
+        """Ensure the "geo_within_box" operator works with a 2dsphere
+        index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
+
+        # check that within_box works
+        box = [(-125.0, 35.0), (-100.0, 40.0)]
+        events = self.Event.objects(location__geo_within_box=box)
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0].id, event2.id)
+
+    def test_2dsphere_geo_within_polygon(self):
+        """Ensure the "geo_within_polygon" operator works with a
+        2dsphere index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
+
+        polygon = [
+            (-87.694445, 41.912114),
+            (-87.69084, 41.919395),
+            (-87.681742, 41.927186),
+            (-87.654276, 41.911731),
+            (-87.656164, 41.898061),
+        ]
+        events = self.Event.objects(location__geo_within_polygon=polygon)
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0].id, event1.id)
+
+        polygon2 = [
+            (-1.742249, 54.033586),
+            (-1.225891, 52.792797),
+            (-4.40094, 53.389881)
+        ]
+        events = self.Event.objects(location__geo_within_polygon=polygon2)
+        self.assertEqual(events.count(), 0)
+
+    # $minDistance was added in MongoDB v2.6, but continued being buggy
+    # until v3.0; skip for older versions
+    @needs_mongodb_v3
+    def test_2dsphere_near_and_min_max_distance(self):
+        """Ensure "min_distace" and "max_distance" operators work well
+        together with the "near" operator in a 2dsphere index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
+
+        # ensure min_distance and max_distance combine well
+        events = self.Event.objects(
+            location__near=[-87.67892, 41.9120459],
+            location__min_distance=1000,
+            location__max_distance=10000
+        ).order_by("-date")
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event3)
+
+        # ensure ordering is respected by "near" with "min_distance"
+        events = self.Event.objects(
+            location__near=[-87.67892, 41.9120459],
+            location__min_distance=10000
+        ).order_by("-date")
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event2)
+
+    def test_2dsphere_geo_within_center(self):
+        """Make sure the "geo_within_center" operator works with a
+        2dsphere index.
+        """
+        event1, event2, event3 = self._create_event_data(
+            point_field_class=PointField
+        )
+
+        # find events within 5 degrees of pitchfork office, chicago
+        point_and_distance = [[-87.67892, 41.9120459], 2]
+        events = self.Event.objects(
+            location__geo_within_center=point_and_distance)
+        self.assertEqual(events.count(), 2)
+        events = list(events)
+        self.assertTrue(event2 not in events)
+        self.assertTrue(event1 in events)
+        self.assertTrue(event3 in events)
+
+    def _test_embedded(self, point_field_class):
+        """Helper test method ensuring given point field class works
+        well in an embedded document.
+        """
         class Venue(EmbeddedDocument):
-            location = GeoPointField()
+            location = point_field_class()
             name = StringField()
 
         class Event(Document):
@@ -148,16 +320,18 @@ class GeoQueriesTest(unittest.TestCase):
         self.assertEqual(events.count(), 3)
         self.assertEqual(list(events), [event1, event3, event2])
 
-    def test_spherical_geospatial_operators(self):
+    def test_geo_spatial_embedded(self):
-        """Ensure that spherical geospatial queries are working
+        """Make sure GeoPointField works properly in an embedded document."""
-        """
+        self._test_embedded(point_field_class=GeoPointField)
-        # Needs MongoDB > 2.6.4 https://jira.mongodb.org/browse/SERVER-14039
-        connection = get_connection()
-        info = connection.test.command('buildInfo')
-        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
-        if mongodb_version < (2, 6, 4):
-            raise SkipTest("Need MongoDB version 2.6.4+")
 
+    def test_2dsphere_point_embedded(self):
+        """Make sure PointField works properly in an embedded document."""
+        self._test_embedded(point_field_class=PointField)
+
+    # Needs MongoDB > 2.6.4 https://jira.mongodb.org/browse/SERVER-14039
+    @needs_mongodb_v3
+    def test_spherical_geospatial_operators(self):
+        """Ensure that spherical geospatial queries are working."""
         class Point(Document):
             location = GeoPointField()
 
@@ -177,7 +351,10 @@ class GeoQueriesTest(unittest.TestCase):
 
         # Same behavior for _within_spherical_distance
         points = Point.objects(
-            location__within_spherical_distance=[[-122, 37.5], 60 / earth_radius]
+            location__within_spherical_distance=[
+                [-122, 37.5],
+                60 / earth_radius
+            ]
         )
         self.assertEqual(points.count(), 2)
 
@@ -194,14 +371,9 @@ class GeoQueriesTest(unittest.TestCase):
         # Test query works with min_distance, being farer from one point
         points = Point.objects(location__near_sphere=[-122, 37.8],
                                location__min_distance=60 / earth_radius)
-        # The following real test passes on MongoDB 3 but minDistance seems
-        # buggy on older MongoDB versions
-        if get_connection().server_info()['versionArray'][0] > 2:
         self.assertEqual(points.count(), 1)
         far_point = points.first()
         self.assertNotEqual(close_point, far_point)
-        else:
-            self.assertTrue(points.count() >= 1)
 
         # Finds both points, but orders the north point first because it's
         # closer to the reference point to the north.
@@ -220,141 +392,15 @@ class GeoQueriesTest(unittest.TestCase):
         # Finds only one point because only the first point is within 60km of
         # the reference point to the south.
         points = Point.objects(
-            location__within_spherical_distance=[[-122, 36.5], 60/earth_radius])
+            location__within_spherical_distance=[
+                [-122, 36.5],
+                60 / earth_radius
+            ]
+        )
         self.assertEqual(points.count(), 1)
         self.assertEqual(points[0].id, south_point.id)
 
-    def test_2dsphere_point(self):
-
-        class Event(Document):
-            title = StringField()
-            date = DateTimeField()
-            location = PointField()
-
-            def __unicode__(self):
-                return self.title
-
-        Event.drop_collection()
-
-        event1 = Event(title="Coltrane Motion @ Double Door",
-                       date=datetime.now() - timedelta(days=1),
-                       location=[-87.677137, 41.909889])
-        event1.save()
-        event2 = Event(title="Coltrane Motion @ Bottom of the Hill",
-                       date=datetime.now() - timedelta(days=10),
-                       location=[-122.4194155, 37.7749295]).save()
-        event3 = Event(title="Coltrane Motion @ Empty Bottle",
-                       date=datetime.now(),
-                       location=[-87.686638, 41.900474]).save()
-
-        # find all events "near" pitchfork office, chicago.
-        # note that "near" will show the san francisco event, too,
-        # although it sorts to last.
-        events = Event.objects(location__near=[-87.67892, 41.9120459])
-        self.assertEqual(events.count(), 3)
-        self.assertEqual(list(events), [event1, event3, event2])
-
-        # find events within 5 degrees of pitchfork office, chicago
-        point_and_distance = [[-87.67892, 41.9120459], 2]
-        events = Event.objects(location__geo_within_center=point_and_distance)
-        self.assertEqual(events.count(), 2)
-        events = list(events)
-        self.assertTrue(event2 not in events)
-        self.assertTrue(event1 in events)
-        self.assertTrue(event3 in events)
-
-        # ensure ordering is respected by "near"
-        events = Event.objects(location__near=[-87.67892, 41.9120459])
-        events = events.order_by("-date")
-        self.assertEqual(events.count(), 3)
-        self.assertEqual(list(events), [event3, event1, event2])
-
-        # find events within 10km of san francisco
-        point = [-122.415579, 37.7566023]
-        events = Event.objects(location__near=point, location__max_distance=10000)
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0], event2)
-
-        # find events within 1km of greenpoint, broolyn, nyc, ny
-        events = Event.objects(location__near=[-73.9509714, 40.7237134], location__max_distance=1000)
-        self.assertEqual(events.count(), 0)
-
-        # ensure ordering is respected by "near"
-        events = Event.objects(location__near=[-87.67892, 41.9120459],
-                               location__max_distance=10000).order_by("-date")
-        self.assertEqual(events.count(), 2)
-        self.assertEqual(events[0], event3)
-
-        # ensure min_distance and max_distance combine well
-        events = Event.objects(location__near=[-87.67892, 41.9120459],
-                               location__min_distance=1000,
-                               location__max_distance=10000).order_by("-date")
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0], event3)
-
-        # ensure ordering is respected by "near"
-        events = Event.objects(location__near=[-87.67892, 41.9120459],
-                               # location__min_distance=10000
-                               location__min_distance=10000).order_by("-date")
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0], event2)
-
-        # check that within_box works
-        box = [(-125.0, 35.0), (-100.0, 40.0)]
-        events = Event.objects(location__geo_within_box=box)
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0].id, event2.id)
-
-        polygon = [
-            (-87.694445, 41.912114),
-            (-87.69084, 41.919395),
-            (-87.681742, 41.927186),
-            (-87.654276, 41.911731),
-            (-87.656164, 41.898061),
-        ]
-        events = Event.objects(location__geo_within_polygon=polygon)
-        self.assertEqual(events.count(), 1)
-        self.assertEqual(events[0].id, event1.id)
-
-        polygon2 = [
-            (-1.742249, 54.033586),
-            (-1.225891, 52.792797),
-            (-4.40094, 53.389881)
-        ]
-        events = Event.objects(location__geo_within_polygon=polygon2)
-        self.assertEqual(events.count(), 0)
-
-    def test_2dsphere_point_embedded(self):
-
-        class Venue(EmbeddedDocument):
-            location = GeoPointField()
-            name = StringField()
-
-        class Event(Document):
-            title = StringField()
-            venue = EmbeddedDocumentField(Venue)
-
-        Event.drop_collection()
-
-        venue1 = Venue(name="The Rock", location=[-87.677137, 41.909889])
-        venue2 = Venue(name="The Bridge", location=[-122.4194155, 37.7749295])
-
-        event1 = Event(title="Coltrane Motion @ Double Door",
-                       venue=venue1).save()
-        event2 = Event(title="Coltrane Motion @ Bottom of the Hill",
-                       venue=venue2).save()
-        event3 = Event(title="Coltrane Motion @ Empty Bottle",
-                       venue=venue1).save()
-
-        # find all events "near" pitchfork office, chicago.
-        # note that "near" will show the san francisco event, too,
-        # although it sorts to last.
-        events = Event.objects(venue__location__near=[-87.67892, 41.9120459])
-        self.assertEqual(events.count(), 3)
-        self.assertEqual(list(events), [event1, event3, event2])
-
     def test_linestring(self):
-
         class Road(Document):
             name = StringField()
             line = LineStringField()
@@ -410,7 +456,6 @@ class GeoQueriesTest(unittest.TestCase):
         self.assertEqual(1, roads)
 
     def test_polygon(self):
-
         class Road(Document):
             name = StringField()
             poly = PolygonField()
@@ -507,5 +552,6 @@ class GeoQueriesTest(unittest.TestCase):
         loc = Location.objects.as_pymongo()[0]
         self.assertEqual(loc["poly"], {"type": "Polygon", "coordinates": [[[40, 4], [40, 6], [41, 6], [40, 4]]]})
 
+
 if __name__ == '__main__':
     unittest.main()

tests/queryset/queryset.py

@@ -19,6 +19,9 @@ from mongoengine.python_support import IS_PYMONGO_3
 from mongoengine.queryset import (DoesNotExist, MultipleObjectsReturned,
                                   QuerySet, QuerySetManager, queryset_manager)
 
+from tests.utils import needs_mongodb_v26, skip_pymongo3
+
+
 __all__ = ("QuerySetTest",)
 
 
@@ -32,37 +35,6 @@ class db_ops_tracker(query_counter):
         return list(self.db.system.profile.find(ignore_query))
 
 
-def skip_older_mongodb(f):
-    def _inner(*args, **kwargs):
-        connection = get_connection()
-        info = connection.test.command('buildInfo')
-        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
-
-        if mongodb_version < (2, 6):
-            raise SkipTest("Need MongoDB version 2.6+")
-
-        return f(*args, **kwargs)
-
-    _inner.__name__ = f.__name__
-    _inner.__doc__ = f.__doc__
-
-    return _inner
-
-
-def skip_pymongo3(f):
-    def _inner(*args, **kwargs):
-
-        if IS_PYMONGO_3:
-            raise SkipTest("Useless with PyMongo 3+")
-
-        return f(*args, **kwargs)
-
-    _inner.__name__ = f.__name__
-    _inner.__doc__ = f.__doc__
-
-    return _inner
-
-
 class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
@@ -599,16 +571,23 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(post.comments[0].by, 'joe')
         self.assertEqual(post.comments[0].votes.score, 4)
 
+    @needs_mongodb_v26
     def test_update_min_max(self):
         class Scores(Document):
             high_score = IntField()
             low_score = IntField()
-        scores = Scores(high_score=800, low_score=200)
+
-        scores.save()
+        scores = Scores.objects.create(high_score=800, low_score=200)
+
         Scores.objects(id=scores.id).update(min__low_score=150)
-        self.assertEqual(Scores.objects(id=scores.id).get().low_score, 150)
+        self.assertEqual(Scores.objects.get(id=scores.id).low_score, 150)
         Scores.objects(id=scores.id).update(min__low_score=250)
-        self.assertEqual(Scores.objects(id=scores.id).get().low_score, 150)
+        self.assertEqual(Scores.objects.get(id=scores.id).low_score, 150)
+
+        Scores.objects(id=scores.id).update(max__high_score=1000)
+        self.assertEqual(Scores.objects.get(id=scores.id).high_score, 1000)
+        Scores.objects(id=scores.id).update(max__high_score=500)
+        self.assertEqual(Scores.objects.get(id=scores.id).high_score, 1000)
 
     def test_updates_can_have_match_operators(self):
 
@@ -1012,7 +991,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(person.name, "User A")
         self.assertEqual(person.age, 20)
 
-    @skip_older_mongodb
+    @needs_mongodb_v26
     @skip_pymongo3
     def test_cursor_args(self):
         """Ensures the cursor args can be set as expected
@@ -3129,7 +3108,7 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(Foo.objects.distinct("bar"), [bar])
 
-    @skip_older_mongodb
+    @needs_mongodb_v26
     def test_text_indexes(self):
         class News(Document):
             title = StringField()
@@ -3216,7 +3195,7 @@ class QuerySetTest(unittest.TestCase):
             'brasil').order_by('$text_score').first()
         self.assertEqual(item.get_text_score(), max_text_score)
 
-    @skip_older_mongodb
+    @needs_mongodb_v26
     def test_distinct_handles_references_to_alias(self):
         register_connection('testdb', 'mongoenginetest2')
 
@@ -4891,6 +4870,7 @@ class QuerySetTest(unittest.TestCase):
             self.assertTrue(Person.objects._has_data(),
                             'Cursor has data and returned False')
 
+    @needs_mongodb_v26
     def test_queryset_aggregation_framework(self):
         class Person(Document):
             name = StringField()
@@ -4925,17 +4905,13 @@ class QuerySetTest(unittest.TestCase):
             {'_id': p1.pk, 'name': "ISABELLA LUANNA"}
         ])
 
-        data = Person.objects(
+        data = Person.objects(age__gte=17, age__lte=40).order_by('-age').aggregate({
-            age__gte=17, age__lte=40).order_by('-age').aggregate(
+            '$group': {
-                {'$group': {
                 '_id': None,
                 'total': {'$sum': 1},
                 'avg': {'$avg': '$age'}
             }
-                }
+        })
-
-        )
-
         self.assertEqual(list(data), [
             {'_id': None, 'avg': 29, 'total': 2}
         ])
@@ -4976,11 +4952,13 @@ class QuerySetTest(unittest.TestCase):
         self.assertEquals(Animal.objects(folded_ears=True).count(), 1)
         self.assertEquals(Animal.objects(whiskers_length=5.1).count(), 1)
 
-    def test_loop_via_invalid_id_does_not_crash(self):
+    def test_loop_over_invalid_id_does_not_crash(self):
         class Person(Document):
             name = StringField()
-        Person.objects.delete()
+
-        Person._get_collection().update({"name": "a"}, {"$set": {"_id": ""}}, upsert=True)
+        Person.drop_collection()
+
+        Person._get_collection().insert({'name': 'a', 'id': ''})
         for p in Person.objects():
             self.assertEqual(p.name, 'a')
 

tests/test_connection.py

@@ -35,8 +35,7 @@ class ConnectionTest(unittest.TestCase):
         mongoengine.connection._dbs = {}
 
     def test_connect(self):
-        """Ensure that the connect() method works properly.
+        """Ensure that the connect() method works properly."""
-        """
         connect('mongoenginetest')
 
         conn = get_connection()
@@ -146,8 +145,7 @@ class ConnectionTest(unittest.TestCase):
         self.assertEqual(expected_connection, actual_connection)
 
     def test_connect_uri(self):
-        """Ensure that the connect() method works properly with uri's
+        """Ensure that the connect() method works properly with URIs."""
-        """
         c = connect(db='mongoenginetest', alias='admin')
         c.admin.system.users.remove({})
         c.mongoenginetest.system.users.remove({})
@@ -200,19 +198,6 @@ class ConnectionTest(unittest.TestCase):
         self.assertTrue(isinstance(db, pymongo.database.Database))
         self.assertEqual(db.name, 'test')
 
-    def test_connect_uri_with_replicaset(self):
-        """Ensure connect() works when specifying a replicaSet."""
-        if IS_PYMONGO_3:
-            c = connect(host='mongodb://localhost/test?replicaSet=local-rs')
-            db = get_db()
-            self.assertTrue(isinstance(db, pymongo.database.Database))
-            self.assertEqual(db.name, 'test')
-        else:
-            # PyMongo < v3.x raises an exception:
-            # "localhost:27017 is not a member of replica set local-rs"
-            with self.assertRaises(MongoEngineConnectionError):
-                c = connect(host='mongodb://localhost/test?replicaSet=local-rs')
-
     def test_uri_without_credentials_doesnt_override_conn_settings(self):
         """Ensure connect() uses the username & password params if the URI
         doesn't explicitly specify them.
@@ -227,9 +212,8 @@ class ConnectionTest(unittest.TestCase):
         self.assertRaises(OperationFailure, get_db)
 
     def test_connect_uri_with_authsource(self):
-        """Ensure that the connect() method works well with
+        """Ensure that the connect() method works well with `authSource`
-        the option `authSource` in URI.
+        option in the URI.
-        This feature was introduced in MongoDB 2.4 and removed in 2.6
         """
         # Create users
         c = connect('mongoenginetest')
@@ -238,30 +222,31 @@ class ConnectionTest(unittest.TestCase):
 
         # Authentication fails without "authSource"
         if IS_PYMONGO_3:
-            test_conn = connect('mongoenginetest', alias='test1',
+            test_conn = connect(
-                                host='mongodb://username2:password@localhost/mongoenginetest')
+                'mongoenginetest', alias='test1',
+                host='mongodb://username2:password@localhost/mongoenginetest'
+            )
             self.assertRaises(OperationFailure, test_conn.server_info)
         else:
             self.assertRaises(
-                MongoEngineConnectionError, connect, 'mongoenginetest',
+                MongoEngineConnectionError,
-                alias='test1',
+                connect, 'mongoenginetest', alias='test1',
                 host='mongodb://username2:password@localhost/mongoenginetest'
             )
             self.assertRaises(MongoEngineConnectionError, get_db, 'test1')
 
         # Authentication succeeds with "authSource"
-        connect(
+        authd_conn = connect(
             'mongoenginetest', alias='test2',
             host=('mongodb://username2:password@localhost/'
                   'mongoenginetest?authSource=admin')
         )
-        # This will fail starting from MongoDB 2.6+
         db = get_db('test2')
         self.assertTrue(isinstance(db, pymongo.database.Database))
         self.assertEqual(db.name, 'mongoenginetest')
 
         # Clear all users
-        c.admin.system.users.remove({})
+        authd_conn.admin.system.users.remove({})
 
     def test_register_connection(self):
         """Ensure that connections with different aliases may be registered.
@@ -285,8 +270,7 @@ class ConnectionTest(unittest.TestCase):
         self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
 
     def test_connection_kwargs(self):
-        """Ensure that connection kwargs get passed to pymongo.
+        """Ensure that connection kwargs get passed to pymongo."""
-        """
         connect('mongoenginetest', alias='t1', tz_aware=True)
         conn = get_connection('t1')
 
@@ -296,6 +280,32 @@ class ConnectionTest(unittest.TestCase):
         conn = get_connection('t2')
         self.assertFalse(get_tz_awareness(conn))
 
+    def test_connection_pool_via_kwarg(self):
+        """Ensure we can specify a max connection pool size using
+        a connection kwarg.
+        """
+        # Use "max_pool_size" or "maxpoolsize" depending on PyMongo version
+        # (former was changed to the latter as described in
+        # https://jira.mongodb.org/browse/PYTHON-854).
+        # TODO remove once PyMongo < 3.0 support is dropped
+        if pymongo.version_tuple[0] >= 3:
+            pool_size_kwargs = {'maxpoolsize': 100}
+        else:
+            pool_size_kwargs = {'max_pool_size': 100}
+
+        conn = connect('mongoenginetest', alias='max_pool_size_via_kwarg', **pool_size_kwargs)
+        self.assertEqual(conn.max_pool_size, 100)
+
+    def test_connection_pool_via_uri(self):
+        """Ensure we can specify a max connection pool size using
+        an option in a connection URI.
+        """
+        if pymongo.version_tuple[0] == 2 and pymongo.version_tuple[1] < 9:
+            raise SkipTest('maxpoolsize as a URI option is only supported in PyMongo v2.9+')
+
+        conn = connect(host='mongodb://localhost/test?maxpoolsize=100', alias='max_pool_size_via_uri')
+        self.assertEqual(conn.max_pool_size, 100)
+
     def test_write_concern(self):
         """Ensure write concern can be specified in connect() via
         a kwarg or as part of the connection URI.
@@ -309,6 +319,38 @@ class ConnectionTest(unittest.TestCase):
             self.assertEqual(dict(conn1.write_concern), {'w': 1, 'j': True})
             self.assertEqual(dict(conn2.write_concern), {'w': 1, 'j': True})
 
+    def test_connect_with_replicaset_via_uri(self):
+        """Ensure connect() works when specifying a replicaSet via the
+        MongoDB URI.
+        """
+        if IS_PYMONGO_3:
+            c = connect(host='mongodb://localhost/test?replicaSet=local-rs')
+            db = get_db()
+            self.assertTrue(isinstance(db, pymongo.database.Database))
+            self.assertEqual(db.name, 'test')
+        else:
+            # PyMongo < v3.x raises an exception:
+            # "localhost:27017 is not a member of replica set local-rs"
+            with self.assertRaises(MongoEngineConnectionError):
+                c = connect(host='mongodb://localhost/test?replicaSet=local-rs')
+
+    def test_connect_with_replicaset_via_kwargs(self):
+        """Ensure connect() works when specifying a replicaSet via the
+        connection kwargs
+        """
+        if IS_PYMONGO_3:
+            c = connect(replicaset='local-rs')
+            self.assertEqual(c._MongoClient__options.replica_set_name,
+                             'local-rs')
+            db = get_db()
+            self.assertTrue(isinstance(db, pymongo.database.Database))
+            self.assertEqual(db.name, 'test')
+        else:
+            # PyMongo < v3.x raises an exception:
+            # "localhost:27017 is not a member of replica set local-rs"
+            with self.assertRaises(MongoEngineConnectionError):
+                c = connect(replicaset='local-rs')
+
     def test_datetime(self):
         connect('mongoenginetest', tz_aware=True)
         d = datetime.datetime(2010, 5, 5, tzinfo=utc)

tests/utils.py

@@ -0,0 +1,78 @@
+import unittest
+
+from nose.plugins.skip import SkipTest
+
+from mongoengine import connect
+from mongoengine.connection import get_db, get_connection
+from mongoengine.python_support import IS_PYMONGO_3
+
+
+MONGO_TEST_DB = 'mongoenginetest'
+
+
+class MongoDBTestCase(unittest.TestCase):
+    """Base class for tests that need a mongodb connection
+    db is being dropped automatically
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls._connection = connect(db=MONGO_TEST_DB)
+        cls._connection.drop_database(MONGO_TEST_DB)
+        cls.db = get_db()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._connection.drop_database(MONGO_TEST_DB)
+
+
+def get_mongodb_version():
+    """Return the version tuple of the MongoDB server that the default
+    connection is connected to.
+    """
+    return tuple(get_connection().server_info()['versionArray'])
+
+def _decorated_with_ver_requirement(func, ver_tuple):
+    """Return a given function decorated with the version requirement
+    for a particular MongoDB version tuple.
+    """
+    def _inner(*args, **kwargs):
+        mongodb_ver = get_mongodb_version()
+        if mongodb_ver >= ver_tuple:
+            return func(*args, **kwargs)
+
+        raise SkipTest('Needs MongoDB v{}+'.format(
+            '.'.join([str(v) for v in ver_tuple])
+        ))
+
+    _inner.__name__ = func.__name__
+    _inner.__doc__ = func.__doc__
+
+    return _inner
+
+def needs_mongodb_v26(func):
+    """Raise a SkipTest exception if we're working with MongoDB version
+    lower than v2.6.
+    """
+    return _decorated_with_ver_requirement(func, (2, 6))
+
+def needs_mongodb_v3(func):
+    """Raise a SkipTest exception if we're working with MongoDB version
+    lower than v3.0.
+    """
+    return _decorated_with_ver_requirement(func, (3, 0))
+
+def skip_pymongo3(f):
+    """Raise a SkipTest exception if we're running a test against
+    PyMongo v3.x.
+    """
+    def _inner(*args, **kwargs):
+        if IS_PYMONGO_3:
+            raise SkipTest("Useless with PyMongo 3+")
+        return f(*args, **kwargs)
+
+    _inner.__name__ = f.__name__
+    _inner.__doc__ = f.__doc__
+
+    return _inner
+

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py26,py27,py33,py34,py35,pypy,pypy3}-{mg27,mg28},flake8
+envlist = {py27,py35,pypy,pypy3}-{mg27,mg28,mg30}
 
 [testenv]
 commands =
@@ -7,16 +7,7 @@ commands =
 deps =
     nose
     mg27: PyMongo<2.8
-    mg28: PyMongo>=2.8,<3.0
+    mg28: PyMongo>=2.8,<2.9
     mg30: PyMongo>=3.0
-    mgdev: https://github.com/mongodb/mongo-python-driver/tarball/master
 setenv =
     PYTHON_EGG_CACHE = {envdir}/python-eggs
-passenv = windir
-
-[testenv:flake8]
-deps =
-    flake8
-    flake8-import-order
-commands =
-   flake8