fix flake8

Fix BaseQuerySet#sum and BaseQuerySet#average for fields that specify a db_field

.travis.yml

@@ -1,67 +1,58 @@
 language: python
+
 python:
-- '2.6'
+- '2.6'  # TODO remove in v0.11.0
 - '2.7'
-- '3.2'
 - '3.3'
 - '3.4'
+- '3.5'
 - pypy
 - pypy3
+
 env:
-- PYMONGO=2.7.2 DJANGO=dev
+- PYMONGO=2.7
-- PYMONGO=2.7.2 DJANGO=1.7.1
+- PYMONGO=2.8
-- PYMONGO=2.7.2 DJANGO=1.6.8
+- PYMONGO=3.0
-- PYMONGO=2.7.2 DJANGO=1.5.11
+- PYMONGO=dev
-- PYMONGO=2.8 DJANGO=dev
+
-- PYMONGO=2.8 DJANGO=1.7.1
-- PYMONGO=2.8 DJANGO=1.6.8
-- PYMONGO=2.8 DJANGO=1.5.11
 matrix:
-  exclude:
-  - python: '2.6'
-    env: PYMONGO=2.7.2 DJANGO=dev
-  - python: '2.6'
-    env: PYMONGO=2.8 DJANGO=dev
-  - python: '2.6'
-    env: PYMONGO=2.7.2 DJANGO=1.7.1
-  - python: '2.6'
-    env: PYMONGO=2.8 DJANGO=1.7.1
-  allow_failures:
-  - python: pypy3
   fast_finish: true
+
 before_install:
 - travis_retry sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 - echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' |
   sudo tee /etc/apt/sources.list.d/mongodb.list
 - travis_retry sudo apt-get update
 - travis_retry sudo apt-get install mongodb-org-server
+
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
   libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev
   python-tk
-- if [[ $PYMONGO == 'dev' ]]; then travis_retry pip install https://github.com/mongodb/mongo-python-driver/tarball/master;
+- travis_retry pip install --upgrade pip
-  true; fi
-- if [[ $PYMONGO != 'dev' ]]; then travis_retry pip install pymongo==$PYMONGO; true;
-  fi
-- if [[ $DJANGO == 'dev' ]]; then travis_retry pip install git+https://github.com/django/django.git;
-  fi
-- if [[ $DJANGO != 'dev' ]]; then travis_retry pip install Django==$DJANGO; fi
-- travis_retry pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b
 - travis_retry pip install coveralls
-- travis_retry python setup.py install
+- travis_retry pip install flake8
+- travis_retry pip install tox>=1.9
+- travis_retry pip install "virtualenv<14.0.0"  # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
+- travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
+
+# Run flake8 for py27
+before_script:
+- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then tox -e flake8; fi
+
 script:
-- travis_retry python setup.py test
+- tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
-- if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then 2to3 . -w; fi;
+
-- coverage run --source=mongoengine setup.py test
-- coverage report -m
-- python benchmark.py
 after_script: coveralls --verbose
+
 notifications:
   irc: irc.freenode.org#mongoengine
+
 branches:
   only:
   - master
   - /^v.*$/
+
 deploy:
   provider: pypi
   user: the_drow

AUTHORS

@@ -12,7 +12,7 @@ Laine Herron https://github.com/LaineHerron
 
 CONTRIBUTORS
 
-Dervived from the git logs, inevitably incomplete but all of whom and others
+Derived from the git logs, inevitably incomplete but all of whom and others
 have submitted patches, reported bugs and generally helped make MongoEngine
 that much better:
 
@@ -119,7 +119,7 @@ that much better:
  * Anton Kolechkin
  * Sergey Nikitin
  * psychogenic
- * Stefan Wójcik
+ * Stefan Wójcik (https://github.com/wojcikstefan)
  * dimonb
  * Garry Polley
  * James Slagle
@@ -138,7 +138,6 @@ that much better:
  * hellysmile
  * Jaepil Jeong
  * Daniil Sharou
- * Stefan Wójcik
  * Pete Campton
  * Martyn Smith
  * Marcelo Anton
@@ -218,3 +217,28 @@ that much better:
  * Matthew Ellison (https://github.com/seglberg)
  * Jimmy Shen (https://github.com/jimmyshen)
  * J. Fernando Sánchez (https://github.com/balkian)
+ * Michael Chase (https://github.com/rxsegrxup)
+ * Eremeev Danil (https://github.com/elephanter)
+ * Catstyle Lee (https://github.com/Catstyle)
+ * Kiryl Yermakou (https://github.com/rma4ok)
+ * Matthieu Rigal (https://github.com/MRigal)
+ * Charanpal Dhanjal (https://github.com/charanpald)
+ * Emmanuel Leblond (https://github.com/touilleMan)
+ * Breeze.Kay (https://github.com/9nix00)
+ * Vicki Donchenko (https://github.com/kivistein)
+ * Emile Caron (https://github.com/emilecaron)
+ * Amit Lichtenberg (https://github.com/amitlicht)
+ * Gang Li (https://github.com/iici-gli)
+ * Lars Butler (https://github.com/larsbutler)
+ * George Macon (https://github.com/gmacon)
+ * Ashley Whetter (https://github.com/AWhetter)
+ * Paul-Armand Verhaegen (https://github.com/paularmand)
+ * Steven Rossiter (https://github.com/BeardedSteve)
+ * Luo Peng (https://github.com/RussellLuo)
+ * Bryan Bennett (https://github.com/bbenne10)
+ * Gilb's Gilb's (https://github.com/gilbsgilbs)
+ * Joshua Nedrud (https://github.com/Neurostack)
+ * Shu Shen (https://github.com/shushen)
+ * xiaost7 (https://github.com/xiaost7)
+ * Victor Varvaryuk
+ * Stanislav Kaledin (https://github.com/sallyruthstruik)

CONTRIBUTING.rst

@@ -29,7 +29,10 @@ Style Guide
 -----------
 
 MongoEngine aims to follow `PEP8 <http://www.python.org/dev/peps/pep-0008/>`_
-including 4 space indents and 79 character line limits.
+including 4 space indents. When possible we try to stick to 79 character line limits.
+However, screens got bigger and an ORM has a strong focus on readability and
+if it can help, we accept 119 as maximum line length, in a similar way as
+`django does <https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/#python-style>`_
 
 Testing
 -------
@@ -38,14 +41,21 @@ All tests are run on `Travis <http://travis-ci.org/MongoEngine/mongoengine>`_
 and any pull requests are automatically tested by Travis. Any pull requests
 without tests will take longer to be integrated and might be refused.
 
+You may also submit a simple failing test as a PullRequest if you don't know
+how to fix it, it will be easier for other people to work on it and it may get
+fixed faster.
+
 General Guidelines
 ------------------
 
 - Avoid backward breaking changes if at all possible.
 - Write inline documentation for new classes and methods.
 - Write tests and make sure they pass (make sure you have a mongod
-  running on the default port, then execute ``python setup.py test``
+  running on the default port, then execute ``python setup.py nosetests``
   from the cmd line to run the test suite).
+- Ensure tests pass on every Python and PyMongo versions.
+  You can test on these versions locally by executing ``tox``
+- Add enhancements or problematic bug fixes to docs/changelog.rst
 - Add yourself to AUTHORS :)
 
 Documentation

README.rst

@@ -6,27 +6,29 @@ MongoEngine
 :Author: Harry Marr (http://github.com/hmarr)
 :Maintainer: Ross Lawley (http://github.com/rozza)
 
-.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master
+.. image:: https://travis-ci.org/MongoEngine/mongoengine.svg?branch=master
-  :target: http://travis-ci.org/MongoEngine/mongoengine
+  :target: https://travis-ci.org/MongoEngine/mongoengine
 
-.. image:: https://coveralls.io/repos/MongoEngine/mongoengine/badge.png?branch=master 
+.. image:: https://coveralls.io/repos/github/MongoEngine/mongoengine/badge.svg?branch=master
-  :target: https://coveralls.io/r/MongoEngine/mongoengine?branch=master
+  :target: https://coveralls.io/github/MongoEngine/mongoengine?branch=master
 
-.. image:: https://landscape.io/github/MongoEngine/mongoengine/master/landscape.png
+.. image:: https://landscape.io/github/MongoEngine/mongoengine/master/landscape.svg?style=flat
-   :target: https://landscape.io/github/MongoEngine/mongoengine/master
+  :target: https://landscape.io/github/MongoEngine/mongoengine/master
-   :alt: Code Health
+  :alt: Code Health
 
 About
 =====
 MongoEngine is a Python Object-Document Mapper for working with MongoDB.
-Documentation available at http://mongoengine-odm.rtfd.org - there is currently
+Documentation available at https://mongoengine-odm.readthedocs.io - there is currently
-a `tutorial <http://readthedocs.org/docs/mongoengine-odm/en/latest/tutorial.html>`_, a `user guide
+a `tutorial <https://mongoengine-odm.readthedocs.io/tutorial.html>`_, a `user guide
-<https://mongoengine-odm.readthedocs.org/en/latest/guide/index.html>`_ and an `API reference
+<https://mongoengine-odm.readthedocs.io/guide/index.html>`_ and an `API reference
-<http://readthedocs.org/docs/mongoengine-odm/en/latest/apireference.html>`_.
+<https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Installation
 ============
-If you have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
+We recommend the use of `virtualenv <https://virtualenv.pypa.io/>`_ and of
+`pip <https://pip.pypa.io/>`_. You can then use ``pip install -U mongoengine``.
+You may also have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_ and thus
 you can use ``easy_install -U mongoengine``. Otherwise, you can download the
 source from `GitHub <http://github.com/MongoEngine/mongoengine>`_ and run ``python
 setup.py install``.
@@ -38,21 +40,26 @@ Dependencies
 
 Optional Dependencies
 ---------------------
-- **Django Integration:** Django>=1.4.0 for Python 2.x or PyPy and Django>=1.5.0 for Python 3.x
+- **Image Fields**: Pillow>=2.0.0
-- **Image Fields**: Pillow>=2.0.0 or PIL (not recommended since MongoEngine is tested with Pillow)
 - dateutil>=2.1.0
 
 .. note
-   MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: Django 1.6.5
+   MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: PyMongo 3.0.1
 
 Examples
 ========
-Some simple examples of what MongoEngine code looks like::
+Some simple examples of what MongoEngine code looks like:
+
+.. code :: python
+
+    from mongoengine import *
+    connect('mydb')
 
     class BlogPost(Document):
         title = StringField(required=True, max_length=200)
         posted = DateTimeField(default=datetime.datetime.now)
         tags = ListField(StringField(max_length=50))
+        meta = {'allow_inheritance': True}
 
     class TextPost(BlogPost):
         content = StringField(required=True)
@@ -82,7 +89,7 @@ Some simple examples of what MongoEngine code looks like::
 
     >>> len(BlogPost.objects)
     2
-    >>> len(HtmlPost.objects)
+    >>> len(TextPost.objects)
     1
     >>> len(LinkPost.objects)
     1
@@ -96,7 +103,26 @@ Some simple examples of what MongoEngine code looks like::
 Tests
 =====
 To run the test suite, ensure you are running a local instance of MongoDB on
-the standard port, and run: ``python setup.py test``.
+the standard port and have ``nose`` installed. Then, run: ``python setup.py nosetests``.
+
+To run the test suite on every supported Python version and every supported PyMongo version,
+you can use ``tox``.
+tox and each supported Python version should be installed in your environment:
+
+.. code-block:: shell
+
+    # Install tox
+    $ pip install tox
+    # Run the test suites
+    $ tox
+
+If you wish to run one single or selected tests, use the nosetest convention. It will find the folder,
+eventually the file, go to the TestClass specified after the colon and eventually right to the single test.
+Also use the -s argument if you want to print out whatever or access pdb while testing.
+
+.. code-block:: shell
+
+    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest.test_cls_field -s
 
 Community
 =========

docs/_themes/sphinx_rtd_theme/footer.html

@@ -2,7 +2,7 @@
   {% if next or prev %}
     <div class="rst-footer-buttons">
       {% if next %}
-        <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}"/>Next <span class="icon icon-circle-arrow-right"></span></a>
+        <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}">Next <span class="icon icon-circle-arrow-right"></span></a>
       {% endif %}
       {% if prev %}
         <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}"><span class="icon icon-circle-arrow-left"></span> Previous</a>

docs/apireference.rst

@@ -34,6 +34,9 @@ Documents
 .. autoclass:: mongoengine.ValidationError
   :members:
 
+.. autoclass:: mongoengine.FieldDoesNotExist
+
+
 Context Managers
 ================
 

docs/changelog.rst

@@ -2,9 +2,107 @@
 Changelog
 =========
 
+Changes in 0.10.8
+=================
+- Added ability to specify an authentication mechanism (e.g. X.509) #1333
+- Added support for falsey primary keys (e.g. doc.pk = 0) #1354
+- Fixed BaseQuerySet#sum/average for fields w/ explicit db_field #1417
 
-Changes in 0.9.X - DEV
+Changes in 0.10.7
-======================
+=================
+- Dropped Python 3.2 support #1390
+- Fixed the bug where dynamic doc has index inside a dict field #1278
+- Fixed: ListField minus index assignment does not work #1128
+- Fixed cascade delete mixing among collections #1224
+- Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206
+- Raise `OperationError` when trying to do a `drop_collection` on document with no collection set.
+- count on ListField of EmbeddedDocumentField fails. #1187
+- Fixed long fields stored as int32 in Python 3. #1253
+- MapField now handles unicodes keys correctly. #1267
+- ListField now handles negative indicies correctly. #1270
+- Fixed AttributeError when initializing EmbeddedDocument with positional args. #681
+- Fixed no_cursor_timeout error with pymongo 3.0+ #1304
+- Replaced map-reduce based QuerySet.sum/average with aggregation-based implementations #1336
+- Fixed support for `__` to escape field names that match operators names in `update` #1351
+- Fixed BaseDocument#_mark_as_changed #1369
+- Added support for pickling QuerySet instances. #1397
+- Fixed connecting to a list of hosts #1389
+- Fixed a bug where accessing broken references wouldn't raise a DoesNotExist error #1334
+- Fixed not being able to specify use_db_field=False on ListField(EmbeddedDocumentField) instances #1218
+- Improvements to the dictionary fields docs #1383
+
+Changes in 0.10.6
+=================
+- Add support for mocking MongoEngine based on mongomock. #1151
+- Fixed not being able to run tests on Windows. #1153
+- Allow creation of sparse compound indexes. #1114
+- count on ListField of EmbeddedDocumentField fails. #1187
+
+Changes in 0.10.5
+=================
+- Fix for reloading of strict with special fields. #1156
+
+Changes in 0.10.4
+=================
+- SaveConditionError is now importable from the top level package. #1165
+- upsert_one method added. #1157
+
+Changes in 0.10.3
+=================
+- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042
+
+Changes in 0.10.2
+=================
+- Allow shard key to point to a field in an embedded document. #551
+- Allow arbirary metadata in fields. #1129
+- ReferenceFields now support abstract document types. #837
+
+Changes in 0.10.1
+=================
+- Fix infinite recursion with CASCADE delete rules under specific conditions. #1046
+- Fix CachedReferenceField bug when loading cached docs as DBRef but failing to save them. #1047
+- Fix ignored chained options #842
+- Document save's save_condition error raises `SaveConditionError` exception #1070
+- Fix Document.reload for DynamicDocument. #1050
+- StrictDict & SemiStrictDict are shadowed at init time. #1105
+- Fix ListField minus index assignment does not work. #1119
+- Remove code that marks field as changed when the field has default but not existed in database #1126
+- Remove test dependencies (nose and rednose) from install dependencies list. #1079
+- Recursively build query when using elemMatch operator. #1130
+- Fix instance back references for lists of embedded documents. #1131
+
+Changes in 0.10.0
+=================
+- Django support was removed and will be available as a separate extension. #958
+- Allow to load undeclared field with meta attribute 'strict': False #957
+- Support for PyMongo 3+ #946
+- Removed get_or_create() deprecated since 0.8.0. #300
+- Improve Document._created status when switch collection and db #1020
+- Queryset update doesn't go through field validation #453
+- Added support for specifying authentication source as option `authSource` in URI. #967
+- Fixed mark_as_changed to handle higher/lower level fields changed. #927
+- ListField of embedded docs doesn't set the _instance attribute when iterating over it #914
+- Support += and *= for ListField #595
+- Use sets for populating dbrefs to dereference
+- Fixed unpickled documents replacing the global field's list. #888
+- Fixed storage of microseconds in ComplexDateTimeField and unused separator option. #910
+- Don't send a "cls" option to ensureIndex (related to https://jira.mongodb.org/browse/SERVER-769)
+- Fix for updating sorting in SortedListField. #978
+- Added __ support to escape field name in fields lookup keywords that match operators names #949
+- Fix for issue where FileField deletion did not free space in GridFS.
+- No_dereference() not respected on embedded docs containing reference. #517
+- Document save raise an exception if save_condition fails #1005
+- Fixes some internal _id handling issue. #961
+- Updated URL and Email Field regex validators, added schemes argument to URLField validation. #652
+- Capped collection multiple of 256. #1011
+- Added `BaseQuerySet.aggregate_sum` and `BaseQuerySet.aggregate_average` methods.
+- Fix for delete with write_concern {'w': 0}. #1008
+- Allow dynamic lookup for more than two parts. #882
+- Added support for min_distance on geo queries. #831
+- Allow to add custom metadata to fields #705
+
+Changes in 0.9.0
+================
 - Update FileField when creating a new file #714
 - Added `EmbeddedDocumentListField` for Lists of Embedded Documents. #826
 - ComplexDateTimeField should fall back to None when null=True #864
@@ -78,6 +176,7 @@ Changes in 0.9.X - DEV
 - Fixed a few instances where reverse_delete_rule was written as reverse_delete_rules. #791
 - Make `in_bulk()` respect `no_dereference()` #775
 - Handle None from model __str__; Fixes #753 #754
+- _get_changed_fields fix for embedded documents with id field. #925
 
 Changes in 0.8.7
 ================

docs/code/tumblelog.py

@@ -17,6 +17,10 @@ class Post(Document):
     tags = ListField(StringField(max_length=30))
     comments = ListField(EmbeddedDocumentField(Comment))
 
+    # bugfix
+    meta = {'allow_inheritance': True}
+
+
 class TextPost(Post):
     content = StringField()
 
@@ -45,7 +49,8 @@ print 'ALL POSTS'
 print
 for post in Post.objects:
     print post.title
-    print '=' * post.title.count()
+    #print '=' * post.title.count()
+    print "=" * 20
 
     if isinstance(post, TextPost):
         print post.content

docs/django.rst

@@ -2,176 +2,18 @@
 Django Support
 ==============
 
-.. note:: Updated to support Django 1.5
+.. note:: Django support has been split from the main MongoEngine
-
+    repository. The *legacy* Django extension may be found bundled with the
-Connecting
+    0.9 release of MongoEngine.
-==========
-In your **settings.py** file, ignore the standard database settings (unless you
-also plan to use the ORM in your project), and instead call
-:func:`~mongoengine.connect` somewhere in the settings module.
-
-.. note::
-   If you are not using another Database backend you may need to add a dummy
-   database backend to ``settings.py`` eg::
-
-        DATABASES = {
-            'default': {
-                'ENGINE': 'django.db.backends.dummy'
-            }
-        }
-
-Authentication
-==============
-MongoEngine includes a Django authentication backend, which uses MongoDB. The
-:class:`~mongoengine.django.auth.User` model is a MongoEngine
-:class:`~mongoengine.Document`, but implements most of the methods and
-attributes that the standard Django :class:`User` model does - so the two are
-moderately compatible. Using this backend will allow you to store users in
-MongoDB but still use many of the Django authentication infrastructure (such as
-the :func:`login_required` decorator and the :func:`authenticate` function). To
-enable the MongoEngine auth backend, add the following to your **settings.py**
-file::
-
-    AUTHENTICATION_BACKENDS = (
-        'mongoengine.django.auth.MongoEngineBackend',
-    )
-
-The :mod:`~mongoengine.django.auth` module also contains a
-:func:`~mongoengine.django.auth.get_user` helper function, that takes a user's
-:attr:`id` and returns a :class:`~mongoengine.django.auth.User` object.
-
-.. versionadded:: 0.1.3
-
-Custom User model
-=================
-Django 1.5 introduced `Custom user Models
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/#auth-custom-user>`_
-which can be used as an alternative to the MongoEngine authentication backend.
-
-The main advantage of this option is that other components relying on
-:mod:`django.contrib.auth` and supporting the new swappable user model are more
-likely to work. For example, you can use the ``createsuperuser`` management
-command as usual.
-
-To enable the custom User model in Django, add ``'mongoengine.django.mongo_auth'``
-in your ``INSTALLED_APPS`` and set ``'mongo_auth.MongoUser'`` as the custom user
-user model to use. In your **settings.py** file you will have::
-
-    INSTALLED_APPS = (
-        ...
-        'django.contrib.auth',
-        'mongoengine.django.mongo_auth',
-        ...
-    )
-
-    AUTH_USER_MODEL = 'mongo_auth.MongoUser'
-
-An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the
-:class:`~mongoengine.django.auth.User` class with another class of your choice::
-
-    MONGOENGINE_USER_DOCUMENT = 'mongoengine.django.auth.User'
-
-The custom :class:`User` must be a :class:`~mongoengine.Document` class, but
-otherwise has the same requirements as a standard custom user model,
-as specified in the `Django Documentation
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/>`_.
-In particular, the custom class must define :attr:`USERNAME_FIELD` and
-:attr:`REQUIRED_FIELDS` attributes.
-
-Sessions
-========
-Django allows the use of different backend stores for its sessions. MongoEngine
-provides a MongoDB-based session backend for Django, which allows you to use
-sessions in your Django application with just MongoDB. To enable the MongoEngine
-session backend, ensure that your settings module has
-``'django.contrib.sessions.middleware.SessionMiddleware'`` in the
-``MIDDLEWARE_CLASSES`` field  and ``'django.contrib.sessions'`` in your
-``INSTALLED_APPS``. From there, all you need to do is add the following line
-into your settings module::
-
-    SESSION_ENGINE = 'mongoengine.django.sessions'
-    SESSION_SERIALIZER = 'mongoengine.django.sessions.BSONSerializer'
-
-Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesn't delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports  `mongodb TTL
-<http://docs.mongodb.org/manual/tutorial/expire-data/>`_.
-
-.. note:: ``SESSION_SERIALIZER`` is only necessary in Django 1.6 as the default
-   serializer is based around JSON and doesn't know how to convert
-   ``bson.objectid.ObjectId`` instances to strings.
-
-.. versionadded:: 0.2.1
-
-Storage
-=======
-With MongoEngine's support for GridFS via the :class:`~mongoengine.fields.FileField`,
-it is useful to have a Django file storage backend that wraps this. The new
-storage module is called :class:`~mongoengine.django.storage.GridFSStorage`.
-Using it is very similar to using the default FileSystemStorage.::
-
-    from mongoengine.django.storage import GridFSStorage
-    fs = GridFSStorage()
-
-    filename = fs.save('hello.txt', 'Hello, World!')
-
-All of the `Django Storage API methods
-<http://docs.djangoproject.com/en/dev/ref/files/storage/>`_ have been
-implemented except :func:`path`. If the filename provided already exists, an
-underscore and a number (before # the file extension, if one exists) will be
-appended to the filename until the generated filename doesn't exist. The
-:func:`save` method will return the new filename.::
-
-    >>> fs.exists('hello.txt')
-    True
-    >>> fs.open('hello.txt').read()
-    'Hello, World!'
-    >>> fs.size('hello.txt')
-    13
-    >>> fs.url('hello.txt')
-    'http://your_media_url/hello.txt'
-    >>> fs.open('hello.txt').name
-    'hello.txt'
-    >>> fs.listdir()
-    ([], [u'hello.txt'])
-
-All files will be saved and retrieved in GridFS via the :class:`FileDocument`
-document, allowing easy access to the files without the GridFSStorage
-backend.::
-
-    >>> from mongoengine.django.storage import FileDocument
-    >>> FileDocument.objects()
-    [<FileDocument: FileDocument object>]
-
-.. versionadded:: 0.4
-
-Shortcuts
-=========
-Inspired by the `Django shortcut get_object_or_404
-<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-object-or-404>`_,
-the :func:`~mongoengine.django.shortcuts.get_document_or_404` method returns 
-a document or raises an Http404 exception if the document does not exist::
-
-    from mongoengine.django.shortcuts import get_document_or_404
-    
-    admin_user = get_document_or_404(User, username='root')
-
-The first argument may be a Document or QuerySet object. All other passed arguments
-and keyword arguments are used in the query::
-
-    foo_email = get_document_or_404(User.objects.only('email'), username='foo', is_active=True).email
-
-.. note:: Like with :func:`get`, a MultipleObjectsReturned will be raised if more than one
-    object is found.
 
 
-Also inspired by the `Django shortcut get_list_or_404
-<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-list-or-404>`_,
-the :func:`~mongoengine.django.shortcuts.get_list_or_404` method returns a list of
-documents or raises an Http404 exception if the list is empty::
 
-    from mongoengine.django.shortcuts import get_list_or_404
+Help Wanted!
-    
+------------
-    active_users = get_list_or_404(User, is_active=True)
-
-The first argument may be a Document or QuerySet object. All other passed
-arguments and keyword arguments are used to filter the query.
 
+The MongoEngine team is looking for help contributing and maintaining a new
+Django extension for MongoEngine! If you have Django experience and would like
+to help contribute to the project, please get in touch on the 
+`mailing list <http://groups.google.com/group/mongoengine-users>`_ or by 
+simply contributing on
+`GitHub <https://github.com/MongoEngine/django-mongoengine>`_.

docs/guide/defining-documents.rst

@@ -29,7 +29,7 @@ documents are serialized based on their field order.
 
 Dynamic document schemas
 ========================
-One of the benefits of MongoDb is dynamic schemas for a collection, whilst data
+One of the benefits of MongoDB is dynamic schemas for a collection, whilst data
 should be planned and organised (after all explicit is better than implicit!)
 there are scenarios where having dynamic / expando style documents is desirable.
 
@@ -75,6 +75,7 @@ are as follows:
 * :class:`~mongoengine.fields.DynamicField`
 * :class:`~mongoengine.fields.EmailField`
 * :class:`~mongoengine.fields.EmbeddedDocumentField`
+* :class:`~mongoengine.fields.EmbeddedDocumentListField`
 * :class:`~mongoengine.fields.FileField`
 * :class:`~mongoengine.fields.FloatField`
 * :class:`~mongoengine.fields.GenericEmbeddedDocumentField`
@@ -114,7 +115,7 @@ arguments can be set on all fields:
 :attr:`default` (Default: None)
     A value to use when no value is set for this field.
 
-    The definion of default parameters follow `the general rules on Python
+    The definition of default parameters follow `the general rules on Python
     <http://docs.python.org/reference/compound_stmts.html#function-definitions>`__,
     which means that some care should be taken when dealing with default mutable objects
     (like in :class:`~mongoengine.fields.ListField` or :class:`~mongoengine.fields.DictField`)::
@@ -146,6 +147,8 @@ arguments can be set on all fields:
     When True, use this field as a primary key for the collection.  `DictField`
     and `EmbeddedDocuments` both support being the primary key for a document.
 
+    .. note:: If set, this field is also accessible through the `pk` field.
+
 :attr:`choices` (Default: None)
     An iterable (e.g. a list or tuple) of choices to which the value of this
     field should be limited.
@@ -170,11 +173,11 @@ arguments can be set on all fields:
         class Shirt(Document):
             size = StringField(max_length=3, choices=SIZE)
 
-:attr:`help_text` (Default: None)
+:attr:`**kwargs` (Optional)
-    Optional help text to output with the field -- used by form libraries
+    You can supply additional metadata as arbitrary additional keyword
-
+    arguments.  You can not override existing attributes, however.  Common
-:attr:`verbose_name` (Default: None)
+    choices include `help_text` and `verbose_name`, commonly used by form and
-    Optional human-readable name for the field -- used by form libraries
+    widget libraries.
 
 
 List fields
@@ -211,9 +214,9 @@ document class as the first argument::
 
 Dictionary Fields
 -----------------
-Often, an embedded document may be used instead of a dictionary -- generally
+Often, an embedded document may be used instead of a dictionary – generally 
-this is recommended as dictionaries don't support validation or custom field
+embedded documents are recommended as dictionaries don’t support validation 
-types. However, sometimes you will not know the structure of what you want to
+or custom field types. However, sometimes you will not know the structure of what you want to
 store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate::
 
     class SurveyResponse(Document):
@@ -313,12 +316,12 @@ reference with a delete rule specification.  A delete rule is specified by
 supplying the :attr:`reverse_delete_rule` attributes on the
 :class:`ReferenceField` definition, like this::
 
-    class Employee(Document):
+    class ProfilePage(Document):
         ...
-        profile_page = ReferenceField('ProfilePage', reverse_delete_rule=mongoengine.NULLIFY)
+        employee = ReferenceField('Employee', reverse_delete_rule=mongoengine.CASCADE)
 
 The declaration in this example means that when an :class:`Employee` object is
-removed, the :class:`ProfilePage` that belongs to that employee is removed as
+removed, the :class:`ProfilePage` that references that employee is removed as
 well.  If a whole batch of employees is removed, all profile pages that are
 linked are removed as well.
 
@@ -401,7 +404,7 @@ MongoEngine allows you to specify that a field should be unique across a
 collection by providing ``unique=True`` to a :class:`~mongoengine.fields.Field`\ 's
 constructor. If you try to save a document that has the same value for a unique
 field as a document that is already in the database, a
-:class:`~mongoengine.OperationError` will be raised. You may also specify
+:class:`~mongoengine.NotUniqueError` will be raised. You may also specify
 multi-field uniqueness constraints by using :attr:`unique_with`, which may be
 either a single field name, or a list or tuple of field names::
 
@@ -445,8 +448,10 @@ A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying
 :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary.
 :attr:`max_documents` is the maximum number of documents that is allowed to be
 stored in the collection, and :attr:`max_size` is the maximum size of the
-collection in bytes. If :attr:`max_size` is not specified and
+collection in bytes. :attr:`max_size` is rounded up to the next multiple of 256
-:attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB).
+by MongoDB internally and mongoengine before. Use also a multiple of 256 to
+avoid confusions. If :attr:`max_size` is not specified and
+:attr:`max_documents` is, :attr:`max_size` defaults to 10485760 bytes (10MB).
 The following example shows a :class:`Log` document that will be limited to
 1000 entries and 2MB of disk space::
 
@@ -463,19 +468,26 @@ You can specify indexes on collections to make querying faster. This is done
 by creating a list of index specifications called :attr:`indexes` in the
 :attr:`~mongoengine.Document.meta` dictionary, where an index specification may
 either be a single field name, a tuple containing multiple field names, or a
-dictionary containing a full index definition. A direction may be specified on
+dictionary containing a full index definition.
-fields by prefixing the field name with a **+** (for ascending) or a **-** sign
+
-(for descending). Note that direction only matters on multi-field indexes.
+A direction may be specified on fields by prefixing the field name with a
-Text indexes may be specified by prefixing the field name with a **$**. ::
+**+** (for ascending) or a **-** sign (for descending). Note that direction
+only matters on multi-field indexes. Text indexes may be specified by prefixing
+the field name with a **$**. Hashed indexes may be specified by prefixing
+the field name with a **#**::
 
     class Page(Document):
+        category = IntField()
         title = StringField()
         rating = StringField()
         created = DateTimeField()
         meta = {
             'indexes': [
                 'title',
+                '$title',  # text index
+                '#title',  # hashed index
                 ('title', '-rating'),
+                ('category', '_cls'),
                 {
                     'fields': ['created'],
                     'expireAfterSeconds': 3600
@@ -530,11 +542,14 @@ There are a few top level defaults for all indexes that can be set::
 :attr:`index_background` (Optional)
     Set the default value for if an index should be indexed in the background
 
+:attr:`index_cls` (Optional)
+    A way to turn off a specific index for _cls.
+
 :attr:`index_drop_dups` (Optional)
     Set the default value for if an index should drop duplicates
 
-:attr:`index_cls` (Optional)
+.. note:: Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning
-    A way to turn off a specific index for _cls.
+    and has no effect
 
 
 Compound Indexes and Indexing sub documents

docs/guide/index.rst

@@ -13,3 +13,4 @@ User Guide
    gridfs
    signals
    text-indexes
+   mongomock

docs/guide/mongomock.rst

@@ -0,0 +1,21 @@
+==============================
+Use mongomock for testing
+==============================
+
+`mongomock <https://github.com/vmalloc/mongomock/>`_ is a package to do just 
+what the name implies, mocking a mongo database.
+
+To use with mongoengine, simply specify mongomock when connecting with 
+mongoengine:
+
+.. code-block:: python
+
+    connect('mongoenginetest', host='mongomock://localhost')
+    conn = get_connection()
+
+or with an alias:
+
+.. code-block:: python
+
+    connect('mongoenginetest', host='mongomock://localhost', alias='testdb')
+    conn = get_connection('testdb')

docs/guide/querying.rst

@@ -39,6 +39,14 @@ syntax::
     # been written by a user whose 'country' field is set to 'uk'
     uk_pages = Page.objects(author__country='uk')
 
+.. note::
+
+   (version **0.9.1+**) if your field name is like mongodb operator name (for example
+   type, lte, lt...) and you want to place it at the end of lookup keyword
+   mongoengine automatically  prepend $ to it. To avoid this use  __ at the end of
+   your lookup keyword. For example if your field name is ``type`` and you want to
+   query by this field you must use ``.objects(user__type__="admin")`` instead of
+   ``.objects(user__type="admin")``
 
 Query operators
 ===============
@@ -138,9 +146,10 @@ The following were added in MongoEngine 0.8 for
         loc.objects(point__near=[40, 5])
         loc.objects(point__near={"type": "Point", "coordinates": [40, 5]})
 
-  You can also set the maximum distance in meters as well::
+  You can also set the maximum and/or the minimum distance in meters as well::
 
         loc.objects(point__near=[40, 5], point__max_distance=1000)
+        loc.objects(point__near=[40, 5], point__min_distance=100)
 
 The older 2D indexes are still supported with the
 :class:`~mongoengine.fields.GeoPointField`:
@@ -160,7 +169,8 @@ The older 2D indexes are still supported with the
 
 * ``max_distance`` -- can be added to your location queries to set a maximum
   distance.
-
+* ``min_distance`` -- can be added to your location queries to set a minimum
+  distance.
 
 Querying lists
 --------------
@@ -227,7 +237,7 @@ is preferred for achieving this::
     # All except for the first 5 people
     users = User.objects[5:]
 
-    # 5 users, starting from the 10th user found
+    # 5 users, starting from the 11th user found
     users = User.objects[10:15]
 
 You may also index the query to retrieve a single result. If an item at that
@@ -255,21 +265,11 @@ no document matches the query, and
 if more than one document matched the query.  These exceptions are merged into
 your document definitions eg: `MyDoc.DoesNotExist`
 
-A variation of this method exists,
+A variation of this method, get_or_create() existed, but it was unsafe. It
-:meth:`~mongoengine.queryset.QuerySet.get_or_create`, that will create a new
+could not be made safe, because there are no transactions in mongoDB. Other
-document with the query arguments if no documents match the query. An
+approaches should be investigated, to ensure you don't accidentally duplicate
-additional keyword argument, :attr:`defaults` may be provided, which will be
+data when using something similar to this method. Therefore it was deprecated
-used as default values for the new document, in the case that it should need
+in 0.8 and removed in 0.10.
-to be created::
-
-    >>> a, created = User.objects.get_or_create(name='User A', defaults={'age': 30})
-    >>> b, created = User.objects.get_or_create(name='User A', defaults={'age': 40})
-    >>> a.name == b.name and a.age == b.age
-    True
-
-.. warning::
-    :meth:`~mongoengine.queryset.QuerySet.get_or_create` method is deprecated
-    since :mod:`mongoengine` 0.8.
 
 Default Document queries
 ========================
@@ -347,6 +347,8 @@ way of achieving this::
 
     num_users = len(User.objects)
 
+Even if len() is the Pythonic way of counting results, keep in mind that if you concerned about performance, :meth:`~mongoengine.queryset.QuerySet.count` is the way to go since it only execute a server side count query, while len() retrieves the results, places them in cache, and finally counts them. If we compare the performance of the two operations, len() is much slower than :meth:`~mongoengine.queryset.QuerySet.count`.
+
 Further aggregation
 -------------------
 You may sum over the values of a specific field on documents using
@@ -598,7 +600,7 @@ Some variables are made available in the scope of the Javascript function:
 
 The following example demonstrates the intended usage of
 :meth:`~mongoengine.queryset.QuerySet.exec_js` by defining a function that sums
-over a field on a document (this functionality is already available throught
+over a field on a document (this functionality is already available through
 :meth:`~mongoengine.queryset.QuerySet.sum` but is shown here for sake of
 example)::
 
@@ -663,4 +665,3 @@ following example shows how the substitutions are made::
         return comments;
     }
     """)
-

docs/guide/text-indexes.rst

@@ -17,7 +17,7 @@ Use the *$* prefix to set a text index, Look the declaration::
       meta = {'indexes': [
           {'fields': ['$title', "$content"],
            'default_language': 'english',
-           'weight': {'title': 10, 'content': 2}
+           'weights': {'title': 10, 'content': 2}
           }
       ]}
 

docs/upgrade.rst

@@ -2,6 +2,16 @@
 Upgrading
 #########
 
+0.9.0
+*****
+
+The 0.8.7 package on pypi was corrupted.  If upgrading from 0.8.7 to 0.9.0 please follow: ::
+
+    pip uninstall pymongo
+    pip uninstall mongoengine
+    pip install pymongo==2.8
+    pip install mongoengine
+
 0.8.7
 *****
 

mongoengine/__init__.py

@@ -1,21 +1,20 @@
-import document
-from document import *
-import fields
-from fields import *
 import connection
 from connection import *
+import document
+from document import *
+import errors
+from errors import *
+import fields
+from fields import *
 import queryset
 from queryset import *
 import signals
 from signals import *
-from errors import *
-import errors
-import django
 
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
 
-VERSION = (0, 9, 0)
+VERSION = (0, 10, 7)
 
 
 def get_version():
@@ -23,4 +22,5 @@ def get_version():
         return '.'.join(map(str, VERSION[:-1])) + VERSION[-1]
     return '.'.join(map(str, VERSION))
 
+
 __version__ = get_version()

mongoengine/base/datastructures.py

@@ -1,6 +1,6 @@
-import weakref
-import functools
 import itertools
+import weakref
+
 from mongoengine.common import _import_class
 from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
 
@@ -21,7 +21,7 @@ class BaseDict(dict):
         if isinstance(instance, (Document, EmbeddedDocument)):
             self._instance = weakref.proxy(instance)
         self._name = name
-        return super(BaseDict, self).__init__(dict_items)
+        super(BaseDict, self).__init__(dict_items)
 
     def __getitem__(self, key, *args, **kwargs):
         value = super(BaseDict, self).__getitem__(key)
@@ -66,7 +66,7 @@ class BaseDict(dict):
 
     def clear(self, *args, **kwargs):
         self._mark_as_changed()
-        return super(BaseDict, self).clear(*args, **kwargs)
+        return super(BaseDict, self).clear()
 
     def pop(self, *args, **kwargs):
         self._mark_as_changed()
@@ -74,7 +74,7 @@ class BaseDict(dict):
 
     def popitem(self, *args, **kwargs):
         self._mark_as_changed()
-        return super(BaseDict, self).popitem(*args, **kwargs)
+        return super(BaseDict, self).popitem()
 
     def setdefault(self, *args, **kwargs):
         self._mark_as_changed()
@@ -125,6 +125,10 @@ class BaseList(list):
             value._instance = self._instance
         return value
 
+    def __iter__(self):
+        for i in xrange(self.__len__()):
+            yield self[i]
+
     def __setitem__(self, key, value, *args, **kwargs):
         if isinstance(key, slice):
             self._mark_as_changed()
@@ -156,6 +160,14 @@ class BaseList(list):
         self = state
         return self
 
+    def __iadd__(self, other):
+        self._mark_as_changed()
+        return super(BaseList, self).__iadd__(other)
+
+    def __imul__(self, other):
+        self._mark_as_changed()
+        return super(BaseList, self).__imul__(other)
+
     def append(self, *args, **kwargs):
         self._mark_as_changed()
         return super(BaseList, self).append(*args, **kwargs)
@@ -178,7 +190,7 @@ class BaseList(list):
 
     def reverse(self, *args, **kwargs):
         self._mark_as_changed()
-        return super(BaseList, self).reverse(*args, **kwargs)
+        return super(BaseList, self).reverse()
 
     def sort(self, *args, **kwargs):
         self._mark_as_changed()
@@ -187,7 +199,9 @@ class BaseList(list):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, '_mark_as_changed'):
             if key:
-                self._instance._mark_as_changed('%s.%s' % (self._name, key))
+                self._instance._mark_as_changed(
+                    '%s.%s' % (self._name, key % len(self))
+                )
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -198,7 +212,7 @@ class EmbeddedDocumentList(BaseList):
     def __match_all(cls, i, kwargs):
         items = kwargs.items()
         return all([
-            getattr(i, k) == v or str(getattr(i, k)) == v for k, v in items
+            getattr(i, k) == v or unicode(getattr(i, k)) == v for k, v in items
         ])
 
     @classmethod
@@ -357,25 +371,31 @@ class StrictDict(object):
     __slots__ = ()
     _special_fields = set(['get', 'pop', 'iteritems', 'items', 'keys', 'create'])
     _classes = {}
+
     def __init__(self, **kwargs):
-        for k,v in kwargs.iteritems():
+        for k, v in kwargs.iteritems():
             setattr(self, k, v)
+
     def __getitem__(self, key):
         key = '_reserved_' + key if key in self._special_fields else key
         try:
             return getattr(self, key)
         except AttributeError:
             raise KeyError(key)
+
     def __setitem__(self, key, value):
         key = '_reserved_' + key if key in self._special_fields else key
         return setattr(self, key, value)
+
     def __contains__(self, key):
         return hasattr(self, key)
+
     def get(self, key, default=None):
         try:
             return self[key]
         except KeyError:
             return default
+
     def pop(self, key, default=None):
         v = self.get(key, default)
         try:
@@ -383,19 +403,29 @@ class StrictDict(object):
         except AttributeError:
             pass
         return v
+
     def iteritems(self):
         for key in self:
             yield key, self[key]
+
     def items(self):
         return [(k, self[k]) for k in iter(self)]
+
+    def iterkeys(self):
+        return iter(self)
+
     def keys(self):
         return list(iter(self))
+
     def __iter__(self):
         return (key for key in self.__slots__ if hasattr(self, key))
+
     def __len__(self):
         return len(list(self.iteritems()))
+
     def __eq__(self, other):
         return self.items() == other.items()
+
     def __neq__(self, other):
         return self.items() != other.items()
 
@@ -406,15 +436,18 @@ class StrictDict(object):
         if allowed_keys not in cls._classes:
             class SpecificStrictDict(cls):
                 __slots__ = allowed_keys_tuple
+
                 def __repr__(self):
-                    return "{%s}" % ', '.join('"{0!s}": {0!r}'.format(k,v) for (k,v) in self.iteritems())
+                    return "{%s}" % ', '.join('"{0!s}": {1!r}'.format(k, v) for k, v in self.items())
+
             cls._classes[allowed_keys] = SpecificStrictDict
         return cls._classes[allowed_keys]
 
 
 class SemiStrictDict(StrictDict):
-    __slots__ = ('_extras')
+    __slots__ = ('_extras', )
     _classes = {}
+
     def __getattr__(self, attr):
         try:
             super(SemiStrictDict, self).__getattr__(attr)
@@ -423,6 +456,7 @@ class SemiStrictDict(StrictDict):
                 return self.__getattribute__('_extras')[attr]
             except KeyError as e:
                 raise AttributeError(e)
+
     def __setattr__(self, attr, value):
         try:
             super(SemiStrictDict, self).__setattr__(attr, value)

mongoengine/base/document.py

@@ -1,29 +1,28 @@
 import copy
-import operator
 import numbers
+import operator
 from collections import Hashable
 from functools import partial
 
-import pymongo
+from bson import ObjectId, json_util
-from bson import json_util, ObjectId
 from bson.dbref import DBRef
 from bson.son import SON
+import pymongo
 
 from mongoengine import signals
-from mongoengine.common import _import_class
+from mongoengine.base.common import ALLOW_INHERITANCE, get_document
-from mongoengine.errors import (ValidationError, InvalidDocumentError,
-                                LookUpError, FieldDoesNotExist)
-from mongoengine.python_support import PY3, txt_type
-
-from mongoengine.base.common import get_document, ALLOW_INHERITANCE
 from mongoengine.base.datastructures import (
     BaseDict,
     BaseList,
     EmbeddedDocumentList,
-    StrictDict,
+    SemiStrictDict,
-    SemiStrictDict
+    StrictDict
 )
 from mongoengine.base.fields import ComplexBaseField
+from mongoengine.common import _import_class
+from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
+                                LookUpError, ValidationError)
+from mongoengine.python_support import PY3, txt_type
 
 __all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
@@ -52,7 +51,7 @@ class BaseDocument(object):
             # We only want named arguments.
             field = iter(self._fields_ordered)
             # If its an automatic id field then skip to the first defined field
-            if self._auto_id_field:
+            if getattr(self, '_auto_id_field', False):
                 next(field)
             for value in args:
                 name = next(field)
@@ -70,15 +69,16 @@ class BaseDocument(object):
 
         signals.pre_init.send(self.__class__, document=self, values=values)
 
-        # Check if there are undefined fields supplied, if so raise an
+        # Check if there are undefined fields supplied to the constructor,
-        # Exception.
+        # if so raise an Exception.
-        if not self._dynamic:
+        if not self._dynamic and (self._meta.get('strict', True) or _created):
-            for var in values.keys():
+            _undefined_fields = set(values.keys()) - set(
-                if var not in self._fields.keys() + ['id', 'pk', '_cls', '_text_score']:
+                self._fields.keys() + ['id', 'pk', '_cls', '_text_score'])
-                    msg = (
+            if _undefined_fields:
-                        "The field '{0}' does not exist on the document '{1}'"
+                msg = (
-                    ).format(var, self._class_name)
+                    "The fields '{0}' do not exist on the document '{1}'"
-                    raise FieldDoesNotExist(msg)
+                ).format(_undefined_fields, self._class_name)
+                raise FieldDoesNotExist(msg)
 
         if self.STRICT and not self._dynamic:
             self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
@@ -86,7 +86,6 @@ class BaseDocument(object):
             self._data = SemiStrictDict.create(
                 allowed_keys=self._fields_ordered)()
 
-        self._data = {}
         self._dynamic_fields = SON()
 
         # Assign default values to instance
@@ -122,7 +121,7 @@ class BaseDocument(object):
                 else:
                     self._data[key] = value
 
-        # Set any get_fieldname_display methods
+        # Set any get_<field>_display methods
         self.__set_field_display()
 
         if self._dynamic:
@@ -150,7 +149,6 @@ class BaseDocument(object):
         # Handle dynamic data only if an initialised dynamic document
         if self._dynamic and not self._dynamic_lock:
 
-            field = None
             if not hasattr(self, name) and not name.startswith('_'):
                 DynamicField = _import_class("DynamicField")
                 field = DynamicField(db_field=name)
@@ -183,8 +181,8 @@ class BaseDocument(object):
         except AttributeError:
             self__initialised = False
         # Check if the user has created a new instance of a class
-        if (self._is_document and self__initialised
+        if (self._is_document and self__initialised and
-                and self__created and name == self._meta['id_field']):
+                self__created and name == self._meta.get('id_field')):
             super(BaseDocument, self).__setattr__('_created', False)
 
         super(BaseDocument, self).__setattr__(name, value)
@@ -206,7 +204,12 @@ class BaseDocument(object):
             if k in data:
                 setattr(self, k, data[k])
         if '_fields_ordered' in data:
-            setattr(type(self), '_fields_ordered', data['_fields_ordered'])
+            if self._dynamic:
+                setattr(self, '_fields_ordered', data['_fields_ordered'])
+            else:
+                _super_fields_ordered = type(self)._fields_ordered
+                setattr(self, '_fields_ordered', _super_fields_ordered)
+
         dynamic_fields = data.get('_dynamic_fields') or SON()
         for k in dynamic_fields.keys():
             setattr(self, k, data["_data"].get(k))
@@ -307,7 +310,7 @@ class BaseDocument(object):
         data = SON()
         data["_id"] = None
         data['_cls'] = self._class_name
-        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+
         # only root fields ['test1.a', 'test2'] => ['test1', 'test2']
         root_fields = set([f.split('.')[0] for f in fields])
 
@@ -322,21 +325,20 @@ class BaseDocument(object):
                 field = self._dynamic_fields.get(field_name)
 
             if value is not None:
+                f_inputs = field.to_mongo.__code__.co_varnames
+                ex_vars = {}
+                if fields and 'fields' in f_inputs:
+                    key = '%s.' % field_name
+                    embedded_fields = [
+                        i.replace(key, '') for i in fields
+                        if i.startswith(key)]
 
-                if isinstance(field, (EmbeddedDocumentField)):
+                    ex_vars['fields'] = embedded_fields
-                    if fields:
-                        key = '%s.' % field_name
-                        embedded_fields = [
-                            i.replace(key, '') for i in fields
-                            if i.startswith(key)]
 
-                    else:
+                if 'use_db_field' in f_inputs:
-                        embedded_fields = []
+                    ex_vars['use_db_field'] = use_db_field
 
-                    value = field.to_mongo(value, use_db_field=use_db_field,
+                value = field.to_mongo(value, **ex_vars)
-                                           fields=embedded_fields)
-                else:
-                    value = field.to_mongo(value)
 
             # Handle self generating fields
             if value is None and field._auto_gen:
@@ -412,10 +414,11 @@ class BaseDocument(object):
 
     def to_json(self, *args, **kwargs):
         """Converts a document to JSON.
-        :param use_db_field: Set to True by default but enables the output of the json structure with the field names and not the mongodb store db_names in case of set to False
+        :param use_db_field: Set to True by default but enables the output of the json structure with the field names
+            and not the mongodb store db_names in case of set to False
         """
         use_db_field = kwargs.pop('use_db_field', True)
-        return json_util.dumps(self.to_mongo(use_db_field),  *args, **kwargs)
+        return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs)
 
     @classmethod
     def from_json(cls, json_data, created=False):
@@ -478,7 +481,19 @@ class BaseDocument(object):
             key = self._db_field_map.get(key, key)
 
         if key not in self._changed_fields:
-            self._changed_fields.append(key)
+            levels, idx = key.split('.'), 1
+            while idx <= len(levels):
+                if '.'.join(levels[:idx]) in self._changed_fields:
+                    break
+                idx += 1
+            else:
+                self._changed_fields.append(key)
+                # remove lower level changed fields
+                level = '.'.join(levels[:idx]) + '.'
+                remove = self._changed_fields.remove
+                for field in self._changed_fields[:]:
+                    if field.startswith(level):
+                        remove(field)
 
     def _clear_changed_fields(self):
         """Using get_changed_fields iterate and remove any fields that are
@@ -530,6 +545,7 @@ class BaseDocument(object):
         EmbeddedDocument = _import_class("EmbeddedDocument")
         DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument")
         ReferenceField = _import_class("ReferenceField")
+        SortedListField = _import_class("SortedListField")
         changed_fields = []
         changed_fields += getattr(self, '_changed_fields', [])
 
@@ -548,12 +564,13 @@ class BaseDocument(object):
             if hasattr(data, 'id'):
                 if data.id in inspected:
                     continue
-                inspected.add(data.id)
             if isinstance(field, ReferenceField):
                 continue
-            elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
+            elif (
-                  and db_field_name not in changed_fields):
+                isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument)) and
-                 # Find all embedded fields that have been changed
+                db_field_name not in changed_fields
+            ):
+                # Find all embedded fields that have been changed
                 changed = data._get_changed_fields(inspected)
                 changed_fields += ["%s%s" % (key, k) for k in changed if k]
             elif (isinstance(data, (list, tuple, dict)) and
@@ -561,6 +578,12 @@ class BaseDocument(object):
                 if (hasattr(field, 'field') and
                         isinstance(field.field, ReferenceField)):
                     continue
+                elif isinstance(field, SortedListField) and field._ordering:
+                    # if ordering is affected whole list is changed
+                    if any(map(lambda d: field._ordering in d._changed_fields, data)):
+                        changed_fields.append(db_field_name)
+                        continue
+
                 self._nestable_types_changed_fields(
                     changed_fields, key, data, inspected)
         return changed_fields
@@ -585,7 +608,9 @@ class BaseDocument(object):
                 for p in parts:
                     if isinstance(d, (ObjectId, DBRef)):
                         break
-                    elif isinstance(d, list) and p.isdigit():
+                    elif isinstance(d, list) and p.lstrip('-').isdigit():
+                        if p[0] == '-':
+                            p = str(len(d) + int(p))
                         try:
                             d = d[int(p)]
                         except IndexError:
@@ -598,18 +623,18 @@ class BaseDocument(object):
         else:
             set_data = doc
             if '_id' in set_data:
-                del(set_data['_id'])
+                del set_data['_id']
 
         # Determine if any changed items were actually unset.
         for path, value in set_data.items():
             if value or isinstance(value, (numbers.Number, bool)):
                 continue
 
-            # If we've set a value that ain't the default value dont unset it.
+            # If we've set a value that ain't the default value don't unset it.
             default = None
             if (self._dynamic and len(parts) and parts[0] in
                     self._dynamic_fields):
-                del(set_data[path])
+                del set_data[path]
                 unset_data[path] = 1
                 continue
             elif path in self._fields:
@@ -619,7 +644,9 @@ class BaseDocument(object):
                 parts = path.split('.')
                 db_field_name = parts.pop()
                 for p in parts:
-                    if isinstance(d, list) and p.isdigit():
+                    if isinstance(d, list) and p.lstrip('-').isdigit():
+                        if p[0] == '-':
+                            p = str(len(d) + int(p))
                         d = d[int(p)]
                     elif (hasattr(d, '__getattribute__') and
                           not isinstance(d, dict)):
@@ -643,13 +670,13 @@ class BaseDocument(object):
             if default != value:
                 continue
 
-            del(set_data[path])
+            del set_data[path]
             unset_data[path] = 1
         return set_data, unset_data
 
     @classmethod
     def _get_collection_name(cls):
-        """Returns the collection name for this class.
+        """Returns the collection name for this class. None for abstract class
         """
         return cls._meta.get('collection', None)
 
@@ -687,14 +714,6 @@ class BaseDocument(object):
                         del data[field.db_field]
                 except (AttributeError, ValueError), e:
                     errors_dict[field_name] = e
-            elif field.default:
-                default = field.default
-                if callable(default):
-                    default = default()
-                if isinstance(default, BaseDocument):
-                    changed_fields.append(field_name)
-                elif not only_fields or field_name in only_fields:
-                    changed_fields.append(field_name)
 
         if errors_dict:
             errors = "\n".join(["%s - %s" % (k, v)
@@ -758,8 +777,12 @@ class BaseDocument(object):
         # Check to see if we need to include _cls
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
-        include_cls = (allow_inheritance and not spec.get('sparse', False) and
+        include_cls = (
-                       spec.get('cls',  True))
+            allow_inheritance and
+            not spec.get('sparse', False) and
+            spec.get('cls', True) and
+            '_cls' not in spec['fields']
+        )
 
         # 733: don't include cls if index_cls is False unless there is an explicit cls with the index
         include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
@@ -772,16 +795,25 @@ class BaseDocument(object):
 
             # ASCENDING from +
             # DESCENDING from -
-            # GEO2D from *
             # TEXT from $
+            # HASHED from #
+            # GEOSPHERE from (
+            # GEOHAYSTACK from )
+            # GEO2D from *
             direction = pymongo.ASCENDING
             if key.startswith("-"):
                 direction = pymongo.DESCENDING
-            elif key.startswith("*"):
-                direction = pymongo.GEO2D
             elif key.startswith("$"):
                 direction = pymongo.TEXT
-            if key.startswith(("+", "-", "*", "$")):
+            elif key.startswith("#"):
+                direction = pymongo.HASHED
+            elif key.startswith("("):
+                direction = pymongo.GEOSPHERE
+            elif key.startswith(")"):
+                direction = pymongo.GEOHAYSTACK
+            elif key.startswith("*"):
+                direction = pymongo.GEO2D
+            if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
                 key = key[1:]
 
             # Use real field name, do it manually because we need field
@@ -789,7 +821,6 @@ class BaseDocument(object):
             parts = key.split('.')
             if parts in (['pk'], ['id'], ['_id']):
                 key = '_id'
-                fields = []
             else:
                 fields = cls._lookup_field(parts)
                 parts = []
@@ -804,15 +835,12 @@ class BaseDocument(object):
             index_list.append((key, direction))
 
         # Don't add cls to a geo index
-        if include_cls and direction is not pymongo.GEO2D:
+        if include_cls and direction not in (
+                pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE):
             index_list.insert(0, ('_cls', 1))
 
         if index_list:
             spec['fields'] = index_list
-        if spec.get('sparse', False) and len(spec['fields']) > 1:
-            raise ValueError(
-                'Sparse indexes can only have one field in them. '
-                'See https://jira.mongodb.org/browse/SERVER-2193')
 
         return spec
 
@@ -902,6 +930,7 @@ class BaseDocument(object):
         """
 
         ListField = _import_class("ListField")
+        DynamicField = _import_class('DynamicField')
 
         if not isinstance(parts, (list, tuple)):
             parts = [parts]
@@ -911,7 +940,6 @@ class BaseDocument(object):
         for field_name in parts:
             # Handle ListField indexing:
             if field_name.isdigit() and isinstance(field, ListField):
-                new_field = field.field
                 fields.append(field_name)
                 continue
 
@@ -923,11 +951,9 @@ class BaseDocument(object):
                 if field_name in cls._fields:
                     field = cls._fields[field_name]
                 elif cls._dynamic:
-                    DynamicField = _import_class('DynamicField')
                     field = DynamicField(db_field=field_name)
                 elif cls._meta.get("allow_inheritance", False) or cls._meta.get("abstract", False):
                     # 744: in case the field is defined in a subclass
-                    field = None
                     for subcls in cls.__subclasses__():
                         try:
                             field = subcls._lookup_field([field_name])[0]
@@ -949,21 +975,20 @@ class BaseDocument(object):
                                       '__'.join(parts))
                 if hasattr(getattr(field, 'field', None), 'lookup_member'):
                     new_field = field.field.lookup_member(field_name)
-                else:
+                elif cls._dynamic and (isinstance(field, DynamicField) or
-                   # Look up subfield on the previous field
+                                       getattr(getattr(field, 'document_type', None), '_dynamic', None)):
-                    new_field = field.lookup_member(field_name)
-                if not new_field and isinstance(field, ComplexBaseField):
-                    if hasattr(field.field, 'document_type') and cls._dynamic \
-                            and field.field.document_type._dynamic:
-                        DynamicField = _import_class('DynamicField')
-                        new_field = DynamicField(db_field=field_name)
-                    else:
-                        fields.append(field_name)
-                        continue
-                elif not new_field and hasattr(field, 'document_type') and cls._dynamic \
-                        and field.document_type._dynamic:
-                    DynamicField = _import_class('DynamicField')
                     new_field = DynamicField(db_field=field_name)
+                else:
+                    # Look up subfield on the previous field or raise
+                    try:
+                        new_field = field.lookup_member(field_name)
+                    except AttributeError:
+                        raise LookUpError('Cannot resolve subfield or operator {} '
+                                          'on the field {}'.format(
+                                              field_name, field.name))
+                if not new_field and isinstance(field, ComplexBaseField):
+                    fields.append(field_name)
+                    continue
                 elif not new_field:
                     raise LookUpError('Cannot resolve field "%s"'
                                       % field_name)
@@ -980,19 +1005,18 @@ class BaseDocument(object):
         return '.'.join(parts)
 
     def __set_field_display(self):
-        """Dynamically set the display value for a field with choices"""
+        """For each field that specifies choices, create a
-        for attr_name, field in self._fields.items():
+        get_<field>_display method.
-            if field.choices:
+        """
-                if self._dynamic:
+        fields_with_choices = [(n, f) for n, f in self._fields.items()
-                    obj = self
+                               if f.choices]
-                else:
+        for attr_name, field in fields_with_choices:
-                    obj = type(self)
+            setattr(self,
-                setattr(obj,
+                    'get_%s_display' % attr_name,
-                        'get_%s_display' % attr_name,
+                    partial(self.__get_field_display, field=field))
-                        partial(self.__get_field_display, field=field))
 
     def __get_field_display(self, field):
-        """Returns the display value for a choice field"""
+        """Return the display value for a choice field"""
         value = getattr(self, field.name)
         if field.choices and isinstance(field.choices[0], (list, tuple)):
             return dict(field.choices).get(value, value)

mongoengine/base/fields.py

@@ -5,20 +5,23 @@ import weakref
 from bson import DBRef, ObjectId, SON
 import pymongo
 
-from mongoengine.common import _import_class
-from mongoengine.errors import ValidationError
-
 from mongoengine.base.common import ALLOW_INHERITANCE
 from mongoengine.base.datastructures import (
     BaseDict, BaseList, EmbeddedDocumentList
 )
+from mongoengine.common import _import_class
+from mongoengine.errors import ValidationError
 
 __all__ = ("BaseField", "ComplexBaseField",
            "ObjectIdField", "GeoJsonBaseField")
 
 
-class BaseField(object):
+UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'pop', 'push',
+                        'push_all', 'pull', 'pull_all', 'add_to_set',
+                        'set_on_insert', 'min', 'max'])
 
+
+class BaseField(object):
     """A base class for fields in a MongoDB document. Instances of this class
     may be added to subclasses of `Document` to define a document's schema.
 
@@ -38,8 +41,8 @@ class BaseField(object):
 
     def __init__(self, db_field=None, name=None, required=False, default=None,
                  unique=False, unique_with=None, primary_key=False,
-                 validation=None, choices=None, verbose_name=None,
+                 validation=None, choices=None, null=False, sparse=False,
-                 help_text=None, null=False, sparse=False):
+                 **kwargs):
         """
         :param db_field: The database field to store this field in
             (defaults to the name of the field)
@@ -57,15 +60,15 @@ class BaseField(object):
             field.  Generally this is deprecated in favour of the
             `FIELD.validate` method
         :param choices: (optional) The valid choices
-        :param verbose_name: (optional)  The verbose name for the field.
-            Designed to be human readable and is often used when generating
-            model forms from the document model.
-        :param help_text: (optional) The help text for this field and is often
-            used when generating model forms from the document model.
         :param null: (optional) Is the field value can be null. If no and there is a default value
             then the default value is set
         :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False`
             means that uniqueness won't be enforced for `None` values
+        :param **kwargs: (optional) Arbitrary indirection-free metadata for
+            this field can be supplied as additional keyword arguments and
+            accessed as attributes of the field. Must not conflict with any
+            existing attributes. Common metadata includes `verbose_name` and
+            `help_text`.
         """
         self.db_field = (db_field or name) if not primary_key else '_id'
 
@@ -79,10 +82,19 @@ class BaseField(object):
         self.primary_key = primary_key
         self.validation = validation
         self.choices = choices
-        self.verbose_name = verbose_name
-        self.help_text = help_text
         self.null = null
         self.sparse = sparse
+        self._owner_document = None
+
+        # Detect and report conflicts between metadata and base properties.
+        conflicts = set(dir(self)) & set(kwargs)
+        if conflicts:
+            raise TypeError("%s already has attribute(s): %s" % (
+                self.__class__.__name__, ', '.join(conflicts)))
+
+        # Assign metadata to the instance
+        # This efficient method is available because no __slots__ are defined.
+        self.__dict__.update(kwargs)
 
         # Adjust the appropriate creation counter, and save our local copy.
         if self.db_field == '_id':
@@ -106,7 +118,7 @@ class BaseField(object):
         """Descriptor for assigning a value to a field in a document.
         """
 
-        # If setting to None and theres a default
+        # If setting to None and there is a default
         # Then set the value to the default value
         if value is None:
             if self.null:
@@ -121,7 +133,7 @@ class BaseField(object):
                 if (self.name not in instance._data or
                         instance._data[self.name] != value):
                     instance._mark_as_changed(self.name)
-            except:
+            except Exception:
                 # Values cant be compared eg: naive and tz datetimes
                 # So mark it as changed
                 instance._mark_as_changed(self.name)
@@ -129,6 +141,10 @@ class BaseField(object):
         EmbeddedDocument = _import_class('EmbeddedDocument')
         if isinstance(value, EmbeddedDocument):
             value._instance = weakref.proxy(instance)
+        elif isinstance(value, (list, tuple)):
+            for v in value:
+                if isinstance(v, EmbeddedDocument):
+                    v._instance = weakref.proxy(instance)
         instance._data[self.name] = value
 
     def error(self, message="", errors=None, field_name=None):
@@ -147,9 +163,24 @@ class BaseField(object):
         """
         return self.to_python(value)
 
+    def _to_mongo_safe_call(self, value, use_db_field=True, fields=None):
+        """A helper method to call to_mongo with proper inputs
+        """
+        f_inputs = self.to_mongo.__code__.co_varnames
+        ex_vars = {}
+        if 'fields' in f_inputs:
+            ex_vars['fields'] = fields
+
+        if 'use_db_field' in f_inputs:
+            ex_vars['use_db_field'] = use_db_field
+
+        return self.to_mongo(value, **ex_vars)
+
     def prepare_query_value(self, op, value):
         """Prepare a value that is being used in a query for PyMongo.
         """
+        if op in UPDATE_OPERATORS:
+            self.validate(value)
         return value
 
     def validate(self, value, clean=True):
@@ -157,26 +188,28 @@ class BaseField(object):
         """
         pass
 
-    def _validate(self, value, **kwargs):
+    def _validate_choices(self, value):
         Document = _import_class('Document')
         EmbeddedDocument = _import_class('EmbeddedDocument')
 
+        choice_list = self.choices
+        if isinstance(choice_list[0], (list, tuple)):
+            choice_list = [k for k, _ in choice_list]
+
+        # Choices which are other types of Documents
+        if isinstance(value, (Document, EmbeddedDocument)):
+            if not any(isinstance(value, c) for c in choice_list):
+                self.error(
+                    'Value must be instance of %s' % unicode(choice_list)
+                )
+        # Choices which are types other than Documents
+        elif value not in choice_list:
+            self.error('Value must be one of %s' % unicode(choice_list))
+
+    def _validate(self, value, **kwargs):
         # Check the Choices Constraint
         if self.choices:
-
+            self._validate_choices(value)
-            choice_list = self.choices
-            if isinstance(self.choices[0], (list, tuple)):
-                choice_list = [k for k, v in self.choices]
-
-            # Choices which are other types of Documents
-            if isinstance(value, (Document, EmbeddedDocument)):
-                if not any(isinstance(value, c) for c in choice_list):
-                    self.error(
-                        'Value must be instance of %s' % unicode(choice_list)
-                    )
-            # Choices which are types other than Documents
-            elif value not in choice_list:
-                self.error('Value must be one of %s' % unicode(choice_list))
 
         # check validation argument
         if self.validation is not None:
@@ -189,9 +222,19 @@ class BaseField(object):
 
         self.validate(value, **kwargs)
 
+    @property
+    def owner_document(self):
+        return self._owner_document
+
+    def _set_owner_document(self, owner_document):
+        self._owner_document = owner_document
+
+    @owner_document.setter
+    def owner_document(self, owner_document):
+        self._set_owner_document(owner_document)
+
 
 class ComplexBaseField(BaseField):
-
     """Handles complex fields, such as lists / dictionaries.
 
     Allows for nesting of embedded documents inside complex types.
@@ -241,8 +284,8 @@ class ComplexBaseField(BaseField):
             instance._data[self.name] = value
 
         if (self._auto_dereference and instance._initialised and
-                isinstance(value, (BaseList, BaseDict))
+                isinstance(value, (BaseList, BaseDict)) and
-                and not value._dereferenced):
+                not value._dereferenced):
             value = _dereference(
                 value, max_depth=1, instance=instance, name=self.name
             )
@@ -254,8 +297,6 @@ class ComplexBaseField(BaseField):
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
         """
-        Document = _import_class('Document')
-
         if isinstance(value, basestring):
             return value
 
@@ -271,9 +312,11 @@ class ComplexBaseField(BaseField):
                 return value
 
         if self.field:
+            self.field._auto_dereference = self._auto_dereference
             value_dict = dict([(key, self.field.to_python(item))
                                for key, item in value.items()])
         else:
+            Document = _import_class('Document')
             value_dict = {}
             for k, v in value.items():
                 if isinstance(v, Document):
@@ -289,11 +332,11 @@ class ComplexBaseField(BaseField):
                     value_dict[k] = self.to_python(v)
 
         if is_list:  # Convert back to a list
-            return [v for k, v in sorted(value_dict.items(),
+            return [v for _, v in sorted(value_dict.items(),
                                          key=operator.itemgetter(0))]
         return value_dict
 
-    def to_mongo(self, value):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB-compatible type.
         """
         Document = _import_class("Document")
@@ -307,9 +350,9 @@ class ComplexBaseField(BaseField):
             if isinstance(value, Document):
                 return GenericReferenceField().to_mongo(value)
             cls = value.__class__
-            val = value.to_mongo()
+            val = value.to_mongo(use_db_field, fields)
-            # If we its a document thats not inherited add _cls
+            # If it's a document that is not inherited add _cls
-            if (isinstance(value, EmbeddedDocument)):
+            if isinstance(value, EmbeddedDocument):
                 val['_cls'] = cls.__name__
             return val
 
@@ -322,7 +365,7 @@ class ComplexBaseField(BaseField):
                 return value
 
         if self.field:
-            value_dict = dict([(key, self.field.to_mongo(item))
+            value_dict = dict([(key, self.field._to_mongo_safe_call(item, use_db_field, fields))
                                for key, item in value.iteritems()])
         else:
             value_dict = {}
@@ -347,16 +390,16 @@ class ComplexBaseField(BaseField):
                         value_dict[k] = DBRef(collection, v.pk)
                 elif hasattr(v, 'to_mongo'):
                     cls = v.__class__
-                    val = v.to_mongo()
+                    val = v.to_mongo(use_db_field, fields)
-                    # If we its a document thats not inherited add _cls
+                    # If it's a document that is not inherited add _cls
-                    if (isinstance(v, (Document, EmbeddedDocument))):
+                    if isinstance(v, (Document, EmbeddedDocument)):
                         val['_cls'] = cls.__name__
                     value_dict[k] = val
                 else:
-                    value_dict[k] = self.to_mongo(v)
+                    value_dict[k] = self.to_mongo(v, use_db_field, fields)
 
         if is_list:  # Convert back to a list
-            return [v for k, v in sorted(value_dict.items(),
+            return [v for _, v in sorted(value_dict.items(),
                                          key=operator.itemgetter(0))]
         return value_dict
 
@@ -398,20 +441,17 @@ class ComplexBaseField(BaseField):
             self.field.owner_document = owner_document
         self._owner_document = owner_document
 
-    def _get_owner_document(self, owner_document):
-        self._owner_document = owner_document
-
-    owner_document = property(_get_owner_document, _set_owner_document)
-
 
 class ObjectIdField(BaseField):
-
     """A field wrapper around MongoDB's ObjectIds.
     """
 
     def to_python(self, value):
-        if not isinstance(value, ObjectId):
+        try:
-            value = ObjectId(value)
+            if not isinstance(value, ObjectId):
+                value = ObjectId(value)
+        except Exception:
+            pass
         return value
 
     def to_mongo(self, value):
@@ -429,12 +469,11 @@ class ObjectIdField(BaseField):
     def validate(self, value):
         try:
             ObjectId(unicode(value))
-        except:
+        except Exception:
             self.error('Invalid Object ID')
 
 
 class GeoJsonBaseField(BaseField):
-
     """A geo json field storing a geojson style object.
 
     .. versionadded:: 0.8
@@ -482,7 +521,7 @@ class GeoJsonBaseField(BaseField):
         # Quick and dirty validator
         try:
             value[0][0][0]
-        except:
+        except (TypeError, IndexError):
             return "Invalid Polygon must contain at least one valid linestring"
 
         errors = []
@@ -506,7 +545,7 @@ class GeoJsonBaseField(BaseField):
         # Quick and dirty validator
         try:
             value[0][0]
-        except:
+        except (TypeError, IndexError):
             return "Invalid LineString must contain at least one valid point"
 
         errors = []
@@ -537,7 +576,7 @@ class GeoJsonBaseField(BaseField):
         # Quick and dirty validator
         try:
             value[0][0]
-        except:
+        except (TypeError, IndexError):
             return "Invalid MultiPoint must contain at least one valid point"
 
         errors = []
@@ -556,7 +595,7 @@ class GeoJsonBaseField(BaseField):
         # Quick and dirty validator
         try:
             value[0][0][0]
-        except:
+        except (TypeError, IndexError):
             return "Invalid MultiLineString must contain at least one valid linestring"
 
         errors = []
@@ -578,7 +617,7 @@ class GeoJsonBaseField(BaseField):
         # Quick and dirty validator
         try:
             value[0][0][0][0]
-        except:
+        except (TypeError, IndexError):
             return "Invalid MultiPolygon must contain at least one valid Polygon"
 
         errors = []

mongoengine/base/metaclasses.py

@@ -1,25 +1,22 @@
 import warnings
 
-import pymongo
+from mongoengine.base.common import ALLOW_INHERITANCE, _document_registry
-
+from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidDocumentError
 from mongoengine.python_support import PY3
 from mongoengine.queryset import (DO_NOTHING, DoesNotExist,
                                   MultipleObjectsReturned,
-                                  QuerySet, QuerySetManager)
+                                  QuerySetManager)
 
-from mongoengine.base.common import _document_registry, ALLOW_INHERITANCE
-from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
 
 __all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass')
 
 
 class DocumentMetaclass(type):
+    """Metaclass for all documents."""
 
-    """Metaclass for all documents.
+    # TODO lower complexity of this method
-    """
-
     def __new__(cls, name, bases, attrs):
         flattened_bases = cls._get_bases(bases)
         super_new = super(DocumentMetaclass, cls).__new__
@@ -113,7 +110,7 @@ class DocumentMetaclass(type):
         for base in flattened_bases:
             if (not getattr(base, '_is_base_cls', True) and
                     not getattr(base, '_meta', {}).get('abstract', True)):
-                # Collate heirarchy for _cls and _subclasses
+                # Collate hierarchy for _cls and _subclasses
                 class_name.append(base.__name__)
 
             if hasattr(base, '_meta'):
@@ -146,7 +143,7 @@ class DocumentMetaclass(type):
         for base in document_bases:
             if _cls not in base._subclasses:
                 base._subclasses += (_cls,)
-            base._types = base._subclasses   # TODO depreciate _types
+            base._types = base._subclasses  # TODO depreciate _types
 
         (Document, EmbeddedDocument, DictField,
          CachedReferenceField) = cls._import_classes()
@@ -165,7 +162,7 @@ class DocumentMetaclass(type):
         # copies __func__ into im_func and __self__ into im_self for
         # classmethod objects in Document derived classes.
         if PY3:
-            for key, val in new_class.__dict__.items():
+            for val in new_class.__dict__.values():
                 if isinstance(val, classmethod):
                     f = val.__get__(new_class)
                     if hasattr(f, '__func__') and not hasattr(f, 'im_func'):
@@ -176,7 +173,8 @@ class DocumentMetaclass(type):
         # Handle delete rules
         for field in new_class._fields.itervalues():
             f = field
-            f.owner_document = new_class
+            if f.owner_document is None:
+                f.owner_document = new_class
             delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING)
             if isinstance(f, CachedReferenceField):
 
@@ -185,7 +183,7 @@ class DocumentMetaclass(type):
                         "CachedReferenceFields is not allowed in EmbeddedDocuments")
                 if not f.document_type:
                     raise InvalidDocumentError(
-                        "Document is not avaiable to sync")
+                        "Document is not available to sync")
 
                 if f.auto_sync:
                     f.start_listener()
@@ -247,11 +245,10 @@ class DocumentMetaclass(type):
         EmbeddedDocument = _import_class('EmbeddedDocument')
         DictField = _import_class('DictField')
         CachedReferenceField = _import_class('CachedReferenceField')
-        return (Document, EmbeddedDocument, DictField, CachedReferenceField)
+        return Document, EmbeddedDocument, DictField, CachedReferenceField
 
 
 class TopLevelDocumentMetaclass(DocumentMetaclass):
-
     """Metaclass for top-level documents (i.e. documents that have their own
     collection in the database.
     """
@@ -261,7 +258,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         super_new = super(TopLevelDocumentMetaclass, cls).__new__
 
         # Set default _meta data if base class, otherwise get user defined meta
-        if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass):
+        if attrs.get('my_metaclass') == TopLevelDocumentMetaclass:
             # defaults
             attrs['_meta'] = {
                 'abstract': True,
@@ -280,7 +277,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             attrs['_meta'].update(attrs.get('meta', {}))
         else:
             attrs['_meta'] = attrs.get('meta', {})
-            # Explictly set abstract to false unless set
+            # Explicitly set abstract to false unless set
             attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False)
             attrs['_is_base_cls'] = False
 
@@ -295,7 +292,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         # Clean up top level meta
         if 'meta' in attrs:
-            del(attrs['meta'])
+            del attrs['meta']
 
         # Find the parent document class
         parent_doc_cls = [b for b in flattened_bases
@@ -304,11 +301,11 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         # Prevent classes setting collection different to their parents
         # If parent wasn't an abstract class
-        if (parent_doc_cls and 'collection' in attrs.get('_meta', {})
+        if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) and
-                and not parent_doc_cls._meta.get('abstract', True)):
+                not parent_doc_cls._meta.get('abstract', True)):
             msg = "Trying to set a collection on a subclass (%s)" % name
             warnings.warn(msg, SyntaxWarning)
-            del(attrs['_meta']['collection'])
+            del attrs['_meta']['collection']
 
         # Ensure abstract documents have abstract bases
         if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'):
@@ -386,15 +383,17 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         new_class._auto_id_field = getattr(parent_doc_cls,
                                            '_auto_id_field', False)
         if not new_class._meta.get('id_field'):
+            # After 0.10, find not existing names, instead of overwriting
+            id_name, id_db_name = cls.get_auto_id_names(new_class)
             new_class._auto_id_field = True
-            new_class._meta['id_field'] = 'id'
+            new_class._meta['id_field'] = id_name
-            new_class._fields['id'] = ObjectIdField(db_field='_id')
+            new_class._fields[id_name] = ObjectIdField(db_field=id_db_name)
-            new_class._fields['id'].name = 'id'
+            new_class._fields[id_name].name = id_name
-            new_class.id = new_class._fields['id']
+            new_class.id = new_class._fields[id_name]
-
+            new_class._db_field_map[id_name] = id_db_name
-        # Prepend id field to _fields_ordered
+            new_class._reverse_db_field_map[id_db_name] = id_name
-        if 'id' in new_class._fields and 'id' not in new_class._fields_ordered:
+            # Prepend id field to _fields_ordered
-            new_class._fields_ordered = ('id', ) + new_class._fields_ordered
+            new_class._fields_ordered = (id_name, ) + new_class._fields_ordered
 
         # Merge in exceptions with parent hierarchy
         exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned)
@@ -409,9 +408,22 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         return new_class
 
+    @classmethod
+    def get_auto_id_names(cls, new_class):
+        id_name, id_db_name = ('id', '_id')
+        if id_name not in new_class._fields and \
+                id_db_name not in (v.db_field for v in new_class._fields.values()):
+            return id_name, id_db_name
+        id_basename, id_db_basename, i = 'auto_id', '_auto_id', 0
+        while id_name in new_class._fields or \
+                id_db_name in (v.db_field for v in new_class._fields.values()):
+            id_name = '{0}_{1}'.format(id_basename, i)
+            id_db_name = '{0}_{1}'.format(id_db_basename, i)
+            i += 1
+        return id_name, id_db_name
+
 
 class MetaDict(dict):
-
     """Custom dictionary for meta classes.
     Handles the merging of set indexes
     """
@@ -426,6 +438,5 @@ class MetaDict(dict):
 
 
 class BasesTuple(tuple):
-
     """Special class to handle introspection of bases tuple in __new__"""
     pass

mongoengine/connection.py

@@ -1,6 +1,5 @@
-import pymongo
+from pymongo import MongoClient, ReadPreference, uri_parser
-from pymongo import MongoClient, MongoReplicaSetClient, uri_parser
+from mongoengine.python_support import (IS_PYMONGO_3, str_types)
-
 
 __all__ = ['ConnectionError', 'connect', 'register_connection',
            'DEFAULT_CONNECTION_NAME']
@@ -8,6 +7,12 @@ __all__ = ['ConnectionError', 'connect', 'register_connection',
 
 DEFAULT_CONNECTION_NAME = 'default'
 
+if IS_PYMONGO_3:
+    READ_PREFERENCE = ReadPreference.PRIMARY
+else:
+    from pymongo import MongoReplicaSetClient
+    READ_PREFERENCE = False
+
 
 class ConnectionError(Exception):
     pass
@@ -19,8 +24,10 @@ _dbs = {}
 
 
 def register_connection(alias, name=None, host=None, port=None,
-                        read_preference=False,
+                        read_preference=READ_PREFERENCE,
-                        username=None, password=None, authentication_source=None,
+                        username=None, password=None,
+                        authentication_source=None,
+                        authentication_mechanism=None,
                         **kwargs):
     """Add a connection.
 
@@ -34,8 +41,14 @@ def register_connection(alias, name=None, host=None, port=None,
     :param username: username to authenticate with
     :param password: password to authenticate with
     :param authentication_source: database to authenticate against
+    :param authentication_mechanism: database authentication mechanisms.
+        By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
+        MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
+    :param is_mock: explicitly use mongomock for this connection
+        (can also be done by using `mongomock://` as db host prefix)
     :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver
 
+    .. versionchanged:: 0.10.6 - added mongomock support
     """
     global _connection_settings
 
@@ -46,20 +59,48 @@ def register_connection(alias, name=None, host=None, port=None,
         'read_preference': read_preference,
         'username': username,
         'password': password,
-        'authentication_source': authentication_source
+        'authentication_source': authentication_source,
+        'authentication_mechanism': authentication_mechanism
     }
 
     # Handle uri style connections
-    if "://" in conn_settings['host']:
+    conn_host = conn_settings['host']
-        uri_dict = uri_parser.parse_uri(conn_settings['host'])
+    # host can be a list or a string, so if string, force to a list
-        conn_settings.update({
+    if isinstance(conn_host, str_types):
-            'name': uri_dict.get('database') or name,
+        conn_host = [conn_host]
-            'username': uri_dict.get('username'),
+
-            'password': uri_dict.get('password'),
+    resolved_hosts = []
-            'read_preference': read_preference,
+    for entity in conn_host:
-        })
+
-        if "replicaSet" in conn_settings['host']:
+        # Handle Mongomock
-            conn_settings['replicaSet'] = True
+        if entity.startswith('mongomock://'):
+            conn_settings['is_mock'] = True
+            # `mongomock://` is not a valid url prefix and must be replaced by `mongodb://`
+            resolved_hosts.append(entity.replace('mongomock://', 'mongodb://', 1))
+
+        # Handle URI style connections, only updating connection params which
+        # were explicitly specified in the URI.
+        elif '://' in entity:
+            uri_dict = uri_parser.parse_uri(entity)
+            resolved_hosts.append(entity)
+
+            if uri_dict.get('database'):
+                conn_settings['name'] = uri_dict.get('database')
+
+            for param in ('read_preference', 'username', 'password'):
+                if uri_dict.get(param):
+                    conn_settings[param] = uri_dict[param]
+
+            uri_options = uri_dict['options']
+            if 'replicaset' in uri_options:
+                conn_settings['replicaSet'] = True
+            if 'authsource' in uri_options:
+                conn_settings['authentication_source'] = uri_options['authsource']
+            if 'authmechanism' in uri_options:
+                conn_settings['authentication_mechanism'] = uri_options['authmechanism']
+        else:
+            resolved_hosts.append(entity)
+    conn_settings['host'] = resolved_hosts
 
     # Deprecated parameters that should not be passed on
     kwargs.pop('slaves', None)
@@ -74,7 +115,7 @@ def disconnect(alias=DEFAULT_CONNECTION_NAME):
     global _dbs
 
     if alias in _connections:
-        get_connection(alias=alias).disconnect()
+        get_connection(alias=alias).close()
         del _connections[alias]
     if alias in _dbs:
         del _dbs[alias]
@@ -98,25 +139,42 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         conn_settings.pop('username', None)
         conn_settings.pop('password', None)
         conn_settings.pop('authentication_source', None)
+        conn_settings.pop('authentication_mechanism', None)
+
+        is_mock = conn_settings.pop('is_mock', None)
+        if is_mock:
+            # Use MongoClient from mongomock
+            try:
+                import mongomock
+            except ImportError:
+                raise RuntimeError('You need mongomock installed '
+                                   'to mock MongoEngine.')
+            connection_class = mongomock.MongoClient
+        else:
+            # Use MongoClient from pymongo
+            connection_class = MongoClient
 
-        connection_class = MongoClient
         if 'replicaSet' in conn_settings:
-            conn_settings['hosts_or_uri'] = conn_settings.pop('host', None)
             # Discard port since it can't be used on MongoReplicaSetClient
             conn_settings.pop('port', None)
             # Discard replicaSet if not base string
             if not isinstance(conn_settings['replicaSet'], basestring):
                 conn_settings.pop('replicaSet', None)
-            connection_class = MongoReplicaSetClient
+            if not IS_PYMONGO_3:
+                connection_class = MongoReplicaSetClient
+                conn_settings['hosts_or_uri'] = conn_settings.pop('host', None)
 
         try:
             connection = None
             # check for shared connections
-            connection_settings_iterator = ((db_alias, settings.copy()) for db_alias, settings in _connection_settings.iteritems())
+            connection_settings_iterator = (
+                (db_alias, settings.copy()) for db_alias, settings in _connection_settings.iteritems())
             for db_alias, connection_settings in connection_settings_iterator:
                 connection_settings.pop('name', None)
                 connection_settings.pop('username', None)
                 connection_settings.pop('password', None)
+                connection_settings.pop('authentication_source', None)
+                connection_settings.pop('authentication_mechanism', None)
                 if conn_settings == connection_settings and _connections.get(db_alias, None):
                     connection = _connections[db_alias]
                     break
@@ -136,11 +194,13 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         conn = get_connection(alias)
         conn_settings = _connection_settings[alias]
         db = conn[conn_settings['name']]
+        auth_kwargs = {'source': conn_settings['authentication_source']}
+        if conn_settings['authentication_mechanism'] is not None:
+            auth_kwargs['mechanism'] = conn_settings['authentication_mechanism']
         # Authenticate if necessary
-        if conn_settings['username'] and conn_settings['password']:
+        if conn_settings['username'] and (conn_settings['password'] or
-            db.authenticate(conn_settings['username'],
+                                          conn_settings['authentication_mechanism'] == 'MONGODB-X509'):
-                            conn_settings['password'],
+            db.authenticate(conn_settings['username'], conn_settings['password'], **auth_kwargs)
-                            source=conn_settings['authentication_source'])
         _dbs[alias] = db
     return _dbs[alias]
 

mongoengine/dereference.py

@@ -1,17 +1,17 @@
 from bson import DBRef, SON
 
-from base import (
+from .base import (
     BaseDict, BaseList, EmbeddedDocumentList,
     TopLevelDocumentMetaclass, get_document
 )
-from fields import (ReferenceField, ListField, DictField, MapField)
+from .connection import get_db
-from connection import get_db
+from .document import Document, EmbeddedDocument
-from queryset import QuerySet
+from .fields import DictField, ListField, MapField, ReferenceField
-from document import Document, EmbeddedDocument
+from .python_support import txt_type
+from .queryset import QuerySet
 
 
 class DeReference(object):
-
     def __call__(self, items, max_depth=1, instance=None, name=None):
         """
         Cheaply dereferences the items to a set depth.
@@ -49,8 +49,8 @@ class DeReference(object):
 
                 if is_list and all([i.__class__ == doc_type for i in items]):
                     return items
-                elif not is_list and all([i.__class__ == doc_type
+                elif not is_list and all(
-                                         for i in items.values()]):
+                        [i.__class__ == doc_type for i in items.values()]):
                     return items
                 elif not field.dbref:
                     if not hasattr(items, 'items'):
@@ -101,25 +101,25 @@ class DeReference(object):
             if isinstance(item, (Document, EmbeddedDocument)):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, (DBRef)):
+                    if isinstance(v, DBRef):
-                        reference_map.setdefault(field.document_type, []).append(v.id)
+                        reference_map.setdefault(field.document_type, set()).add(v.id)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
-                        reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id)
+                        reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                         field_cls = getattr(getattr(field, 'field', None), 'document_type', None)
                         references = self._find_references(v, depth)
                         for key, refs in references.iteritems():
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
-                            reference_map.setdefault(key, []).extend(refs)
+                            reference_map.setdefault(key, set()).update(refs)
-            elif isinstance(item, (DBRef)):
+            elif isinstance(item, DBRef):
-                reference_map.setdefault(item.collection, []).append(item.id)
+                reference_map.setdefault(item.collection, set()).add(item.id)
             elif isinstance(item, (dict, SON)) and '_ref' in item:
-                reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id)
+                reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)
             elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:
                 references = self._find_references(item, depth - 1)
                 for key, refs in references.iteritems():
-                    reference_map.setdefault(key, []).extend(refs)
+                    reference_map.setdefault(key, set()).update(refs)
 
         return reference_map
 
@@ -128,21 +128,25 @@ class DeReference(object):
         """
         object_map = {}
         for collection, dbrefs in self.reference_map.iteritems():
-            keys = object_map.keys()
-            refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys]))
             if hasattr(collection, 'objects'):  # We have a document class for the refs
+                col_name = collection._get_collection_name()
+                refs = [dbref for dbref in dbrefs
+                        if (col_name, dbref) not in object_map]
                 references = collection.objects.in_bulk(refs)
                 for key, doc in references.iteritems():
-                    object_map[key] = doc
+                    object_map[(col_name, key)] = doc
             else:  # Generic reference: use the refs data to convert to document
                 if isinstance(doc_type, (ListField, DictField, MapField,)):
                     continue
 
+                refs = [dbref for dbref in dbrefs
+                        if (collection, dbref) not in object_map]
+
                 if doc_type:
                     references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
                         doc = doc_type._from_son(ref)
-                        object_map[doc.id] = doc
+                        object_map[(collection, doc.id)] = doc
                 else:
                     references = get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
@@ -151,10 +155,10 @@ class DeReference(object):
                         elif doc_type is None:
                             doc = get_document(
                                 ''.join(x.capitalize()
-                                    for x in collection.split('_')))._from_son(ref)
+                                        for x in collection.split('_')))._from_son(ref)
                         else:
                             doc = doc_type._from_son(ref)
-                        object_map[doc.id] = doc
+                        object_map[(collection, doc.id)] = doc
         return object_map
 
     def _attach_objects(self, items, depth=0, instance=None, name=None):
@@ -180,7 +184,8 @@ class DeReference(object):
 
         if isinstance(items, (dict, SON)):
             if '_ref' in items:
-                return self.object_map.get(items['_ref'].id, items)
+                return self.object_map.get(
+                    (items['_ref'].collection, items['_ref'].id), items)
             elif '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
                 _cls = doc._data.pop('_cls', None)
@@ -215,19 +220,20 @@ class DeReference(object):
             elif isinstance(v, (Document, EmbeddedDocument)):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
-                    if isinstance(v, (DBRef)):
+                    if isinstance(v, DBRef):
-                        data[k]._data[field_name] = self.object_map.get(v.id, v)
+                        data[k]._data[field_name] = self.object_map.get(
+                            (v.collection, v.id), v)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
-                        data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
+                        data[k]._data[field_name] = self.object_map.get(
-                    elif isinstance(v, dict) and depth <= self.max_depth:
+                            (v['_ref'].collection, v['_ref'].id), v)
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                    elif isinstance(v, (list, tuple)) and depth <= self.max_depth:
+                        item_name = txt_type("{0}.{1}.{2}").format(name, k, field_name)
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                 item_name = '%s.%s' % (name, k) if name else name
                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
             elif hasattr(v, 'id'):
-                data[k] = self.object_map.get(v.id, v)
+                data[k] = self.object_map.get((v.collection, v.id), v)
 
         if instance and name:
             if is_list:

mongoengine/django/auth.py

@@ -1,412 +0,0 @@
-from mongoengine import *
-
-from django.utils.encoding import smart_str
-from django.contrib.auth.models import _user_has_perm, _user_get_all_permissions, _user_has_module_perms
-from django.db import models
-from django.contrib.contenttypes.models import ContentTypeManager
-from django.contrib import auth
-from django.contrib.auth.models import AnonymousUser
-from django.utils.translation import ugettext_lazy as _
-
-from .utils import datetime_now
-
-REDIRECT_FIELD_NAME = 'next'
-
-try:
-    from django.contrib.auth.hashers import check_password, make_password
-except ImportError:
-    """Handle older versions of Django"""
-    from django.utils.hashcompat import md5_constructor, sha_constructor
-
-    def get_hexdigest(algorithm, salt, raw_password):
-        raw_password, salt = smart_str(raw_password), smart_str(salt)
-        if algorithm == 'md5':
-            return md5_constructor(salt + raw_password).hexdigest()
-        elif algorithm == 'sha1':
-            return sha_constructor(salt + raw_password).hexdigest()
-        raise ValueError('Got unknown password algorithm type in password')
-
-    def check_password(raw_password, password):
-        algo, salt, hash = password.split('$')
-        return hash == get_hexdigest(algo, salt, raw_password)
-
-    def make_password(raw_password):
-        from random import random
-        algo = 'sha1'
-        salt = get_hexdigest(algo, str(random()), str(random()))[:5]
-        hash = get_hexdigest(algo, salt, raw_password)
-        return '%s$%s$%s' % (algo, salt, hash)
-
-
-class ContentType(Document):
-    name = StringField(max_length=100)
-    app_label = StringField(max_length=100)
-    model = StringField(max_length=100, verbose_name=_('python model class name'),
-                        unique_with='app_label')
-    objects = ContentTypeManager()
-
-    class Meta:
-        verbose_name = _('content type')
-        verbose_name_plural = _('content types')
-        # db_table = 'django_content_type'
-        # ordering = ('name',)
-        # unique_together = (('app_label', 'model'),)
-
-    def __unicode__(self):
-        return self.name
-
-    def model_class(self):
-        "Returns the Python model class for this type of content."
-        from django.db import models
-        return models.get_model(self.app_label, self.model)
-
-    def get_object_for_this_type(self, **kwargs):
-        """
-        Returns an object of this type for the keyword arguments given.
-        Basically, this is a proxy around this object_type's get_object() model
-        method. The ObjectNotExist exception, if thrown, will not be caught,
-        so code that calls this method should catch it.
-        """
-        return self.model_class()._default_manager.using(self._state.db).get(**kwargs)
-
-    def natural_key(self):
-        return (self.app_label, self.model)
-
-
-class SiteProfileNotAvailable(Exception):
-    pass
-
-
-class PermissionManager(models.Manager):
-    def get_by_natural_key(self, codename, app_label, model):
-        return self.get(
-            codename=codename,
-            content_type=ContentType.objects.get_by_natural_key(app_label, model)
-        )
-
-
-class Permission(Document):
-    """The permissions system provides a way to assign permissions to specific
-    users and groups of users.
-
-    The permission system is used by the Django admin site, but may also be
-    useful in your own code. The Django admin site uses permissions as follows:
-
-        - The "add" permission limits the user's ability to view the "add"
-          form and add an object.
-        - The "change" permission limits a user's ability to view the change
-          list, view the "change" form and change an object.
-        - The "delete" permission limits the ability to delete an object.
-
-    Permissions are set globally per type of object, not per specific object
-    instance. It is possible to say "Mary may change news stories," but it's
-    not currently possible to say "Mary may change news stories, but only the
-    ones she created herself" or "Mary may only change news stories that have
-    a certain status or publication date."
-
-    Three basic permissions -- add, change and delete -- are automatically
-    created for each Django model.
-    """
-    name = StringField(max_length=50, verbose_name=_('username'))
-    content_type = ReferenceField(ContentType)
-    codename = StringField(max_length=100, verbose_name=_('codename'))
-        # FIXME: don't access field of the other class
-        # unique_with=['content_type__app_label', 'content_type__model'])
-
-    objects = PermissionManager()
-
-    class Meta:
-        verbose_name = _('permission')
-        verbose_name_plural = _('permissions')
-        # unique_together = (('content_type', 'codename'),)
-        # ordering = ('content_type__app_label', 'content_type__model', 'codename')
-
-    def __unicode__(self):
-        return u"%s | %s | %s" % (
-            unicode(self.content_type.app_label),
-            unicode(self.content_type),
-            unicode(self.name))
-
-    def natural_key(self):
-        return (self.codename,) + self.content_type.natural_key()
-    natural_key.dependencies = ['contenttypes.contenttype']
-
-
-class Group(Document):
-    """Groups are a generic way of categorizing users to apply permissions,
-    or some other label, to those users. A user can belong to any number of
-    groups.
-
-    A user in a group automatically has all the permissions granted to that
-    group. For example, if the group Site editors has the permission
-    can_edit_home_page, any user in that group will have that permission.
-
-    Beyond permissions, groups are a convenient way to categorize users to
-    apply some label, or extended functionality, to them. For example, you
-    could create a group 'Special users', and you could write code that would
-    do special things to those users -- such as giving them access to a
-    members-only portion of your site, or sending them members-only
-    e-mail messages.
-    """
-    name = StringField(max_length=80, unique=True, verbose_name=_('name'))
-    permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False))
-
-    class Meta:
-        verbose_name = _('group')
-        verbose_name_plural = _('groups')
-
-    def __unicode__(self):
-        return self.name
-
-
-class UserManager(models.Manager):
-    def create_user(self, username, email, password=None):
-        """
-        Creates and saves a User with the given username, e-mail and password.
-        """
-        now = datetime_now()
-
-        # Normalize the address by lowercasing the domain part of the email
-        # address.
-        try:
-            email_name, domain_part = email.strip().split('@', 1)
-        except ValueError:
-            pass
-        else:
-            email = '@'.join([email_name, domain_part.lower()])
-
-        user = self.model(username=username, email=email, is_staff=False,
-                          is_active=True, is_superuser=False, last_login=now,
-                          date_joined=now)
-
-        user.set_password(password)
-        user.save(using=self._db)
-        return user
-
-    def create_superuser(self, username, email, password):
-        u = self.create_user(username, email, password)
-        u.is_staff = True
-        u.is_active = True
-        u.is_superuser = True
-        u.save(using=self._db)
-        return u
-
-    def make_random_password(self, length=10, allowed_chars='abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'):
-        "Generates a random password with the given length and given allowed_chars"
-        # Note that default value of allowed_chars does not have "I" or letters
-        # that look like it -- just to avoid confusion.
-        from random import choice
-        return ''.join([choice(allowed_chars) for i in range(length)])
-
-
-class User(Document):
-    """A User document that aims to mirror most of the API specified by Django
-    at http://docs.djangoproject.com/en/dev/topics/auth/#users
-    """
-    username = StringField(max_length=30, required=True,
-                           verbose_name=_('username'),
-                           help_text=_("Required. 30 characters or fewer. Letters, numbers and @/./+/-/_ characters"))
-
-    first_name = StringField(max_length=30,
-                             verbose_name=_('first name'))
-
-    last_name = StringField(max_length=30,
-                            verbose_name=_('last name'))
-    email = EmailField(verbose_name=_('e-mail address'))
-    password = StringField(max_length=128,
-                           verbose_name=_('password'),
-                           help_text=_("Use '[algo]$[iterations]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."))
-    is_staff = BooleanField(default=False,
-                            verbose_name=_('staff status'),
-                            help_text=_("Designates whether the user can log into this admin site."))
-    is_active = BooleanField(default=True,
-                             verbose_name=_('active'),
-                             help_text=_("Designates whether this user should be treated as active. Unselect this instead of deleting accounts."))
-    is_superuser = BooleanField(default=False,
-                                verbose_name=_('superuser status'),
-                                help_text=_("Designates that this user has all permissions without explicitly assigning them."))
-    last_login = DateTimeField(default=datetime_now,
-                               verbose_name=_('last login'))
-    date_joined = DateTimeField(default=datetime_now,
-                                verbose_name=_('date joined'))
-
-    user_permissions = ListField(ReferenceField(Permission), verbose_name=_('user permissions'),
-                                                help_text=_('Permissions for the user.'))
-
-    USERNAME_FIELD = 'username'
-    REQUIRED_FIELDS = ['email']
-
-    meta = {
-        'allow_inheritance': True,
-        'indexes': [
-            {'fields': ['username'], 'unique': True, 'sparse': True}
-        ]
-    }
-
-    def __unicode__(self):
-        return self.username
-
-    def get_full_name(self):
-        """Returns the users first and last names, separated by a space.
-        """
-        full_name = u'%s %s' % (self.first_name or '', self.last_name or '')
-        return full_name.strip()
-
-    def is_anonymous(self):
-        return False
-
-    def is_authenticated(self):
-        return True
-
-    def set_password(self, raw_password):
-        """Sets the user's password - always use this rather than directly
-        assigning to :attr:`~mongoengine.django.auth.User.password` as the
-        password is hashed before storage.
-        """
-        self.password = make_password(raw_password)
-        self.save()
-        return self
-
-    def check_password(self, raw_password):
-        """Checks the user's password against a provided password - always use
-        this rather than directly comparing to
-        :attr:`~mongoengine.django.auth.User.password` as the password is
-        hashed before storage.
-        """
-        return check_password(raw_password, self.password)
-
-    @classmethod
-    def create_user(cls, username, password, email=None):
-        """Create (and save) a new user with the given username, password and
-        email address.
-        """
-        now = datetime_now()
-
-        # Normalize the address by lowercasing the domain part of the email
-        # address.
-        if email is not None:
-            try:
-                email_name, domain_part = email.strip().split('@', 1)
-            except ValueError:
-                pass
-            else:
-                email = '@'.join([email_name, domain_part.lower()])
-
-        user = cls(username=username, email=email, date_joined=now)
-        user.set_password(password)
-        user.save()
-        return user
-
-    def get_group_permissions(self, obj=None):
-        """
-        Returns a list of permission strings that this user has through his/her
-        groups. This method queries all available auth backends. If an object
-        is passed in, only permissions matching this object are returned.
-        """
-        permissions = set()
-        for backend in auth.get_backends():
-            if hasattr(backend, "get_group_permissions"):
-                permissions.update(backend.get_group_permissions(self, obj))
-        return permissions
-
-    def get_all_permissions(self, obj=None):
-        return _user_get_all_permissions(self, obj)
-
-    def has_perm(self, perm, obj=None):
-        """
-        Returns True if the user has the specified permission. This method
-        queries all available auth backends, but returns immediately if any
-        backend returns True. Thus, a user who has permission from a single
-        auth backend is assumed to have permission in general. If an object is
-        provided, permissions for this specific object are checked.
-        """
-
-        # Active superusers have all permissions.
-        if self.is_active and self.is_superuser:
-            return True
-
-        # Otherwise we need to check the backends.
-        return _user_has_perm(self, perm, obj)
-
-    def has_module_perms(self, app_label):
-        """
-        Returns True if the user has any permissions in the given app label.
-        Uses pretty much the same logic as has_perm, above.
-        """
-        # Active superusers have all permissions.
-        if self.is_active and self.is_superuser:
-            return True
-
-        return _user_has_module_perms(self, app_label)
-
-    def email_user(self, subject, message, from_email=None):
-        "Sends an e-mail to this User."
-        from django.core.mail import send_mail
-        send_mail(subject, message, from_email, [self.email])
-
-    def get_profile(self):
-        """
-        Returns site-specific profile for this user. Raises
-        SiteProfileNotAvailable if this site does not allow profiles.
-        """
-        if not hasattr(self, '_profile_cache'):
-            from django.conf import settings
-            if not getattr(settings, 'AUTH_PROFILE_MODULE', False):
-                raise SiteProfileNotAvailable('You need to set AUTH_PROFILE_MO'
-                                              'DULE in your project settings')
-            try:
-                app_label, model_name = settings.AUTH_PROFILE_MODULE.split('.')
-            except ValueError:
-                raise SiteProfileNotAvailable('app_label and model_name should'
-                        ' be separated by a dot in the AUTH_PROFILE_MODULE set'
-                        'ting')
-
-            try:
-                model = models.get_model(app_label, model_name)
-                if model is None:
-                    raise SiteProfileNotAvailable('Unable to load the profile '
-                        'model, check AUTH_PROFILE_MODULE in your project sett'
-                        'ings')
-                self._profile_cache = model._default_manager.using(self._state.db).get(user__id__exact=self.id)
-                self._profile_cache.user = self
-            except (ImportError, ImproperlyConfigured):
-                raise SiteProfileNotAvailable
-        return self._profile_cache
-
-
-class MongoEngineBackend(object):
-    """Authenticate using MongoEngine and mongoengine.django.auth.User.
-    """
-
-    supports_object_permissions = False
-    supports_anonymous_user = False
-    supports_inactive_user = False
-    _user_doc = False
-
-    def authenticate(self, username=None, password=None):
-        user = self.user_document.objects(username=username).first()
-        if user:
-            if password and user.check_password(password):
-                backend = auth.get_backends()[0]
-                user.backend = "%s.%s" % (backend.__module__, backend.__class__.__name__)
-                return user
-        return None
-
-    def get_user(self, user_id):
-        return self.user_document.objects.with_id(user_id)
-
-    @property
-    def user_document(self):
-        if self._user_doc is False:
-            from .mongo_auth.models import get_user_document
-            self._user_doc = get_user_document()
-        return self._user_doc
-
-def get_user(userid):
-    """Returns a User object from an id (User.id). Django's equivalent takes
-    request, but taking an id instead leaves it up to the developer to store
-    the id in any way they want (session, signed cookie, etc.)
-    """
-    if not userid:
-        return AnonymousUser()
-    return MongoEngineBackend().get_user(userid) or AnonymousUser()

mongoengine/django/mongo_auth/models.py

@@ -1,119 +0,0 @@
-from django.conf import settings
-from django.contrib.auth.hashers import make_password
-from django.contrib.auth.models import UserManager
-from django.core.exceptions import ImproperlyConfigured
-from django.db import models
-try:
-    from django.utils.module_loading import import_module
-except ImportError:
-    """Handle older versions of Django"""
-    from django.utils.importlib import import_module
-from django.utils.translation import ugettext_lazy as _
-
-
-__all__ = (
-    'get_user_document',
-)
-
-
-MONGOENGINE_USER_DOCUMENT = getattr(
-    settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User')
-
-
-def get_user_document():
-    """Get the user document class used for authentication.
-
-    This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which
-    defaults to `mongoengine.django.auth.User`.
-
-    """
-
-    name = MONGOENGINE_USER_DOCUMENT
-    dot = name.rindex('.')
-    module = import_module(name[:dot])
-    return getattr(module, name[dot + 1:])
-
-
-class MongoUserManager(UserManager):
-    """A User manager wich allows the use of MongoEngine documents in Django.
-
-    To use the manager, you must tell django.contrib.auth to use MongoUser as
-    the user model. In you settings.py, you need:
-
-        INSTALLED_APPS = (
-            ...
-            'django.contrib.auth',
-            'mongoengine.django.mongo_auth',
-            ...
-        )
-        AUTH_USER_MODEL = 'mongo_auth.MongoUser'
-
-    Django will use the model object to access the custom Manager, which will
-    replace the original queryset with MongoEngine querysets.
-
-    By default, mongoengine.django.auth.User will be used to store users. You
-    can specify another document class in MONGOENGINE_USER_DOCUMENT in your
-    settings.py.
-
-    The User Document class has the same requirements as a standard custom user
-    model: https://docs.djangoproject.com/en/dev/topics/auth/customizing/
-
-    In particular, the User Document class must define USERNAME_FIELD and
-    REQUIRED_FIELDS.
-
-    `AUTH_USER_MODEL` has been added in Django 1.5.
-
-    """
-
-    def contribute_to_class(self, model, name):
-        super(MongoUserManager, self).contribute_to_class(model, name)
-        self.dj_model = self.model
-        self.model = get_user_document()
-
-        self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD
-        username = models.CharField(_('username'), max_length=30, unique=True)
-        username.contribute_to_class(self.dj_model, self.dj_model.USERNAME_FIELD)
-
-        self.dj_model.REQUIRED_FIELDS = self.model.REQUIRED_FIELDS
-        for name in self.dj_model.REQUIRED_FIELDS:
-            field = models.CharField(_(name), max_length=30)
-            field.contribute_to_class(self.dj_model, name)
-
-
-    def get(self, *args, **kwargs):
-        try:
-            return self.get_query_set().get(*args, **kwargs)
-        except self.model.DoesNotExist:
-            # ModelBackend expects this exception
-            raise self.dj_model.DoesNotExist
-
-    @property
-    def db(self):
-        raise NotImplementedError
-
-    def get_empty_query_set(self):
-        return self.model.objects.none()
-
-    def get_query_set(self):
-        return self.model.objects
-
-
-class MongoUser(models.Model):
-    """"Dummy user model for Django.
-
-    MongoUser is used to replace Django's UserManager with MongoUserManager.
-    The actual user document class is mongoengine.django.auth.User or any
-    other document class specified in MONGOENGINE_USER_DOCUMENT.
-
-    To get the user document class, use `get_user_document()`.
-
-    """
-
-    objects = MongoUserManager()
-
-    class Meta:
-        app_label = 'mongo_auth'
-
-    def set_password(self, password):
-        """Doesn't do anything, but works around the issue with Django 1.6."""
-        make_password(password)

mongoengine/django/sessions.py

@@ -1,124 +0,0 @@
-from bson import json_util
-from django.conf import settings
-from django.contrib.sessions.backends.base import SessionBase, CreateError
-from django.core.exceptions import SuspiciousOperation
-try:
-    from django.utils.encoding import force_unicode
-except ImportError:
-    from django.utils.encoding import force_text as force_unicode
-
-from mongoengine.document import Document
-from mongoengine import fields
-from mongoengine.queryset import OperationError
-from mongoengine.connection import DEFAULT_CONNECTION_NAME
-
-from .utils import datetime_now
-
-
-MONGOENGINE_SESSION_DB_ALIAS = getattr(
-    settings, 'MONGOENGINE_SESSION_DB_ALIAS',
-    DEFAULT_CONNECTION_NAME)
-
-# a setting for the name of the collection used to store sessions
-MONGOENGINE_SESSION_COLLECTION = getattr(
-    settings, 'MONGOENGINE_SESSION_COLLECTION',
-    'django_session')
-
-# a setting for whether session data is stored encoded or not
-MONGOENGINE_SESSION_DATA_ENCODE = getattr(
-    settings, 'MONGOENGINE_SESSION_DATA_ENCODE',
-    True)
-
-
-class MongoSession(Document):
-    session_key = fields.StringField(primary_key=True, max_length=40)
-    session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \
-                                        else fields.DictField()
-    expire_date = fields.DateTimeField()
-
-    meta = {
-        'collection': MONGOENGINE_SESSION_COLLECTION,
-        'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
-        'allow_inheritance': False,
-        'indexes': [
-            {
-                'fields': ['expire_date'],
-                'expireAfterSeconds': 0
-            }
-        ]
-    }
-
-    def get_decoded(self):
-        return SessionStore().decode(self.session_data)
-
-
-class SessionStore(SessionBase):
-    """A MongoEngine-based session store for Django.
-    """
-
-    def _get_session(self, *args, **kwargs):
-        sess = super(SessionStore, self)._get_session(*args, **kwargs)
-        if sess.get('_auth_user_id', None):
-            sess['_auth_user_id'] = str(sess.get('_auth_user_id'))
-        return sess
-
-    def load(self):
-        try:
-            s = MongoSession.objects(session_key=self.session_key,
-                                     expire_date__gt=datetime_now)[0]
-            if MONGOENGINE_SESSION_DATA_ENCODE:
-                return self.decode(force_unicode(s.session_data))
-            else:
-                return s.session_data
-        except (IndexError, SuspiciousOperation):
-            self.create()
-            return {}
-
-    def exists(self, session_key):
-        return bool(MongoSession.objects(session_key=session_key).first())
-
-    def create(self):
-        while True:
-            self._session_key = self._get_new_session_key()
-            try:
-                self.save(must_create=True)
-            except CreateError:
-                continue
-            self.modified = True
-            self._session_cache = {}
-            return
-
-    def save(self, must_create=False):
-        if self.session_key is None:
-            self._session_key = self._get_new_session_key()
-        s = MongoSession(session_key=self.session_key)
-        if MONGOENGINE_SESSION_DATA_ENCODE:
-            s.session_data = self.encode(self._get_session(no_load=must_create))
-        else:
-            s.session_data = self._get_session(no_load=must_create)
-        s.expire_date = self.get_expiry_date()
-        try:
-            s.save(force_insert=must_create)
-        except OperationError:
-            if must_create:
-                raise CreateError
-            raise
-
-    def delete(self, session_key=None):
-        if session_key is None:
-            if self.session_key is None:
-                return
-            session_key = self.session_key
-        MongoSession.objects(session_key=session_key).delete()
-
-
-class BSONSerializer(object):
-    """
-    Serializer that can handle BSON types (eg ObjectId).
-    """
-    def dumps(self, obj):
-        return json_util.dumps(obj, separators=(',', ':')).encode('ascii')
-
-    def loads(self, data):
-        return json_util.loads(data.decode('ascii'))
-

mongoengine/django/shortcuts.py

@@ -1,47 +0,0 @@
-from mongoengine.queryset import QuerySet
-from mongoengine.base import BaseDocument
-from mongoengine.errors import ValidationError
-
-def _get_queryset(cls):
-    """Inspired by django.shortcuts.*"""
-    if isinstance(cls, QuerySet):
-        return cls
-    else:
-        return cls.objects
-
-def get_document_or_404(cls, *args, **kwargs):
-    """
-    Uses get() to return an document, or raises a Http404 exception if the document
-    does not exist.
-
-    cls may be a Document or QuerySet object. All other passed
-    arguments and keyword arguments are used in the get() query.
-
-    Note: Like with get(), an MultipleObjectsReturned will be raised if more than one
-    object is found.
-
-    Inspired by django.shortcuts.*
-    """
-    queryset = _get_queryset(cls)
-    try:
-        return queryset.get(*args, **kwargs)
-    except (queryset._document.DoesNotExist, ValidationError):
-        from django.http import Http404
-        raise Http404('No %s matches the given query.' % queryset._document._class_name)
-
-def get_list_or_404(cls, *args, **kwargs):
-    """
-    Uses filter() to return a list of documents, or raise a Http404 exception if
-    the list is empty.
-
-    cls may be a Document or QuerySet object. All other passed
-    arguments and keyword arguments are used in the filter() query.
-
-    Inspired by django.shortcuts.*
-    """
-    queryset = _get_queryset(cls)
-    obj_list = list(queryset.filter(*args, **kwargs))
-    if not obj_list:
-        from django.http import Http404
-        raise Http404('No %s matches the given query.' % queryset._document._class_name)
-    return obj_list

mongoengine/django/storage.py

@@ -1,112 +0,0 @@
-import os
-import itertools
-import urlparse
-
-from mongoengine import *
-from django.conf import settings
-from django.core.files.storage import Storage
-from django.core.exceptions import ImproperlyConfigured
-
-
-class FileDocument(Document):
-    """A document used to store a single file in GridFS.
-    """
-    file = FileField()
-
-
-class GridFSStorage(Storage):
-    """A custom storage backend to store files in GridFS
-    """
-
-    def __init__(self, base_url=None):
-
-        if base_url is None:
-            base_url = settings.MEDIA_URL
-        self.base_url = base_url
-        self.document = FileDocument
-        self.field = 'file'
-
-    def delete(self, name):
-        """Deletes the specified file from the storage system.
-        """
-        if self.exists(name):
-            doc = self.document.objects.first()
-            field = getattr(doc, self.field)
-            self._get_doc_with_name(name).delete()  # Delete the FileField
-            field.delete()                          # Delete the FileDocument
-
-    def exists(self, name):
-        """Returns True if a file referened by the given name already exists in the
-        storage system, or False if the name is available for a new file.
-        """
-        doc = self._get_doc_with_name(name)
-        if doc:
-            field = getattr(doc, self.field)
-            return bool(field.name)
-        else:
-            return False
-
-    def listdir(self, path=None):
-        """Lists the contents of the specified path, returning a 2-tuple of lists;
-        the first item being directories, the second item being files.
-        """
-        def name(doc):
-            return getattr(doc, self.field).name
-        docs = self.document.objects
-        return [], [name(d) for d in docs if name(d)]
-
-    def size(self, name):
-        """Returns the total size, in bytes, of the file specified by name.
-        """
-        doc = self._get_doc_with_name(name)
-        if doc:
-            return getattr(doc, self.field).length
-        else:
-            raise ValueError("No such file or directory: '%s'" % name)
-
-    def url(self, name):
-        """Returns an absolute URL where the file's contents can be accessed
-        directly by a web browser.
-        """
-        if self.base_url is None:
-            raise ValueError("This file is not accessible via a URL.")
-        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
-
-    def _get_doc_with_name(self, name):
-        """Find the documents in the store with the given name
-        """
-        docs = self.document.objects
-        doc = [d for d in docs if hasattr(getattr(d, self.field), 'name') and getattr(d, self.field).name == name]
-        if doc:
-            return doc[0]
-        else:
-            return None
-
-    def _open(self, name, mode='rb'):
-        doc = self._get_doc_with_name(name)
-        if doc:
-            return getattr(doc, self.field)
-        else:
-            raise ValueError("No file found with the name '%s'." % name)
-
-    def get_available_name(self, name):
-        """Returns a filename that's free on the target storage system, and
-        available for new content to be written to.
-        """
-        file_root, file_ext = os.path.splitext(name)
-        # If the filename already exists, add an underscore and a number (before
-        # the file extension, if one exists) to the filename until the generated
-        # filename doesn't exist.
-        count = itertools.count(1)
-        while self.exists(name):
-            # file_ext includes the dot.
-            name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
-
-        return name
-
-    def _save(self, name, content):
-        doc = self.document()
-        getattr(doc, self.field).put(content, filename=name)
-        doc.save()
-
-        return name

mongoengine/django/tests.py

@@ -1,31 +0,0 @@
-#coding: utf-8
-
-from unittest import TestCase
-
-from mongoengine import connect
-from mongoengine.connection import get_db
-
-
-class MongoTestCase(TestCase):
-    """
-    TestCase class that clear the collection between the tests
-    """
-
-    @property
-    def db_name(self):
-        from django.conf import settings
-        return 'test_%s' % getattr(settings, 'MONGO_DATABASE_NAME', 'dummy')
-
-    def __init__(self, methodName='runtest'):
-        connect(self.db_name)
-        self.db = get_db()
-        super(MongoTestCase, self).__init__(methodName)
-
-    def dropCollections(self):
-        for collection in self.db.collection_names():
-            if collection.startswith('system.'):
-                continue
-            self.db.drop_collection(collection)
-
-    def tearDown(self):
-        self.dropCollections()

mongoengine/django/utils.py

@@ -1,6 +0,0 @@
-try:
-    # django >= 1.4
-    from django.utils.timezone import now as datetime_now
-except ImportError:
-    from datetime import datetime
-    datetime_now = datetime.now

mongoengine/document.py

@@ -1,29 +1,29 @@
+import re
 import warnings
 
-import hashlib
-import pymongo
-import re
-
-from pymongo.read_preferences import ReadPreference
-from bson import ObjectId
 from bson.dbref import DBRef
+import pymongo
+from pymongo.read_preferences import ReadPreference
+
 from mongoengine import signals
-from mongoengine.common import _import_class
 from mongoengine.base import (
-    DocumentMetaclass,
-    TopLevelDocumentMetaclass,
-    BaseDocument,
-    BaseDict,
-    BaseList,
-    EmbeddedDocumentList,
     ALLOW_INHERITANCE,
+    BaseDict,
+    BaseDocument,
+    BaseList,
+    DocumentMetaclass,
+    EmbeddedDocumentList,
+    TopLevelDocumentMetaclass,
     get_document
 )
-from mongoengine.errors import ValidationError, InvalidQueryError, InvalidDocumentError
+from mongoengine.common import _import_class
-from mongoengine.queryset import (OperationError, NotUniqueError,
+from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
+from mongoengine.context_managers import switch_collection, switch_db
+from mongoengine.errors import (InvalidDocumentError, InvalidQueryError,
+                                SaveConditionError)
+from mongoengine.python_support import IS_PYMONGO_3
+from mongoengine.queryset import (NotUniqueError, OperationError,
                                   QuerySet, transform)
-from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
-from mongoengine.context_managers import switch_db, switch_collection
 
 __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument',
            'DynamicEmbeddedDocument', 'OperationError',
@@ -48,7 +48,6 @@ class InvalidCollectionError(Exception):
 
 
 class EmbeddedDocument(BaseDocument):
-
     """A :class:`~mongoengine.Document` that isn't stored in its own
     collection.  :class:`~mongoengine.EmbeddedDocument`\ s should be used as
     fields on :class:`~mongoengine.Document`\ s through the
@@ -63,7 +62,7 @@ class EmbeddedDocument(BaseDocument):
     dictionary.
     """
 
-    __slots__ = ('_instance')
+    __slots__ = ('_instance', )
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
@@ -91,7 +90,6 @@ class EmbeddedDocument(BaseDocument):
 
 
 class Document(BaseDocument):
-
     """The base class used for defining the structure and properties of
     collections of documents stored in MongoDB. Inherit from this class, and
     add fields as class attributes to define a document's structure.
@@ -116,9 +114,11 @@ class Document(BaseDocument):
     specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta`
     dictionary. :attr:`max_documents` is the maximum number of documents that
     is allowed to be stored in the collection, and :attr:`max_size` is the
-    maximum size of the collection in bytes. If :attr:`max_size` is not
+    maximum size of the collection in bytes. :attr:`max_size` is rounded up
+    to the next multiple of 256 by MongoDB internally and mongoengine before.
+    Use also a multiple of 256 to avoid confusions.  If :attr:`max_size` is not
     specified and :attr:`max_documents` is, :attr:`max_size` defaults to
-    10000000 bytes (10MB).
+    10485760 bytes (10MB).
 
     Indexes may be created by specifying :attr:`indexes` in the :attr:`meta`
     dictionary. The value should be a list of field names or tuples of field
@@ -135,6 +135,11 @@ class Document(BaseDocument):
     doesn't contain a list) if allow_inheritance is True. This can be
     disabled by either setting cls to False on the specific index or
     by setting index_cls to False on the meta dictionary for the document.
+
+    By default, any extra attribute existing in stored data but not declared
+    in your model will raise a :class:`~mongoengine.FieldDoesNotExist` error.
+    This can be disabled by setting :attr:`strict` to ``False``
+    in the :attr:`meta` dictionary.
     """
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
@@ -142,18 +147,22 @@ class Document(BaseDocument):
     my_metaclass = TopLevelDocumentMetaclass
     __metaclass__ = TopLevelDocumentMetaclass
 
-    __slots__ = ('__objects')
+    __slots__ = ('__objects',)
 
     def pk():
         """Primary key alias
         """
 
         def fget(self):
+            if 'id_field' not in self._meta:
+                return None
             return getattr(self, self._meta['id_field'])
 
         def fset(self, value):
             return setattr(self, self._meta['id_field'], value)
+
         return property(fget, fset)
+
     pk = pk()
 
     @classmethod
@@ -164,14 +173,18 @@ class Document(BaseDocument):
     @classmethod
     def _get_collection(cls):
         """Returns the collection for the document."""
+        # TODO: use new get_collection() with PyMongo3 ?
         if not hasattr(cls, '_collection') or cls._collection is None:
             db = cls._get_db()
             collection_name = cls._get_collection_name()
             # Create collection as a capped collection if specified
-            if cls._meta['max_size'] or cls._meta['max_documents']:
+            if cls._meta.get('max_size') or cls._meta.get('max_documents'):
                 # Get max document limit and max byte size from meta
-                max_size = cls._meta['max_size'] or 10000000  # 10MB default
+                max_size = cls._meta.get('max_size') or 10 * 2 ** 20  # 10MB default
-                max_documents = cls._meta['max_documents']
+                max_documents = cls._meta.get('max_documents')
+                # Round up to next 256 bytes as MongoDB would do it to avoid exception
+                if max_size % 256:
+                    max_size = (max_size // 256 + 1) * 256
 
                 if collection_name in db.collection_names():
                     cls._collection = db[collection_name]
@@ -179,7 +192,7 @@ class Document(BaseDocument):
                     # options match the specified capped options
                     options = cls._collection.options()
                     if options.get('max') != max_documents or \
-                       options.get('size') != max_size:
+                            options.get('size') != max_size:
                         msg = (('Cannot create collection "%s" as a capped '
                                 'collection as it already exists')
                                % cls._collection)
@@ -205,7 +218,7 @@ class Document(BaseDocument):
         Returns True if the document has been updated or False if the document
         in the database doesn't match the query.
 
-        .. note:: All unsaved changes that has been made to the document are
+        .. note:: All unsaved changes that have been made to the document are
             rejected if the method returns True.
 
         :param query: the update will be performed only if the document in the
@@ -237,8 +250,8 @@ class Document(BaseDocument):
         return True
 
     def save(self, force_insert=False, validate=True, clean=True,
-             write_concern=None,  cascade=None, cascade_kwargs=None,
+             write_concern=None, cascade=None, cascade_kwargs=None,
-             _refs=None, save_condition=None, **kwargs):
+             _refs=None, save_condition=None, signal_kwargs=None, **kwargs):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -262,7 +275,10 @@ class Document(BaseDocument):
             to cascading saves.  Implies ``cascade=True``.
         :param _refs: A list of processed references used in cascading saves
         :param save_condition: only perform save if matching record in db
-            satisfies condition(s) (e.g., version number)
+            satisfies condition(s) (e.g. version number).
+            Raises :class:`OperationError` if the conditions are not satisfied
+        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+            the signal calls.
 
         .. versionchanged:: 0.5
             In existing documents it only saves changed fields using
@@ -280,8 +296,15 @@ class Document(BaseDocument):
         .. versionchanged:: 0.8.5
             Optional save_condition that only overwrites existing documents
             if the condition is satisfied in the current db record.
+        .. versionchanged:: 0.10
+            :class:`OperationError` exception raised if save_condition fails.
+        .. versionchanged:: 0.10.1
+            :class: save_condition failure now raises a `SaveConditionError`
+        .. versionchanged:: 0.10.7
+            Add signal_kwargs argument
         """
-        signals.pre_save.send(self.__class__, document=self)
+        signal_kwargs = signal_kwargs or {}
+        signals.pre_save.send(self.__class__, document=self, **signal_kwargs)
 
         if validate:
             self.validate(clean=clean)
@@ -294,7 +317,7 @@ class Document(BaseDocument):
         created = ('_id' not in doc or self._created or force_insert)
 
         signals.pre_save_post_validation.send(self.__class__, document=self,
-                                              created=created)
+                                              created=created, **signal_kwargs)
 
         try:
             collection = self._get_collection()
@@ -305,6 +328,15 @@ class Document(BaseDocument):
                     object_id = collection.insert(doc, **write_concern)
                 else:
                     object_id = collection.save(doc, **write_concern)
+                    # In PyMongo 3.0, the save() call calls internally the _update() call
+                    # but they forget to return the _id value passed back, therefore getting it back here
+                    # Correct behaviour in 2.X and in 3.0.1+ versions
+                    if not object_id and pymongo.version_tuple == (3, 0):
+                        pk_as_mongo_obj = self._fields.get(self._meta['id_field']).to_mongo(self.pk)
+                        object_id = (
+                            self._qs.filter(pk=pk_as_mongo_obj).first() and
+                            self._qs.filter(pk=pk_as_mongo_obj).first().pk
+                        )  # TODO doesn't this make 2 queries?
             else:
                 object_id = doc['_id']
                 updates, removals = self._delta()
@@ -317,8 +349,12 @@ class Document(BaseDocument):
                 select_dict['_id'] = object_id
                 shard_key = self.__class__._meta.get('shard_key', tuple())
                 for k in shard_key:
-                    actual_key = self._db_field_map.get(k, k)
+                    path = self._lookup_field(k.split('.'))
-                    select_dict[actual_key] = doc[actual_key]
+                    actual_key = [p.db_field for p in path]
+                    val = doc
+                    for ak in actual_key:
+                        val = val[ak]
+                    select_dict['.'.join(actual_key)] = val
 
                 def is_new_object(last_error):
                     if last_error is not None:
@@ -337,6 +373,9 @@ class Document(BaseDocument):
                     upsert = save_condition is None
                     last_error = collection.update(select_dict, update_query,
                                                    upsert=upsert, **write_concern)
+                    if not upsert and last_error["n"] == 0:
+                        raise SaveConditionError('Race condition preventing'
+                                                 ' document update detected')
                     created = is_new_object(last_error)
 
             if cascade is None:
@@ -369,14 +408,15 @@ class Document(BaseDocument):
         if created or id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
-        signals.post_save.send(self.__class__, document=self, created=created)
+        signals.post_save.send(self.__class__, document=self,
+                               created=created, **signal_kwargs)
         self._clear_changed_fields()
         self._created = False
         return self
 
     def cascade_save(self, *args, **kwargs):
         """Recursively saves any references /
-           generic references on an objects"""
+           generic references on the document"""
         _refs = kwargs.get('_refs', []) or []
 
         ReferenceField = _import_class('ReferenceField')
@@ -417,7 +457,12 @@ class Document(BaseDocument):
         select_dict = {'pk': self.pk}
         shard_key = self.__class__._meta.get('shard_key', tuple())
         for k in shard_key:
-            select_dict[k] = getattr(self, k)
+            path = self._lookup_field(k.split('.'))
+            actual_key = [p.db_field for p in path]
+            val = self
+            for ak in actual_key:
+                val = getattr(val, ak)
+            select_dict['__'.join(actual_key)] = val
         return select_dict
 
     def update(self, **kwargs):
@@ -427,11 +472,11 @@ class Document(BaseDocument):
         Raises :class:`OperationError` if called on an object that has not yet
         been saved.
         """
-        if not self.pk:
+        if self.pk is None:
             if kwargs.get('upsert', False):
                 query = self.to_mongo()
                 if "_cls" in query:
-                    del(query["_cls"])
+                    del query["_cls"]
                 return self._qs.filter(**query).update_one(**kwargs)
             else:
                 raise OperationError(
@@ -440,18 +485,30 @@ class Document(BaseDocument):
         # Need to add shard key to query, or you get an error
         return self._qs.filter(**self._object_key).update_one(**kwargs)
 
-    def delete(self, **write_concern):
+    def delete(self, signal_kwargs=None, **write_concern):
         """Delete the :class:`~mongoengine.Document` from the database. This
         will only take effect if the document has been previously saved.
 
+        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+            the signal calls.
         :param write_concern: Extra keyword arguments are passed down which
             will be used as options for the resultant
             ``getLastError`` command.  For example,
             ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
             wait until at least two servers have recorded the write and
             will force an fsync on the primary server.
+
+        .. versionchanged:: 0.10.7
+            Add signal_kwargs argument
         """
-        signals.pre_delete.send(self.__class__, document=self)
+        signal_kwargs = signal_kwargs or {}
+        signals.pre_delete.send(self.__class__, document=self, **signal_kwargs)
+
+        # Delete FileFields separately
+        FileField = _import_class('FileField')
+        for name, field in self._fields.iteritems():
+            if isinstance(field, FileField):
+                getattr(self, name).delete()
 
         try:
             self._qs.filter(
@@ -459,9 +516,9 @@ class Document(BaseDocument):
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
-        signals.post_delete.send(self.__class__, document=self)
+        signals.post_delete.send(self.__class__, document=self, **signal_kwargs)
 
-    def switch_db(self, db_alias):
+    def switch_db(self, db_alias, keep_created=True):
         """
         Temporarily switch the database for a document instance.
 
@@ -473,6 +530,9 @@ class Document(BaseDocument):
 
         :param str db_alias: The database alias to use for saving the document
 
+        :param bool keep_created: keep self._created value after switching db, else is reset to True
+
+
         .. seealso::
             Use :class:`~mongoengine.context_managers.switch_collection`
             if you need to read from another collection
@@ -483,12 +543,12 @@ class Document(BaseDocument):
         self._get_collection = lambda: collection
         self._get_db = lambda: db
         self._collection = collection
-        self._created = True
+        self._created = True if not keep_created else self._created
         self.__objects = self._qs
         self.__objects._collection_obj = collection
         return self
 
-    def switch_collection(self, collection_name):
+    def switch_collection(self, collection_name, keep_created=True):
         """
         Temporarily switch the collection for a document instance.
 
@@ -501,6 +561,9 @@ class Document(BaseDocument):
         :param str collection_name: The database alias to use for saving the
             document
 
+        :param bool keep_created: keep self._created value after switching collection, else is reset to True
+
+
         .. seealso::
             Use :class:`~mongoengine.context_managers.switch_db`
             if you need to read from another database
@@ -509,7 +572,7 @@ class Document(BaseDocument):
             collection = cls._get_collection()
         self._get_collection = lambda: collection
         self._collection = collection
-        self._created = True
+        self._created = True if not keep_created else self._created
         self.__objects = self._qs
         self.__objects._collection_obj = collection
         return self
@@ -541,26 +604,31 @@ class Document(BaseDocument):
         elif "max_depth" in kwargs:
             max_depth = kwargs["max_depth"]
 
-        if not self.pk:
+        if self.pk is None:
             raise self.DoesNotExist("Document does not exist")
         obj = self._qs.read_preference(ReadPreference.PRIMARY).filter(
-            **self._object_key).only(*fields).limit(1
+            **self._object_key).only(*fields).limit(
-                                                    ).select_related(max_depth=max_depth)
+            1).select_related(max_depth=max_depth)
 
         if obj:
             obj = obj[0]
         else:
             raise self.DoesNotExist("Document does not exist")
 
-        for field in self._fields_ordered:
+        for field in obj._data:
             if not fields or field in fields:
                 try:
                     setattr(self, field, self._reload(field, obj[field]))
-                except KeyError:
+                except (KeyError, AttributeError):
-                    # If field is removed from the database while the object
+                    try:
-                    # is in memory, a reload would cause a KeyError
+                        # If field is a special field, e.g. items is stored as _reserved_items,
-                    # i.e. obj.update(unset__field=1) followed by obj.reload()
+                        # an KeyError is thrown. So try to retrieve the field from _data
-                    delattr(self, field)
+                        setattr(self, field, self._reload(field, obj._data.get(field)))
+                    except KeyError:
+                        # If field is removed from the database while the object
+                        # is in memory, a reload would cause a KeyError
+                        # i.e. obj.update(unset__field=1) followed by obj.reload()
+                        delattr(self, field)
 
         self._changed_fields = obj._changed_fields
         self._created = False
@@ -587,7 +655,7 @@ class Document(BaseDocument):
     def to_dbref(self):
         """Returns an instance of :class:`~bson.dbref.DBRef` useful in
         `__raw__` queries."""
-        if not self.pk:
+        if self.pk is None:
             msg = "Only saved documents can have a valid dbref"
             raise OperationError(msg)
         return DBRef(self.__class__._get_collection_name(), self.pk)
@@ -604,38 +672,76 @@ class Document(BaseDocument):
                      for class_name in document_cls._subclasses
                      if class_name != document_cls.__name__] + [document_cls]
 
-        for cls in classes:
+        for klass in classes:
             for document_cls in documents:
-                delete_rules = cls._meta.get('delete_rules') or {}
+                delete_rules = klass._meta.get('delete_rules') or {}
                 delete_rules[(document_cls, field_name)] = rule
-                cls._meta['delete_rules'] = delete_rules
+                klass._meta['delete_rules'] = delete_rules
 
     @classmethod
     def drop_collection(cls):
         """Drops the entire collection associated with this
         :class:`~mongoengine.Document` type from the database.
+
+        Raises :class:`OperationError` if the document has no collection set
+        (i.g. if it is `abstract`)
+
+        .. versionchanged:: 0.10.7
+            :class:`OperationError` exception raised if no collection available
         """
+        col_name = cls._get_collection_name()
+        if not col_name:
+            raise OperationError('Document %s has no collection defined '
+                                 '(is it abstract ?)' % cls)
         cls._collection = None
         db = cls._get_db()
-        db.drop_collection(cls._get_collection_name())
+        db.drop_collection(col_name)
+
+    @classmethod
+    def create_index(cls, keys, background=False, **kwargs):
+        """Creates the given indexes if required.
+
+        :param keys: a single index key or a list of index keys (to
+            construct a multi-field index); keys may be prefixed with a **+**
+            or a **-** to determine the index ordering
+        :param background: Allows index creation in the background
+        """
+        index_spec = cls._build_index_spec(keys)
+        index_spec = index_spec.copy()
+        fields = index_spec.pop('fields')
+        drop_dups = kwargs.get('drop_dups', False)
+        if IS_PYMONGO_3 and drop_dups:
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
+        elif not IS_PYMONGO_3:
+            index_spec['drop_dups'] = drop_dups
+        index_spec['background'] = background
+        index_spec.update(kwargs)
+
+        if IS_PYMONGO_3:
+            return cls._get_collection().create_index(fields, **index_spec)
+        else:
+            return cls._get_collection().ensure_index(fields, **index_spec)
 
     @classmethod
     def ensure_index(cls, key_or_list, drop_dups=False, background=False,
                      **kwargs):
-        """Ensure that the given indexes are in place.
+        """Ensure that the given indexes are in place. Deprecated in favour
+        of create_index.
 
         :param key_or_list: a single index key or a list of index keys (to
             construct a multi-field index); keys may be prefixed with a **+**
             or a **-** to determine the index ordering
+        :param background: Allows index creation in the background
+        :param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value
+            will be removed if PyMongo3+ is used
         """
-        index_spec = cls._build_index_spec(key_or_list)
+        if IS_PYMONGO_3 and drop_dups:
-        index_spec = index_spec.copy()
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
-        fields = index_spec.pop('fields')
+            warnings.warn(msg, DeprecationWarning)
-        index_spec['drop_dups'] = drop_dups
+        elif not IS_PYMONGO_3:
-        index_spec['background'] = background
+            kwargs.update({'drop_dups': drop_dups})
-        index_spec.update(kwargs)
+        return cls.create_index(key_or_list, background=background, **kwargs)
-
-        return cls._get_collection().ensure_index(fields, **index_spec)
 
     @classmethod
     def ensure_indexes(cls):
@@ -650,6 +756,9 @@ class Document(BaseDocument):
         drop_dups = cls._meta.get('index_drop_dups', False)
         index_opts = cls._meta.get('index_opts') or {}
         index_cls = cls._meta.get('index_cls', True)
+        if IS_PYMONGO_3 and drop_dups:
+            msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
 
         collection = cls._get_collection()
         # 746: when connection is via mongos, the read preference is not necessarily an indication that
@@ -672,18 +781,37 @@ class Document(BaseDocument):
                 cls_indexed = cls_indexed or includes_cls(fields)
                 opts = index_opts.copy()
                 opts.update(spec)
-                collection.ensure_index(fields, background=background,
+
-                                        drop_dups=drop_dups, **opts)
+                # we shouldn't pass 'cls' to the collection.ensureIndex options
+                # because of https://jira.mongodb.org/browse/SERVER-769
+                if 'cls' in opts:
+                    del opts['cls']
+
+                if IS_PYMONGO_3:
+                    collection.create_index(fields, background=background, **opts)
+                else:
+                    collection.ensure_index(fields, background=background,
+                                            drop_dups=drop_dups, **opts)
 
         # If _cls is being used (for polymorphism), it needs an index,
         # only if another index doesn't begin with _cls
         if (index_cls and not cls_indexed and
                 cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
-            collection.ensure_index('_cls', background=background,
+
-                                    **index_opts)
+            # we shouldn't pass 'cls' to the collection.ensureIndex options
+            # because of https://jira.mongodb.org/browse/SERVER-769
+            if 'cls' in index_opts:
+                del index_opts['cls']
+
+            if IS_PYMONGO_3:
+                collection.create_index('_cls', background=background,
+                                        **index_opts)
+            else:
+                collection.ensure_index('_cls', background=background,
+                                        **index_opts)
 
     @classmethod
-    def list_indexes(cls, go_up=True, go_down=True):
+    def list_indexes(cls):
         """ Lists all of the indexes that should be created for given
         collection. It includes all the indexes from super- and sub-classes.
         """
@@ -730,8 +858,8 @@ class Document(BaseDocument):
             return indexes
 
         indexes = []
-        for cls in classes:
+        for klass in classes:
-            for index in get_indexes_spec(cls):
+            for index in get_indexes_spec(klass):
                 if index not in indexes:
                     indexes.append(index)
 
@@ -770,7 +898,6 @@ class Document(BaseDocument):
 
 
 class DynamicDocument(Document):
-
     """A Dynamic Document class allowing flexible, expandable and uncontrolled
     schemas.  As a :class:`~mongoengine.Document` subclass, acts in the same
     way as an ordinary document but has expando style properties.  Any data
@@ -802,7 +929,6 @@ class DynamicDocument(Document):
 
 
 class DynamicEmbeddedDocument(EmbeddedDocument):
-
     """A Dynamic Embedded Document class allowing flexible, expandable and
     uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more
     information about dynamic documents.
@@ -829,7 +955,6 @@ class DynamicEmbeddedDocument(EmbeddedDocument):
 
 
 class MapReduceDocument(object):
-
     """A document returned from a map/reduce query.
 
     :param collection: An instance of :class:`~pymongo.Collection`
@@ -859,7 +984,7 @@ class MapReduceDocument(object):
         if not isinstance(self.key, id_field_type):
             try:
                 self.key = id_field_type(self.key)
-            except:
+            except Exception:
                 raise Exception("Could not cast key as %s" %
                                 id_field_type.__name__)
 

mongoengine/errors.py

@@ -6,7 +6,7 @@ from mongoengine.python_support import txt_type
 __all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError',
            'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError',
            'OperationError', 'NotUniqueError', 'FieldDoesNotExist',
-           'ValidationError')
+           'ValidationError', 'SaveConditionError')
 
 
 class NotRegistered(Exception):
@@ -41,10 +41,21 @@ class NotUniqueError(OperationError):
     pass
 
 
-class FieldDoesNotExist(Exception):
+class SaveConditionError(OperationError):
     pass
 
 
+class FieldDoesNotExist(Exception):
+    """Raised when trying to set a field
+    not declared in a :class:`~mongoengine.Document`
+    or an :class:`~mongoengine.EmbeddedDocument`.
+
+    To avoid this behavior on data loading,
+    you should the :attr:`strict` to ``False``
+    in the :attr:`meta` dictionnary.
+    """
+
+
 class ValidationError(AssertionError):
     """Validation exception.
 
@@ -108,6 +119,7 @@ class ValidationError(AssertionError):
             else:
                 return unicode(source)
             return errors_dict
+
         if not self.errors:
             return {}
         return build_dict(self.errors)
@@ -118,9 +130,9 @@ class ValidationError(AssertionError):
         def generate_key(value, prefix=''):
             if isinstance(value, list):
                 value = ' '.join([generate_key(k) for k in value])
-            if isinstance(value, dict):
+            elif isinstance(value, dict):
                 value = ' '.join(
-                        [generate_key(v, k) for k, v in value.iteritems()])
+                    [generate_key(v, k) for k, v in value.iteritems()])
 
             results = "%s.%s" % (prefix, value) if prefix else value
             return results

mongoengine/fields.py

@@ -8,6 +8,11 @@ import uuid
 import warnings
 from operator import itemgetter
 
+from bson import Binary, DBRef, ObjectId, SON
+import gridfs
+import pymongo
+import six
+
 try:
     import dateutil
 except ImportError:
@@ -15,18 +20,18 @@ except ImportError:
 else:
     import dateutil.parser
 
-import pymongo
+try:
-import gridfs
+    from bson.int64 import Int64
-from bson import Binary, DBRef, SON, ObjectId
+except ImportError:
+    Int64 = long
 
-from mongoengine.errors import ValidationError
+from .base import (BaseDocument, BaseField, ComplexBaseField, GeoJsonBaseField,
-from mongoengine.python_support import (PY3, bin_type, txt_type,
+                   ObjectIdField, get_document)
-                                        str_types, StringIO)
+from .connection import DEFAULT_CONNECTION_NAME, get_db
-from base import (BaseField, ComplexBaseField, ObjectIdField, GeoJsonBaseField,
+from .document import Document, EmbeddedDocument
-                  get_document, BaseDocument)
+from .errors import DoesNotExist, ValidationError
-from queryset import DO_NOTHING, QuerySet
+from .python_support import PY3, StringIO, bin_type, str_types, txt_type
-from document import Document, EmbeddedDocument
+from .queryset import DO_NOTHING, QuerySet
-from connection import get_db, DEFAULT_CONNECTION_NAME
 
 try:
     from PIL import Image, ImageOps
@@ -47,12 +52,10 @@ __all__ = [
     'SequenceField', 'UUIDField', 'MultiPointField', 'MultiLineStringField',
     'MultiPolygonField', 'GeoJsonBaseField']
 
-
 RECURSIVE_REFERENCE_CONSTANT = 'self'
 
 
 class StringField(BaseField):
-
     """A unicode string field.
     """
 
@@ -67,7 +70,7 @@ class StringField(BaseField):
             return value
         try:
             value = value.decode('utf-8')
-        except:
+        except Exception:
             pass
         return value
 
@@ -108,33 +111,41 @@ class StringField(BaseField):
             # escape unsafe characters which could lead to a re.error
             value = re.escape(value)
             value = re.compile(regex % value, flags)
-        return value
+        return super(StringField, self).prepare_query_value(op, value)
 
 
 class URLField(StringField):
-
     """A field that validates input as an URL.
 
     .. versionadded:: 0.3
     """
 
     _URL_REGEX = re.compile(
-        r'^(?:http|ftp)s?://'  # http:// or https://
+        r'^(?:[a-z0-9\.\-]*)://'  # scheme is validated separately
-        # domain...
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'  # domain...
-        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
         r'localhost|'  # localhost...
-        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
+        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
         r'(?::\d+)?'  # optional port
         r'(?:/?|[/?]\S+)$', re.IGNORECASE)
+    _URL_SCHEMES = ['http', 'https', 'ftp', 'ftps']
 
-    def __init__(self, verify_exists=False, url_regex=None, **kwargs):
+    def __init__(self, verify_exists=False, url_regex=None, schemes=None, **kwargs):
         self.verify_exists = verify_exists
         self.url_regex = url_regex or self._URL_REGEX
+        self.schemes = schemes or self._URL_SCHEMES
         super(URLField, self).__init__(**kwargs)
 
     def validate(self, value):
+        # Check first if the scheme is valid
+        scheme = value.split('://')[0].lower()
+        if scheme not in self.schemes:
+            self.error('Invalid scheme {} in URL: {}'.format(scheme, value))
+            return
+
+        # Then check full URL
         if not self.url_regex.match(value):
-            self.error('Invalid URL: %s' % value)
+            self.error('Invalid URL: {}'.format(value))
             return
 
         if self.verify_exists:
@@ -150,8 +161,7 @@ class URLField(StringField):
 
 
 class EmailField(StringField):
-
+    """A field that validates input as an email address.
-    """A field that validates input as an E-Mail-Address.
 
     .. versionadded:: 0.4
     """
@@ -162,17 +172,16 @@ class EmailField(StringField):
         # quoted-string
         r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
         # domain (max length of an ICAAN TLD is 22 characters)
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,22}$', re.IGNORECASE
+        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
     )
 
     def validate(self, value):
         if not EmailField.EMAIL_REGEX.match(value):
-            self.error('Invalid Mail-address: %s' % value)
+            self.error('Invalid email address: %s' % value)
         super(EmailField, self).validate(value)
 
 
 class IntField(BaseField):
-
     """An 32-bit integer field.
     """
 
@@ -190,7 +199,7 @@ class IntField(BaseField):
     def validate(self, value):
         try:
             value = int(value)
-        except:
+        except Exception:
             self.error('%s could not be converted to int' % value)
 
         if self.min_value is not None and value < self.min_value:
@@ -203,11 +212,10 @@ class IntField(BaseField):
         if value is None:
             return value
 
-        return int(value)
+        return super(IntField, self).prepare_query_value(op, int(value))
 
 
 class LongField(BaseField):
-
     """An 64-bit integer field.
     """
 
@@ -222,10 +230,13 @@ class LongField(BaseField):
             pass
         return value
 
+    def to_mongo(self, value):
+        return Int64(value)
+
     def validate(self, value):
         try:
             value = long(value)
-        except:
+        except Exception:
             self.error('%s could not be converted to long' % value)
 
         if self.min_value is not None and value < self.min_value:
@@ -238,11 +249,10 @@ class LongField(BaseField):
         if value is None:
             return value
 
-        return long(value)
+        return super(LongField, self).prepare_query_value(op, long(value))
 
 
 class FloatField(BaseField):
-
     """An floating point number field.
     """
 
@@ -258,10 +268,14 @@ class FloatField(BaseField):
         return value
 
     def validate(self, value):
-        if isinstance(value, int):
+        if isinstance(value, six.integer_types):
-            value = float(value)
+            try:
+                value = float(value)
+            except OverflowError:
+                self.error('The value is too large to be converted to float')
+
         if not isinstance(value, float):
-            self.error('FloatField only accepts float values')
+            self.error('FloatField only accepts float and integer values')
 
         if self.min_value is not None and value < self.min_value:
             self.error('Float value is too small')
@@ -273,11 +287,10 @@ class FloatField(BaseField):
         if value is None:
             return value
 
-        return float(value)
+        return super(FloatField, self).prepare_query_value(op, float(value))
 
 
 class DecimalField(BaseField):
-
     """A fixed-point decimal number field.
 
     .. versionchanged:: 0.8
@@ -324,7 +337,7 @@ class DecimalField(BaseField):
             return value
         return value.quantize(decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding)
 
-    def to_mongo(self, value, use_db_field=True):
+    def to_mongo(self, value):
         if value is None:
             return value
         if self.force_string:
@@ -347,11 +360,10 @@ class DecimalField(BaseField):
             self.error('Decimal value is too large')
 
     def prepare_query_value(self, op, value):
-        return self.to_mongo(value)
+        return super(DecimalField, self).prepare_query_value(op, self.to_mongo(value))
 
 
 class BooleanField(BaseField):
-
     """A boolean field type.
 
     .. versionadded:: 0.1.2
@@ -370,7 +382,6 @@ class BooleanField(BaseField):
 
 
 class DateTimeField(BaseField):
-
     """A datetime field.
 
     Uses the python-dateutil library if available alternatively use time.strptime
@@ -434,11 +445,10 @@ class DateTimeField(BaseField):
                     return None
 
     def prepare_query_value(self, op, value):
-        return self.to_mongo(value)
+        return super(DateTimeField, self).prepare_query_value(op, self.to_mongo(value))
 
 
 class ComplexDateTimeField(StringField):
-
     """
     ComplexDateTimeField handles microseconds exactly instead of rounding
     like DateTimeField does.
@@ -458,37 +468,22 @@ class ComplexDateTimeField(StringField):
     """
 
     def __init__(self, separator=',', **kwargs):
-        self.names = ['year', 'month', 'day', 'hour', 'minute', 'second',
+        self.names = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond']
-                      'microsecond']
+        self.separator = separator
-        self.separtor = separator
+        self.format = separator.join(['%Y', '%m', '%d', '%H', '%M', '%S', '%f'])
         super(ComplexDateTimeField, self).__init__(**kwargs)
 
-    def _leading_zero(self, number):
-        """
-        Converts the given number to a string.
-
-        If it has only one digit, a leading zero so as it has always at least
-        two digits.
-        """
-        if int(number) < 10:
-            return "0%s" % number
-        else:
-            return str(number)
-
     def _convert_from_datetime(self, val):
         """
         Convert a `datetime` object to a string representation (which will be
         stored in MongoDB). This is the reverse function of
         `_convert_from_string`.
 
-        >>> a = datetime(2011, 6, 8, 20, 26, 24, 192284)
+        >>> a = datetime(2011, 6, 8, 20, 26, 24, 92284)
-        >>> RealDateTimeField()._convert_from_datetime(a)
+        >>> ComplexDateTimeField()._convert_from_datetime(a)
-        '2011,06,08,20,26,24,192284'
+        '2011,06,08,20,26,24,092284'
         """
-        data = []
+        return val.strftime(self.format)
-        for name in self.names:
-            data.append(self._leading_zero(getattr(val, name)))
-        return ','.join(data)
 
     def _convert_from_string(self, data):
         """
@@ -496,16 +491,12 @@ class ComplexDateTimeField(StringField):
         will manipulate). This is the reverse function of
         `_convert_from_datetime`.
 
-        >>> a = '2011,06,08,20,26,24,192284'
+        >>> a = '2011,06,08,20,26,24,092284'
         >>> ComplexDateTimeField()._convert_from_string(a)
-        datetime.datetime(2011, 6, 8, 20, 26, 24, 192284)
+        datetime.datetime(2011, 6, 8, 20, 26, 24, 92284)
         """
-        data = data.split(',')
+        values = map(int, data.split(self.separator))
-        data = map(int, data)
+        return datetime.datetime(*values)
-        values = {}
-        for i in range(7):
-            values[self.names[i]] = data[i]
-        return datetime.datetime(**values)
 
     def __get__(self, instance, owner):
         data = super(ComplexDateTimeField, self).__get__(instance, owner)
@@ -529,7 +520,7 @@ class ComplexDateTimeField(StringField):
         original_value = value
         try:
             return self._convert_from_string(value)
-        except:
+        except Exception:
             return original_value
 
     def to_mongo(self, value):
@@ -537,11 +528,10 @@ class ComplexDateTimeField(StringField):
         return self._convert_from_datetime(value)
 
     def prepare_query_value(self, op, value):
-        return self._convert_from_datetime(value)
+        return super(ComplexDateTimeField, self).prepare_query_value(op, self._convert_from_datetime(value))
 
 
 class EmbeddedDocumentField(BaseField):
-
     """An embedded document field - with a declared document_type.
     Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`.
     """
@@ -565,14 +555,13 @@ class EmbeddedDocumentField(BaseField):
 
     def to_python(self, value):
         if not isinstance(value, self.document_type):
-            return self.document_type._from_son(value)
+            return self.document_type._from_son(value, _auto_dereference=self._auto_dereference)
         return value
 
-    def to_mongo(self, value, use_db_field=True, fields=[]):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         if not isinstance(value, self.document_type):
             return value
-        return self.document_type.to_mongo(value, use_db_field,
+        return self.document_type.to_mongo(value, use_db_field, fields)
-                                           fields=fields)
 
     def validate(self, value, clean=True):
         """Make sure that the document instance is an instance of the
@@ -588,11 +577,13 @@ class EmbeddedDocumentField(BaseField):
         return self.document_type._fields.get(member_name)
 
     def prepare_query_value(self, op, value):
+        if value is not None and not isinstance(value, self.document_type):
+            value = self.document_type._from_son(value)
+        super(EmbeddedDocumentField, self).prepare_query_value(op, value)
         return self.to_mongo(value)
 
 
 class GenericEmbeddedDocumentField(BaseField):
-
     """A generic embedded document field - allows any
     :class:`~mongoengine.EmbeddedDocument` to be stored.
 
@@ -604,7 +595,7 @@ class GenericEmbeddedDocumentField(BaseField):
     """
 
     def prepare_query_value(self, op, value):
-        return self.to_mongo(value)
+        return super(GenericEmbeddedDocumentField, self).prepare_query_value(op, self.to_mongo(value))
 
     def to_python(self, value):
         if isinstance(value, dict):
@@ -620,24 +611,23 @@ class GenericEmbeddedDocumentField(BaseField):
 
         value.validate(clean=clean)
 
-    def to_mongo(self, document, use_db_field=True):
+    def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
 
-        data = document.to_mongo(use_db_field)
+        data = document.to_mongo(use_db_field, fields)
-        if not '_cls' in data:
+        if '_cls' not in data:
             data['_cls'] = document._class_name
         return data
 
 
 class DynamicField(BaseField):
-
     """A truly dynamic field type capable of handling different and varying
     types of data.
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
-    def to_mongo(self, value):
+    def to_mongo(self, value, use_db_field=True, fields=None):
         """Convert a Python type to a MongoDB compatible type.
         """
 
@@ -646,11 +636,11 @@ class DynamicField(BaseField):
 
         if hasattr(value, 'to_mongo'):
             cls = value.__class__
-            val = value.to_mongo()
+            val = value.to_mongo(use_db_field, fields)
             # If we its a document thats not inherited add _cls
-            if (isinstance(value,   Document)):
+            if isinstance(value, Document):
                 val = {"_ref": value.to_dbref(), "_cls": cls.__name__}
-            if (isinstance(value, EmbeddedDocument)):
+            if isinstance(value, EmbeddedDocument):
                 val['_cls'] = cls.__name__
             return val
 
@@ -664,7 +654,7 @@ class DynamicField(BaseField):
 
         data = {}
         for k, v in value.iteritems():
-            data[k] = self.to_mongo(v)
+            data[k] = self.to_mongo(v, use_db_field, fields)
 
         value = data
         if is_list:  # Convert back to a list
@@ -685,9 +675,8 @@ class DynamicField(BaseField):
 
     def prepare_query_value(self, op, value):
         if isinstance(value, basestring):
-            from mongoengine.fields import StringField
             return StringField().prepare_query_value(op, value)
-        return self.to_mongo(value)
+        return super(DynamicField, self).prepare_query_value(op, self.to_mongo(value))
 
     def validate(self, value, clean=True):
         if hasattr(value, "validate"):
@@ -695,7 +684,6 @@ class DynamicField(BaseField):
 
 
 class ListField(ComplexBaseField):
-
     """A list field that wraps a standard field, allowing multiple instances
     of the field to be used as a list in the database.
 
@@ -720,9 +708,10 @@ class ListField(ComplexBaseField):
 
     def prepare_query_value(self, op, value):
         if self.field:
-            if op in ('set', 'unset') and (not isinstance(value, basestring)
+            if op in ('set', 'unset', None) and (
-                                           and not isinstance(value, BaseDocument)
+                    not isinstance(value, basestring) and
-                                           and hasattr(value, '__iter__')):
+                    not isinstance(value, BaseDocument) and
+                    hasattr(value, '__iter__')):
                 return [self.field.prepare_query_value(op, v) for v in value]
             return self.field.prepare_query_value(op, value)
         return super(ListField, self).prepare_query_value(op, value)
@@ -740,12 +729,10 @@ class EmbeddedDocumentListField(ListField):
 
     """
 
-    def __init__(self, document_type, *args, **kwargs):
+    def __init__(self, document_type, **kwargs):
         """
         :param document_type: The type of
          :class:`~mongoengine.EmbeddedDocument` the list will hold.
-        :param args: Arguments passed directly into the parent
-         :class:`~mongoengine.ListField`.
         :param kwargs: Keyword arguments passed directly into the parent
          :class:`~mongoengine.ListField`.
         """
@@ -755,7 +742,6 @@ class EmbeddedDocumentListField(ListField):
 
 
 class SortedListField(ListField):
-
     """A ListField that sorts the contents of its list before writing to
     the database in order to ensure that a sorted list is always
     retrieved.
@@ -780,8 +766,8 @@ class SortedListField(ListField):
             self._order_reverse = kwargs.pop('reverse')
         super(SortedListField, self).__init__(field, **kwargs)
 
-    def to_mongo(self, value):
+    def to_mongo(self, value, use_db_field=True, fields=None):
-        value = super(SortedListField, self).to_mongo(value)
+        value = super(SortedListField, self).to_mongo(value, use_db_field, fields)
         if self._ordering is not None:
             return sorted(value, key=itemgetter(self._ordering),
                           reverse=self._order_reverse)
@@ -807,7 +793,6 @@ def key_has_dot_or_dollar(d):
 
 
 class DictField(ComplexBaseField):
-
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
 
@@ -820,6 +805,7 @@ class DictField(ComplexBaseField):
 
     def __init__(self, basecls=None, field=None, *args, **kwargs):
         self.field = field
+        self._auto_dereference = False
         self.basecls = basecls or BaseField
         if not issubclass(self.basecls, BaseField):
             self.error('DictField only accepts dict values')
@@ -863,7 +849,6 @@ class DictField(ComplexBaseField):
 
 
 class MapField(DictField):
-
     """A field that maps a name to a specified field type. Similar to
     a DictField, except the 'value' of each item must match the specified
     field type.
@@ -879,7 +864,6 @@ class MapField(DictField):
 
 
 class ReferenceField(BaseField):
-
     """A reference to a document that will be automatically dereferenced on
     access (lazily).
 
@@ -890,12 +874,11 @@ class ReferenceField(BaseField):
 
     The options are:
 
-      * DO_NOTHING  - don't do anything (default).
+      * DO_NOTHING (0)  - don't do anything (default).
-      * NULLIFY     - Updates the reference to null.
+      * NULLIFY    (1)  - Updates the reference to null.
-      * CASCADE     - Deletes the documents associated with the reference.
+      * CASCADE    (2)  - Deletes the documents associated with the reference.
-      * DENY        - Prevent the deletion of the reference object.
+      * DENY       (3)  - Prevent the deletion of the reference object.
-      * PULL        - Pull the reference from a :class:`~mongoengine.fields.ListField`
+      * PULL       (4)  - Pull the reference from a :class:`~mongoengine.fields.ListField` of references
-                      of references
 
     Alternative syntax for registering delete rules (useful when implementing
     bi-directional delete rules)
@@ -906,7 +889,7 @@ class ReferenceField(BaseField):
             content = StringField()
             foo = ReferenceField('Foo')
 
-        Bar.register_delete_rule(Foo, 'bar', NULLIFY)
+        Foo.register_delete_rule(Bar, 'foo', NULLIFY)
 
     .. note ::
         `reverse_delete_rule` does not trigger pre / post delete signals to be
@@ -923,6 +906,10 @@ class ReferenceField(BaseField):
           or as the :class:`~pymongo.objectid.ObjectId`.id .
         :param reverse_delete_rule: Determines what to do when the referring
           object is deleted
+
+        .. note ::
+            A reference to an abstract document type is always stored as a
+            :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`.
         """
         if not isinstance(document_type, basestring):
             if not issubclass(document_type, (Document, basestring)):
@@ -955,9 +942,16 @@ class ReferenceField(BaseField):
         self._auto_dereference = instance._fields[self.name]._auto_dereference
         # Dereference DBRefs
         if self._auto_dereference and isinstance(value, DBRef):
-            value = self.document_type._get_db().dereference(value)
+            if hasattr(value, 'cls'):
-            if value is not None:
+                # Dereference using the class type specified in the reference
-                instance._data[self.name] = self.document_type._from_son(value)
+                cls = get_document(value.cls)
+            else:
+                cls = self.document_type
+            dereferenced = cls._get_db().dereference(value)
+            if dereferenced is None:
+                raise DoesNotExist('Trying to dereference unknown document %s' % value)
+            else:
+                instance._data[self.name] = cls._from_son(dereferenced)
 
         return super(ReferenceField, self).__get__(instance, owner)
 
@@ -967,21 +961,29 @@ class ReferenceField(BaseField):
                 return document.id
             return document
 
-        id_field_name = self.document_type._meta['id_field']
-        id_field = self.document_type._fields[id_field_name]
-
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
             if id_ is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
+
+            # Use the attributes from the document instance, so that they
+            # override the attributes of this field's document type
+            cls = document
         else:
             id_ = document
+            cls = self.document_type
+
+        id_field_name = cls._meta['id_field']
+        id_field = cls._fields[id_field_name]
 
         id_ = id_field.to_mongo(id_)
-        if self.dbref:
+        if self.document_type._meta.get('abstract'):
-            collection = self.document_type._get_collection_name()
+            collection = cls._get_collection_name()
+            return DBRef(collection, id_, cls=cls._class_name)
+        elif self.dbref:
+            collection = cls._get_collection_name()
             return DBRef(collection, id_)
 
         return id_
@@ -998,6 +1000,7 @@ class ReferenceField(BaseField):
     def prepare_query_value(self, op, value):
         if value is None:
             return None
+        super(ReferenceField, self).prepare_query_value(op, value)
         return self.to_mongo(value)
 
     def validate(self, value):
@@ -1009,14 +1012,21 @@ class ReferenceField(BaseField):
             self.error('You can only reference documents once they have been '
                        'saved to the database')
 
+        if self.document_type._meta.get('abstract') and \
+                not isinstance(value, self.document_type):
+            self.error(
+                '%s is not an instance of abstract reference type %s' % (
+                    self.document_type._class_name)
+            )
+
     def lookup_member(self, member_name):
         return self.document_type._fields.get(member_name)
 
 
 class CachedReferenceField(BaseField):
-
     """
-    A referencefield with cache fields to porpuse pseudo-joins
+    A referencefield with cache fields to purpose pseudo-joins
+
     .. versionadded:: 0.9
     """
 
@@ -1028,7 +1038,6 @@ class CachedReferenceField(BaseField):
         """
         if not isinstance(document_type, basestring) and \
                 not issubclass(document_type, (Document, basestring)):
-
             self.error('Argument to CachedReferenceField constructor must be a'
                        ' document class or a string')
 
@@ -1039,6 +1048,7 @@ class CachedReferenceField(BaseField):
 
     def start_listener(self):
         from mongoengine import signals
+
         signals.post_save.connect(self.on_document_pre_save,
                                   sender=self.document_type)
 
@@ -1061,6 +1071,7 @@ class CachedReferenceField(BaseField):
             collection = self.document_type._get_collection_name()
             value = DBRef(
                 collection, self.document_type.id.to_python(value['_id']))
+            return self.document_type._from_son(self.document_type._get_db().dereference(value))
 
         return value
 
@@ -1083,16 +1094,17 @@ class CachedReferenceField(BaseField):
         self._auto_dereference = instance._fields[self.name]._auto_dereference
         # Dereference DBRefs
         if self._auto_dereference and isinstance(value, DBRef):
-            value = self.document_type._get_db().dereference(value)
+            dereferenced = self.document_type._get_db().dereference(value)
-            if value is not None:
+            if dereferenced is None:
-                instance._data[self.name] = self.document_type._from_son(value)
+                raise DoesNotExist('Trying to dereference unknown document %s' % value)
+            else:
+                instance._data[self.name] = self.document_type._from_son(dereferenced)
 
         return super(CachedReferenceField, self).__get__(instance, owner)
 
-    def to_mongo(self, document):
+    def to_mongo(self, document, use_db_field=True, fields=None):
         id_field_name = self.document_type._meta['id_field']
         id_field = self.document_type._fields[id_field_name]
-        doc_tipe = self.document_type
 
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
@@ -1102,12 +1114,18 @@ class CachedReferenceField(BaseField):
                            ' been saved to the database')
         else:
             self.error('Only accept a document object')
+            # TODO: should raise here or will fail next statement
 
         value = SON((
             ("_id", id_field.to_mongo(id_)),
         ))
 
-        value.update(dict(document.to_mongo(fields=self.fields)))
+        if fields:
+            new_fields = [f for f in self.fields if f in fields]
+        else:
+            new_fields = self.fields
+
+        value.update(dict(document.to_mongo(use_db_field, fields=new_fields)))
         return value
 
     def prepare_query_value(self, op, value):
@@ -1124,7 +1142,7 @@ class CachedReferenceField(BaseField):
 
     def validate(self, value):
 
-        if not isinstance(value, (self.document_type)):
+        if not isinstance(value, self.document_type):
             self.error("A CachedReferenceField only accepts documents")
 
         if isinstance(value, Document) and value.id is None:
@@ -1153,7 +1171,6 @@ class CachedReferenceField(BaseField):
 
 
 class GenericReferenceField(BaseField):
-
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
 
@@ -1167,6 +1184,30 @@ class GenericReferenceField(BaseField):
     .. versionadded:: 0.3
     """
 
+    def __init__(self, *args, **kwargs):
+        choices = kwargs.pop('choices', None)
+        super(GenericReferenceField, self).__init__(*args, **kwargs)
+        self.choices = []
+        # Keep the choices as a list of allowed Document class names
+        if choices:
+            for choice in choices:
+                if isinstance(choice, basestring):
+                    self.choices.append(choice)
+                elif isinstance(choice, type) and issubclass(choice, Document):
+                    self.choices.append(choice._class_name)
+                else:
+                    self.error('Invalid choices provided: must be a list of'
+                               'Document subclasses and/or basestrings')
+
+    def _validate_choices(self, value):
+        if isinstance(value, dict):
+            # If the field has not been dereferenced, it is still a dict
+            # of class and DBRef
+            value = value.get('_cls')
+        elif isinstance(value, Document):
+            value = value._class_name
+        super(GenericReferenceField, self)._validate_choices(value)
+
     def __get__(self, instance, owner):
         if instance is None:
             return self
@@ -1175,7 +1216,11 @@ class GenericReferenceField(BaseField):
 
         self._auto_dereference = instance._fields[self.name]._auto_dereference
         if self._auto_dereference and isinstance(value, (dict, SON)):
-            instance._data[self.name] = self.dereference(value)
+            dereferenced = self.dereference(value)
+            if dereferenced is None:
+                raise DoesNotExist('Trying to dereference unknown document %s' % value)
+            else:
+                instance._data[self.name] = dereferenced
 
         return super(GenericReferenceField, self).__get__(instance, owner)
 
@@ -1200,11 +1245,11 @@ class GenericReferenceField(BaseField):
             doc = doc_cls._from_son(doc)
         return doc
 
-    def to_mongo(self, document, use_db_field=True):
+    def to_mongo(self, document):
         if document is None:
             return None
 
-        if isinstance(document, (dict, SON)):
+        if isinstance(document, (dict, SON, ObjectId, DBRef)):
             return document
 
         id_field_name = document.__class__._meta['id_field']
@@ -1235,7 +1280,6 @@ class GenericReferenceField(BaseField):
 
 
 class BinaryField(BaseField):
-
     """A binary data field.
     """
 
@@ -1267,7 +1311,6 @@ class GridFSError(Exception):
 
 
 class GridFSProxy(object):
-
     """Proxy object to handle writing and reading of files to and from GridFS
 
     .. versionadded:: 0.4
@@ -1281,12 +1324,12 @@ class GridFSProxy(object):
                  instance=None,
                  db_alias=DEFAULT_CONNECTION_NAME,
                  collection_name='fs'):
-        self.grid_id = grid_id                  # Store GridFS id for file
+        self.grid_id = grid_id  # Store GridFS id for file
         self.key = key
         self.instance = instance
         self.db_alias = db_alias
         self.collection_name = collection_name
-        self.newfile = None                     # Used for partial writes
+        self.newfile = None  # Used for partial writes
         self.gridout = None
 
     def __getattr__(self, name):
@@ -1350,7 +1393,7 @@ class GridFSProxy(object):
             if self.gridout is None:
                 self.gridout = self.fs.get(self.grid_id)
             return self.gridout
-        except:
+        except Exception:
             # File has been deleted
             return None
 
@@ -1388,7 +1431,7 @@ class GridFSProxy(object):
         else:
             try:
                 return gridout.read(size)
-            except:
+            except Exception:
                 return ""
 
     def delete(self):
@@ -1413,7 +1456,6 @@ class GridFSProxy(object):
 
 
 class FileField(BaseField):
-
     """A GridFS storage field.
 
     .. versionadded:: 0.4
@@ -1447,14 +1489,14 @@ class FileField(BaseField):
     def __set__(self, instance, value):
         key = self.name
         if ((hasattr(value, 'read') and not
-             isinstance(value, GridFSProxy)) or isinstance(value, str_types)):
+                isinstance(value, GridFSProxy)) or isinstance(value, str_types)):
             # using "FileField() = file/string" notation
             grid_file = instance._data.get(self.name)
             # If a file already exists, delete it
             if grid_file:
                 try:
                     grid_file.delete()
-                except:
+                except Exception:
                     pass
 
             # Create a new proxy object as we don't already have one
@@ -1497,7 +1539,6 @@ class FileField(BaseField):
 
 
 class ImageGridFsProxy(GridFSProxy):
-
     """
     Proxy for ImageField
 
@@ -1521,6 +1562,7 @@ class ImageGridFsProxy(GridFSProxy):
             raise ValidationError('Invalid image: %s' % e)
 
         # Progressive JPEG
+        # TODO: fixme, at least unused, at worst bad implementation
         progressive = img.info.get('progressive') or False
 
         if (kwargs.get('progressive') and
@@ -1581,7 +1623,7 @@ class ImageGridFsProxy(GridFSProxy):
         if out and out.thumbnail_id:
             self.fs.delete(out.thumbnail_id)
 
-        return super(ImageGridFsProxy, self).delete(*args, **kwargs)
+        return super(ImageGridFsProxy, self).delete()
 
     def _put_thumbnail(self, thumbnail, format, progressive, **kwargs):
         w, h = thumbnail.size
@@ -1636,7 +1678,6 @@ class ImproperlyConfigured(Exception):
 
 
 class ImageField(FileField):
-
     """
     A Image File storage field.
 
@@ -1675,7 +1716,6 @@ class ImageField(FileField):
 
 
 class SequenceField(BaseField):
-
     """Provides a sequential counter see:
      http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers
 
@@ -1687,12 +1727,21 @@ class SequenceField(BaseField):
              cluster of machines, it is easier to create an object ID than have
              global, uniformly increasing sequence numbers.
 
+    :param collection_name:  Name of the counter collection (default 'mongoengine.counters')
+    :param sequence_name: Name of the sequence in the collection (default 'ClassName.counter')
+    :param value_decorator: Any callable to use as a counter (default int)
+
     Use any callable as `value_decorator` to transform calculated counter into
     any value suitable for your needs, e.g. string or hexadecimal
     representation of the default integer counter value.
 
-    .. versionadded:: 0.5
+    .. note::
 
+        In case the counter is defined in the abstract document, it will be
+        common to all inherited documents and the default sequence name will
+        be the class name of the abstract document.
+
+    .. versionadded:: 0.5
     .. versionchanged:: 0.8 added `value_decorator`
     """
 
@@ -1707,7 +1756,7 @@ class SequenceField(BaseField):
         self.sequence_name = sequence_name
         self.value_decorator = (callable(value_decorator) and
                                 value_decorator or self.VALUE_DECORATOR)
-        return super(SequenceField, self).__init__(*args, **kwargs)
+        super(SequenceField, self).__init__(*args, **kwargs)
 
     def generate(self):
         """
@@ -1753,7 +1802,7 @@ class SequenceField(BaseField):
         if self.sequence_name:
             return self.sequence_name
         owner = self.owner_document
-        if issubclass(owner, Document):
+        if issubclass(owner, Document) and not owner._meta.get('abstract'):
             return owner._get_collection_name()
         else:
             return ''.join('_%s' % c if c.isupper() else c
@@ -1790,7 +1839,6 @@ class SequenceField(BaseField):
 
 
 class UUIDField(BaseField):
-
     """A UUID field.
 
     .. versionadded:: 0.6
@@ -1816,7 +1864,7 @@ class UUIDField(BaseField):
                 if not isinstance(value, basestring):
                     value = unicode(value)
                 return uuid.UUID(value)
-            except:
+            except Exception:
                 return original_value
         return value
 
@@ -1837,13 +1885,12 @@ class UUIDField(BaseField):
             if not isinstance(value, basestring):
                 value = str(value)
             try:
-                value = uuid.UUID(value)
+                uuid.UUID(value)
             except Exception, exc:
                 self.error('Could not convert to UUID: %s' % exc)
 
 
 class GeoPointField(BaseField):
-
     """A list storing a longitude and latitude coordinate.
 
     .. note:: this represents a generic point in a 2D plane and a legacy way of
@@ -1873,7 +1920,6 @@ class GeoPointField(BaseField):
 
 
 class PointField(GeoJsonBaseField):
-
     """A GeoJSON field storing a longitude and latitude coordinate.
 
     The data is represented as:
@@ -1894,7 +1940,6 @@ class PointField(GeoJsonBaseField):
 
 
 class LineStringField(GeoJsonBaseField):
-
     """A GeoJSON field storing a line of longitude and latitude coordinates.
 
     The data is represented as:
@@ -1914,7 +1959,6 @@ class LineStringField(GeoJsonBaseField):
 
 
 class PolygonField(GeoJsonBaseField):
-
     """A GeoJSON field storing a polygon of longitude and latitude coordinates.
 
     The data is represented as:
@@ -1937,7 +1981,6 @@ class PolygonField(GeoJsonBaseField):
 
 
 class MultiPointField(GeoJsonBaseField):
-
     """A GeoJSON field storing a list of Points.
 
     The data is represented as:
@@ -1958,7 +2001,6 @@ class MultiPointField(GeoJsonBaseField):
 
 
 class MultiLineStringField(GeoJsonBaseField):
-
     """A GeoJSON field storing a list of LineStrings.
 
     The data is represented as:
@@ -1979,7 +2021,6 @@ class MultiLineStringField(GeoJsonBaseField):
 
 
 class MultiPolygonField(GeoJsonBaseField):
-
     """A GeoJSON field storing  list of Polygons.
 
     The data is represented as:

mongoengine/python_support.py

@@ -1,18 +1,39 @@
-"""Helper functions and types to aid with Python 2.5 - 3 support."""
+"""Helper functions and types to aid with Python 2.6 - 3 support."""
 
 import sys
+import warnings
+
+import pymongo
+
+
+# Show a deprecation warning for people using Python v2.6
+# TODO remove in mongoengine v0.11.0
+if sys.version_info[0] == 2 and sys.version_info[1] == 6:
+    warnings.warn(
+        'Python v2.6 support is deprecated and is going to be dropped '
+        'entirely in the upcoming v0.11.0 release. Update your Python '
+        'version if you want to have access to the latest features and '
+        'bug fixes in MongoEngine.',
+        DeprecationWarning
+    )
+
+if pymongo.version_tuple[0] < 3:
+    IS_PYMONGO_3 = False
+else:
+    IS_PYMONGO_3 = True
 
 PY3 = sys.version_info[0] == 3
 
 if PY3:
     import codecs
     from io import BytesIO as StringIO
+
     # return s converted to binary.  b('test') should be equivalent to b'test'
     def b(s):
         return codecs.latin_1_encode(s)[0]
 
     bin_type = bytes
-    txt_type   = str
+    txt_type = str
 else:
     try:
         from cStringIO import StringIO

mongoengine/queryset/__init__.py

@@ -1,6 +1,6 @@
-from mongoengine.errors import (DoesNotExist, MultipleObjectsReturned,
+from mongoengine.errors import (DoesNotExist, InvalidQueryError,
-                                InvalidQueryError, OperationError,
+                                MultipleObjectsReturned, NotUniqueError,
-                                NotUniqueError)
+                                OperationError)
 from mongoengine.queryset.field_list import *
 from mongoengine.queryset.manager import *
 from mongoengine.queryset.queryset import *

mongoengine/queryset/base.py

@@ -7,24 +7,27 @@ import pprint
 import re
 import warnings
 
-from bson import SON
+from bson import SON, json_util
 from bson.code import Code
-from bson import json_util
 import pymongo
 import pymongo.errors
 from pymongo.common import validate_read_preference
 
 from mongoengine import signals
+from mongoengine.base.common import get_document
+from mongoengine.common import _import_class
 from mongoengine.connection import get_db
 from mongoengine.context_managers import switch_db
-from mongoengine.common import _import_class
+from mongoengine.errors import (InvalidQueryError, LookUpError,
-from mongoengine.base.common import get_document
+                                NotUniqueError, OperationError)
-from mongoengine.errors import (OperationError, NotUniqueError,
+from mongoengine.python_support import IS_PYMONGO_3
-                                InvalidQueryError, LookUpError)
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
 
+if IS_PYMONGO_3:
+    from pymongo.collection import ReturnDocument
+
 
 __all__ = ('BaseQuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL')
 
@@ -39,7 +42,6 @@ RE_TYPE = type(re.compile(''))
 
 
 class BaseQuerySet(object):
-
     """A set of results returned from a query. Wraps a MongoDB cursor,
     providing :class:`~mongoengine.Document` objects as the results.
     """
@@ -80,11 +82,12 @@ class BaseQuerySet(object):
         self._limit = None
         self._skip = None
         self._hint = -1  # Using -1 as None is a valid value for hint
+        self._batch_size = None
         self.only_fields = []
         self._max_time_ms = None
 
-    def __call__(self, q_obj=None, class_check=True, slave_okay=False,
+    def __call__(self, q_obj=None, class_check=True, read_preference=None,
-                 read_preference=None, **query):
+                 **query):
         """Filter the selected documents by calling the
         :class:`~mongoengine.queryset.QuerySet` with a query.
 
@@ -94,9 +97,7 @@ class BaseQuerySet(object):
             objects, only the last one will be used
         :param class_check: If set to False bypass class name check when
             querying collection
-        :param slave_okay: if True, allows this query to be run against a
+        :param read_preference: if set, overrides connection-level
-            replica secondary.
-        :params read_preference: if set, overrides connection-level
             read_preference from `ReplicaSetConnection`.
         :param query: Django-style query keyword arguments
         """
@@ -122,9 +123,40 @@ class BaseQuerySet(object):
 
         return queryset
 
-    def __getitem__(self, key):
+    def __getstate__(self):
-        """Support skip and limit using getitem and slicing syntax.
         """
+        Need for pickling queryset
+
+        See https://github.com/MongoEngine/mongoengine/issues/442
+        """
+
+        obj_dict = self.__dict__.copy()
+
+        # don't picke collection, instead pickle collection params
+        obj_dict.pop("_collection_obj")
+
+        # don't pickle cursor
+        obj_dict["_cursor_obj"] = None
+
+        return obj_dict
+
+    def __setstate__(self, obj_dict):
+        """
+        Need for pickling queryset
+
+        See https://github.com/MongoEngine/mongoengine/issues/442
+        """
+
+        obj_dict["_collection_obj"] = obj_dict["_document"]._get_collection()
+
+        # update attributes
+        self.__dict__.update(obj_dict)
+
+        # forse load cursor
+        # self._cursor
+
+    def __getitem__(self, key):
+        """Support skip and limit using getitem and slicing syntax."""
         queryset = self.clone()
 
         # Slice provided
@@ -158,7 +190,8 @@ class BaseQuerySet(object):
             if queryset._as_pymongo:
                 return queryset._get_as_pymongo(queryset._cursor[key])
             return queryset._document._from_son(queryset._cursor[key],
-                                               _auto_dereference=self._auto_dereference, only_fields=self.only_fields)
+                                                _auto_dereference=self._auto_dereference,
+                                                only_fields=self.only_fields)
 
         raise AttributeError
 
@@ -200,7 +233,8 @@ class BaseQuerySet(object):
         :param language:  The language that determines the list of stop words
             for the search and the rules for the stemmer and tokenizer.
             If not specified, the search uses the default language of the index.
-            For supported languages, see `Text Search Languages <http://docs.mongodb.org/manual/reference/text-search-languages/#text-search-languages>`.
+            For supported languages, see
+            `Text Search Languages <http://docs.mongodb.org/manual/reference/text-search-languages/#text-search-languages>`.
         """
         queryset = self.clone()
         if queryset._search_text:
@@ -242,6 +276,8 @@ class BaseQuerySet(object):
         except StopIteration:
             return result
 
+        # If we were able to retrieve the 2nd doc, rewind the cursor and
+        # raise the MultipleObjectsReturned exception.
         queryset.rewind()
         message = u'%d items returned, instead of 1' % queryset.count()
         raise queryset._document.MultipleObjectsReturned(message)
@@ -253,54 +289,6 @@ class BaseQuerySet(object):
         """
         return self._document(**kwargs).save()
 
-    def get_or_create(self, write_concern=None, auto_save=True,
-                      *q_objs, **query):
-        """Retrieve unique object or create, if it doesn't exist. Returns a
-        tuple of ``(object, created)``, where ``object`` is the retrieved or
-        created object and ``created`` is a boolean specifying whether a new
-        object was created. Raises
-        :class:`~mongoengine.queryset.MultipleObjectsReturned` or
-        `DocumentName.MultipleObjectsReturned` if multiple results are found.
-        A new document will be created if the document doesn't exists; a
-        dictionary of default values for the new document may be provided as a
-        keyword argument called :attr:`defaults`.
-
-        .. note:: This requires two separate operations and therefore a
-            race condition exists.  Because there are no transactions in
-            mongoDB other approaches should be investigated, to ensure you
-            don't accidentally duplicate data when using this method.  This is
-            now scheduled to be removed before 1.0
-
-        :param write_concern: optional extra keyword arguments used if we
-            have to create a new document.
-            Passes any write_concern onto :meth:`~mongoengine.Document.save`
-
-        :param auto_save: if the object is to be saved automatically if
-            not found.
-
-        .. deprecated:: 0.8
-        .. versionchanged:: 0.6 - added `auto_save`
-        .. versionadded:: 0.3
-        """
-        msg = ("get_or_create is scheduled to be deprecated.  The approach is "
-               "flawed without transactions. Upserts should be preferred.")
-        warnings.warn(msg, DeprecationWarning)
-
-        defaults = query.get('defaults', {})
-        if 'defaults' in query:
-            del query['defaults']
-
-        try:
-            doc = self.get(*q_objs, **query)
-            return doc, False
-        except self._document.DoesNotExist:
-            query.update(defaults)
-            doc = self._document(**query)
-
-            if auto_save:
-                doc.save(write_concern=write_concern)
-            return doc, True
-
     def first(self):
         """Retrieve the first object matching the query.
         """
@@ -311,10 +299,11 @@ class BaseQuerySet(object):
             result = None
         return result
 
-    def insert(self, doc_or_docs, load_bulk=True, write_concern=None):
+    def insert(self, doc_or_docs, load_bulk=True,
+               write_concern=None, signal_kwargs=None):
         """bulk insert documents
 
-        :param docs_or_doc: a document or list of documents to be inserted
+        :param doc_or_docs: a document or list of documents to be inserted
         :param load_bulk (optional): If True returns the list of document
             instances
         :param write_concern: Extra keyword arguments are passed down to
@@ -324,11 +313,15 @@ class BaseQuerySet(object):
                 ``insert(..., {w: 2, fsync: True})`` will wait until at least
                 two servers have recorded the write and will force an fsync on
                 each server being written to.
+        :parm signal_kwargs: (optional) kwargs dictionary to be passed to
+            the signal calls.
 
         By default returns document instances, set ``load_bulk`` to False to
         return just ``ObjectIds``
 
         .. versionadded:: 0.5
+        .. versionchanged:: 0.10.7
+            Add signal_kwargs argument
         """
         Document = _import_class('Document')
 
@@ -341,7 +334,6 @@ class BaseQuerySet(object):
             return_one = True
             docs = [docs]
 
-        raw = []
         for doc in docs:
             if not isinstance(doc, self._document):
                 msg = ("Some documents inserted aren't instances of %s"
@@ -350,9 +342,12 @@ class BaseQuerySet(object):
             if doc.pk and not doc._created:
                 msg = "Some documents have ObjectIds use doc.update() instead"
                 raise OperationError(msg)
-            raw.append(doc.to_mongo())
 
-        signals.pre_bulk_insert.send(self._document, documents=docs)
+        signal_kwargs = signal_kwargs or {}
+        signals.pre_bulk_insert.send(self._document,
+                                     documents=docs, **signal_kwargs)
+
+        raw = [doc.to_mongo() for doc in docs]
         try:
             ids = self._collection.insert(raw, **write_concern)
         except pymongo.errors.DuplicateKeyError, err:
@@ -369,7 +364,7 @@ class BaseQuerySet(object):
 
         if not load_bulk:
             signals.post_bulk_insert.send(
-                self._document, documents=docs, loaded=False)
+                self._document, documents=docs, loaded=False, **signal_kwargs)
             return return_one and ids[0] or ids
 
         documents = self.in_bulk(ids)
@@ -377,7 +372,7 @@ class BaseQuerySet(object):
         for obj_id in ids:
             results.append(documents.get(obj_id))
         signals.post_bulk_insert.send(
-            self._document, documents=results, loaded=True)
+            self._document, documents=results, loaded=True, **signal_kwargs)
         return return_one and results[0] or results
 
     def count(self, with_limit_and_skip=False):
@@ -391,7 +386,7 @@ class BaseQuerySet(object):
             return 0
         return self._cursor.count(with_limit_and_skip=with_limit_and_skip)
 
-    def delete(self, write_concern=None, _from_doc_delete=False):
+    def delete(self, write_concern=None, _from_doc_delete=False, cascade_refs=None):
         """Delete the documents matched by the query.
 
         :param write_concern: Extra keyword arguments are passed down which
@@ -423,7 +418,7 @@ class BaseQuerySet(object):
         if call_document_delete:
             cnt = 0
             for doc in queryset:
-                doc.delete(write_concern=write_concern)
+                doc.delete(**write_concern)
                 cnt += 1
             return cnt
 
@@ -447,11 +442,15 @@ class BaseQuerySet(object):
                 continue
             rule = doc._meta['delete_rules'][rule_entry]
             if rule == CASCADE:
-                ref_q = document_cls.objects(**{field_name + '__in': self})
+                cascade_refs = set() if cascade_refs is None else cascade_refs
+                # Handle recursive reference
+                if doc._collection == document_cls._collection:
+                    for ref in queryset:
+                        cascade_refs.add(ref.id)
+                ref_q = document_cls.objects(**{field_name + '__in': self, 'pk__nin': cascade_refs})
                 ref_q_count = ref_q.count()
-                if (doc != document_cls and ref_q_count > 0
+                if ref_q_count > 0:
-                        or (doc == document_cls and ref_q_count > 0)):
+                    ref_q.delete(write_concern=write_concern, cascade_refs=cascade_refs)
-                    ref_q.delete(write_concern=write_concern)
             elif rule == NULLIFY:
                 document_cls.objects(**{field_name + '__in': self}).update(
                     write_concern=write_concern, **{'unset__%s' % field_name: 1})
@@ -461,13 +460,14 @@ class BaseQuerySet(object):
                     **{'pull_all__%s' % field_name: self})
 
         result = queryset._collection.remove(queryset._query, **write_concern)
-        return result["n"]
+        if result:
+            return result.get("n")
 
     def update(self, upsert=False, multi=True, write_concern=None,
                full_result=False, **update):
         """Perform an atomic update on the fields matched by the query.
 
-        :param upsert: Any existing document with that "_id" is overwritten.
+        :param upsert: insert if document doesn't exist (default ``False``)
         :param multi: Update multiple documents.
         :param write_concern: Extra keyword arguments are passed down which
             will be used as options for the resultant
@@ -513,10 +513,37 @@ class BaseQuerySet(object):
                 raise OperationError(message)
             raise OperationError(u'Update failed (%s)' % unicode(err))
 
-    def update_one(self, upsert=False, write_concern=None, **update):
+    def upsert_one(self, write_concern=None, **update):
-        """Perform an atomic update on first field matched by the query.
+        """Overwrite or add the first document matched by the query.
 
-        :param upsert: Any existing document with that "_id" is overwritten.
+        :param write_concern: Extra keyword arguments are passed down which
+            will be used as options for the resultant
+            ``getLastError`` command.  For example,
+            ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
+            wait until at least two servers have recorded the write and
+            will force an fsync on the primary server.
+        :param update: Django-style update keyword arguments
+
+        :returns the new or overwritten document
+
+        .. versionadded:: 0.10.2
+        """
+
+        atomic_update = self.update(multi=False, upsert=True,
+                                    write_concern=write_concern,
+                                    full_result=True, **update)
+
+        if atomic_update['updatedExisting']:
+            document = self.get()
+        else:
+            document = self._document.objects.with_id(atomic_update['upserted'])
+        return document
+
+    def update_one(self, upsert=False, write_concern=None, **update):
+        """Perform an atomic update on the fields of the first document
+        matched by the query.
+
+        :param upsert: insert if document doesn't exist (default ``False``)
         :param write_concern: Extra keyword arguments are passed down which
             will be used as options for the resultant
             ``getLastError`` command.  For example,
@@ -545,7 +572,7 @@ class BaseQuerySet(object):
 
         :param upsert: insert if document doesn't exist (default ``False``)
         :param full_response: return the entire response object from the
-            server (default ``False``)
+            server (default ``False``, not available for PyMongo 3+)
         :param remove: remove rather than updating (default ``False``)
         :param new: return updated rather than original document
             (default ``False``)
@@ -563,13 +590,31 @@ class BaseQuerySet(object):
 
         queryset = self.clone()
         query = queryset._query
-        update = transform.update(queryset._document, **update)
+        if not IS_PYMONGO_3 or not remove:
+            update = transform.update(queryset._document, **update)
         sort = queryset._ordering
 
         try:
-            result = queryset._collection.find_and_modify(
+            if IS_PYMONGO_3:
-                query, update, upsert=upsert, sort=sort, remove=remove, new=new,
+                if full_response:
-                full_response=full_response, **self._cursor_args)
+                    msg = "With PyMongo 3+, it is not possible anymore to get the full response."
+                    warnings.warn(msg, DeprecationWarning)
+                if remove:
+                    result = queryset._collection.find_one_and_delete(
+                        query, sort=sort, **self._cursor_args)
+                else:
+                    if new:
+                        return_doc = ReturnDocument.AFTER
+                    else:
+                        return_doc = ReturnDocument.BEFORE
+                    result = queryset._collection.find_one_and_update(
+                        query, update, upsert=upsert, sort=sort, return_document=return_doc,
+                        **self._cursor_args)
+
+            else:
+                result = queryset._collection.find_and_modify(
+                    query, update, upsert=upsert, sort=sort, remove=remove, new=new,
+                    full_response=full_response, **self._cursor_args)
         except pymongo.errors.DuplicateKeyError, err:
             raise NotUniqueError(u"Update failed (%s)" % err)
         except pymongo.errors.OperationFailure, err:
@@ -621,9 +666,10 @@ class BaseQuerySet(object):
                 doc_map[doc['_id']] = self._get_as_pymongo(doc)
         else:
             for doc in docs:
-                doc_map[doc['_id']] = self._document._from_son(doc,
+                doc_map[doc['_id']] = self._document._from_son(
-                        only_fields=self.only_fields,
+                    doc,
-                        _auto_dereference=self._auto_dereference)
+                    only_fields=self.only_fields,
+                    _auto_dereference=self._auto_dereference)
 
         return doc_map
 
@@ -643,7 +689,8 @@ class BaseQuerySet(object):
         return self
 
     def using(self, alias):
-        """This method is for controlling which database the QuerySet will be evaluated against if you are using more than one database.
+        """This method is for controlling which database the QuerySet will be
+        evaluated against if you are using more than one database.
 
         :param alias: The database alias
 
@@ -706,11 +753,7 @@ class BaseQuerySet(object):
         :param n: the maximum number of objects to return
         """
         queryset = self.clone()
-        if n == 0:
+        queryset._limit = n if n != 0 else 1
-            queryset._cursor.limit(1)
-        else:
-            queryset._cursor.limit(n)
-        queryset._limit = n
         # Return self to allow chaining
         return queryset
 
@@ -721,7 +764,6 @@ class BaseQuerySet(object):
         :param n: the number of objects to skip before returning results
         """
         queryset = self.clone()
-        queryset._cursor.skip(n)
         queryset._skip = n
         return queryset
 
@@ -739,10 +781,22 @@ class BaseQuerySet(object):
         .. versionadded:: 0.5
         """
         queryset = self.clone()
-        queryset._cursor.hint(index)
         queryset._hint = index
         return queryset
 
+    def batch_size(self, size):
+        """Limit the number of documents returned in a single batch (each
+        batch requires a round trip to the server).
+
+        See http://api.mongodb.com/python/current/api/pymongo/cursor.html#pymongo.cursor.Cursor.batch_size
+        for details.
+
+        :param size: desired size of each batch.
+        """
+        queryset = self.clone()
+        queryset._batch_size = size
+        return queryset
+
     def distinct(self, field):
         """Return a list of distinct values for a given field.
 
@@ -854,7 +908,6 @@ class BaseQuerySet(object):
         cleaned_fields = []
         for key, value in kwargs.items():
             parts = key.split('__')
-            op = None
             if parts[0] in operators:
                 op = parts.pop(0)
                 value = {'$' + op: value}
@@ -896,6 +949,14 @@ class BaseQuerySet(object):
         queryset._ordering = queryset._get_order_by(keys)
         return queryset
 
+    def comment(self, text):
+        """Add a comment to the query.
+
+        See https://docs.mongodb.com/manual/reference/method/cursor.comment/#cursor.comment
+        for details.
+        """
+        return self._chainable_method("comment", text)
+
     def explain(self, format=False):
         """Return an explain plan record for the
         :class:`~mongoengine.queryset.QuerySet`\ 's cursor.
@@ -907,13 +968,18 @@ class BaseQuerySet(object):
             plan = pprint.pformat(plan)
         return plan
 
+    # DEPRECATED. Has no more impact on PyMongo 3+
     def snapshot(self, enabled):
         """Enable or disable snapshot mode when querying.
 
         :param enabled: whether or not snapshot mode is enabled
 
         ..versionchanged:: 0.5 - made chainable
+        .. deprecated:: Ignored with PyMongo 3+
         """
+        if IS_PYMONGO_3:
+            msg = "snapshot is deprecated as it has no impact when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
         queryset = self.clone()
         queryset._snapshot = enabled
         return queryset
@@ -929,11 +995,17 @@ class BaseQuerySet(object):
         queryset._timeout = enabled
         return queryset
 
+    # DEPRECATED. Has no more impact on PyMongo 3+
     def slave_okay(self, enabled):
         """Enable or disable the slave_okay when querying.
 
         :param enabled: whether or not the slave_okay is enabled
+
+        .. deprecated:: Ignored with PyMongo 3+
         """
+        if IS_PYMONGO_3:
+            msg = "slave_okay is deprecated as it has no impact when using PyMongo 3+."
+            warnings.warn(msg, DeprecationWarning)
         queryset = self.clone()
         queryset._slave_okay = enabled
         return queryset
@@ -947,6 +1019,7 @@ class BaseQuerySet(object):
         validate_read_preference('read_preference', read_preference)
         queryset = self.clone()
         queryset._read_preference = read_preference
+        queryset._cursor_obj = None  # we need to re-create the cursor object whenever we apply read_preference
         return queryset
 
     def scalar(self, *fields):
@@ -980,7 +1053,7 @@ class BaseQuerySet(object):
         """Instead of returning Document instances, return raw values from
         pymongo.
 
-        :param coerce_type: Field types (if applicable) would be use to
+        :param coerce_types: Field types (if applicable) would be use to
             coerce types.
         """
         queryset = self.clone()
@@ -1219,103 +1292,62 @@ class BaseQuerySet(object):
     def sum(self, field):
         """Sum over the values of the specified field.
 
-        :param field: the field to sum over; use dot-notation to refer to
+        :param field: the field to sum over; use dot notation to refer to
             embedded document fields
-
-        .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work
-            with sharding.
         """
-        map_func = """
+        db_field = self._fields_to_dbfields([field]).pop()
-            function() {
+        pipeline = [
-                var path = '{{~%(field)s}}'.split('.'),
+            {'$match': self._query},
-                field = this;
+            {'$group': {'_id': 'sum', 'total': {'$sum': '$' + db_field}}}
+        ]
 
-                for (p in path) {
+        # if we're performing a sum over a list field, we sum up all the
-                    if (typeof field != 'undefined')
+        # elements in the list, hence we need to $unwind the arrays first
-                       field = field[path[p]];
+        ListField = _import_class('ListField')
-                    else
+        field_parts = field.split('.')
-                       break;
+        field_instances = self._document._lookup_field(field_parts)
-                }
+        if isinstance(field_instances[-1], ListField):
+            pipeline.insert(1, {'$unwind': '$' + field})
 
-                if (field && field.constructor == Array) {
+        result = self._document._get_collection().aggregate(pipeline)
-                    field.forEach(function(item) {
+        if IS_PYMONGO_3:
-                        emit(1, item||0);
+            result = tuple(result)
-                    });
-                } else if (typeof field != 'undefined') {
-                    emit(1, field||0);
-                }
-            }
-        """ % dict(field=field)
-
-        reduce_func = Code("""
-            function(key, values) {
-                var sum = 0;
-                for (var i in values) {
-                    sum += values[i];
-                }
-                return sum;
-            }
-        """)
-
-        for result in self.map_reduce(map_func, reduce_func, output='inline'):
-            return result.value
         else:
-            return 0
+            result = result.get('result')
+
+        if result:
+            return result[0]['total']
+        return 0
 
     def average(self, field):
         """Average over the values of the specified field.
 
-        :param field: the field to average over; use dot-notation to refer to
+        :param field: the field to average over; use dot notation to refer to
             embedded document fields
-
-        .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work
-            with sharding.
         """
-        map_func = """
+        db_field = self._fields_to_dbfields([field]).pop()
-            function() {
+        pipeline = [
-                var path = '{{~%(field)s}}'.split('.'),
+            {'$match': self._query},
-                field = this;
+            {'$group': {'_id': 'avg', 'total': {'$avg': '$' + db_field}}}
+        ]
 
-                for (p in path) {
+        # if we're performing an average over a list field, we average out
-                    if (typeof field != 'undefined')
+        # all the elements in the list, hence we need to $unwind the arrays
-                       field = field[path[p]];
+        # first
-                    else
+        ListField = _import_class('ListField')
-                       break;
+        field_parts = field.split('.')
-                }
+        field_instances = self._document._lookup_field(field_parts)
+        if isinstance(field_instances[-1], ListField):
+            pipeline.insert(1, {'$unwind': '$' + field})
 
-                if (field && field.constructor == Array) {
+        result = self._document._get_collection().aggregate(pipeline)
-                    field.forEach(function(item) {
+        if IS_PYMONGO_3:
-                        emit(1, {t: item||0, c: 1});
+            result = tuple(result)
-                    });
-                } else if (typeof field != 'undefined') {
-                    emit(1, {t: field||0, c: 1});
-                }
-            }
-        """ % dict(field=field)
-
-        reduce_func = Code("""
-            function(key, values) {
-                var out = {t: 0, c: 0};
-                for (var i in values) {
-                    var value = values[i];
-                    out.t += value.t;
-                    out.c += value.c;
-                }
-                return out;
-            }
-        """)
-
-        finalize_func = Code("""
-            function(key, value) {
-                return value.t / value.c;
-            }
-        """)
-
-        for result in self.map_reduce(map_func, reduce_func,
-                                      finalize_f=finalize_func, output='inline'):
-            return result.value
         else:
-            return 0
+            result = result.get('result')
+        if result:
+            return result[0]['total']
+        return 0
 
     def item_frequencies(self, field, normalize=False, map_reduce=True):
         """Returns a dictionary of all items present in a field across
@@ -1327,7 +1359,7 @@ class BaseQuerySet(object):
             Can only do direct simple mappings and cannot map across
             :class:`~mongoengine.fields.ReferenceField` or
             :class:`~mongoengine.fields.GenericReferenceField` for more complex
-            counting a manual map reduce call would is required.
+            counting a manual map reduce call is required.
 
         If the field is a :class:`~mongoengine.fields.ListField`, the items within
         each list will be counted individually.
@@ -1383,22 +1415,34 @@ class BaseQuerySet(object):
 
     @property
     def _cursor_args(self):
-        cursor_args = {
+        if not IS_PYMONGO_3:
-            'snapshot': self._snapshot,
+            fields_name = 'fields'
-            'timeout': self._timeout
+            cursor_args = {
-        }
+                'timeout': self._timeout,
-        if self._read_preference is not None:
+                'snapshot': self._snapshot
-            cursor_args['read_preference'] = self._read_preference
+            }
+            if self._read_preference is not None:
+                cursor_args['read_preference'] = self._read_preference
+            else:
+                cursor_args['slave_okay'] = self._slave_okay
         else:
-            cursor_args['slave_okay'] = self._slave_okay
+            fields_name = 'projection'
+            # snapshot is not handled at all by PyMongo 3+
+            # TODO: evaluate similar possibilities using modifiers
+            if self._snapshot:
+                msg = "The snapshot option is not anymore available with PyMongo 3+"
+                warnings.warn(msg, DeprecationWarning)
+            cursor_args = {
+                'no_cursor_timeout': not self._timeout
+            }
         if self._loaded_fields:
-            cursor_args['fields'] = self._loaded_fields.as_dict()
+            cursor_args[fields_name] = self._loaded_fields.as_dict()
 
         if self._search_text:
-            if 'fields' not in cursor_args:
+            if fields_name not in cursor_args:
-                cursor_args['fields'] = {}
+                cursor_args[fields_name] = {}
 
-            cursor_args['fields']['_text_score'] = {'$meta': "textScore"}
+            cursor_args[fields_name]['_text_score'] = {'$meta': "textScore"}
 
         return cursor_args
 
@@ -1406,8 +1450,16 @@ class BaseQuerySet(object):
     def _cursor(self):
         if self._cursor_obj is None:
 
-            self._cursor_obj = self._collection.find(self._query,
+            # In PyMongo 3+, we define the read preference on a collection
-                                                     **self._cursor_args)
+            # level, not a cursor level. Thus, we need to get a cloned
+            # collection object using `with_options` first.
+            if IS_PYMONGO_3 and self._read_preference is not None:
+                self._cursor_obj = self._collection\
+                    .with_options(read_preference=self._read_preference)\
+                    .find(self._query, **self._cursor_args)
+            else:
+                self._cursor_obj = self._collection.find(self._query,
+                                                         **self._cursor_args)
             # Apply where clauses to cursor
             if self._where_clause:
                 where_clause = self._sub_js_fields(self._where_clause)
@@ -1431,6 +1483,9 @@ class BaseQuerySet(object):
             if self._hint != -1:
                 self._cursor_obj.hint(self._hint)
 
+            if self._batch_size is not None:
+                self._cursor_obj.batch_size(self._batch_size)
+
         return self._cursor_obj
 
     def __deepcopy__(self, memo):
@@ -1575,7 +1630,7 @@ class BaseQuerySet(object):
 
         return frequencies
 
-    def _fields_to_dbfields(self, fields, subdoc=False):
+    def _fields_to_dbfields(self, fields):
         """Translate fields paths to its db equivalents"""
         ret = []
         subclasses = []
@@ -1597,7 +1652,7 @@ class BaseQuerySet(object):
                         ret.append(subfield)
                         found = True
                         break
-                    except LookUpError, e:
+                    except LookUpError:
                         pass
 
                 if not found:
@@ -1624,7 +1679,7 @@ class BaseQuerySet(object):
             key = key.replace('__', '.')
             try:
                 key = self._document._translate_field_name(key)
-            except:
+            except Exception:
                 pass
             key_list.append((key, direction))
 

mongoengine/queryset/field_list.py

@@ -1,4 +1,3 @@
-
 __all__ = ('QueryFieldList',)
 
 

mongoengine/queryset/manager.py

@@ -29,7 +29,7 @@ class QuerySetManager(object):
         Document.objects is accessed.
         """
         if instance is not None:
-            # Document class being used rather than a document object
+            # Document object being used rather than a document class
             return self
 
         # owner is the document that contains the QuerySetManager

mongoengine/queryset/queryset.py

@@ -1,6 +1,6 @@
 from mongoengine.errors import OperationError
-from mongoengine.queryset.base import (BaseQuerySet, DO_NOTHING, NULLIFY,
+from mongoengine.queryset.base import (BaseQuerySet, CASCADE, DENY, DO_NOTHING,
-                                       CASCADE, DENY, PULL)
+                                       NULLIFY, PULL)
 
 __all__ = ('QuerySet', 'QuerySetNoCache', 'DO_NOTHING', 'NULLIFY', 'CASCADE',
            'DENY', 'PULL')
@@ -27,9 +27,10 @@ class QuerySet(BaseQuerySet):
         in batches of ``ITER_CHUNK_SIZE``.
 
         If ``self._has_more`` the cursor hasn't been exhausted so cache then
-        batch.  Otherwise iterate the result_cache.
+        batch. Otherwise iterate the result_cache.
         """
         self._iter = True
+
         if self._has_more:
             return self._iter_results()
 
@@ -38,14 +39,16 @@ class QuerySet(BaseQuerySet):
 
     def __len__(self):
         """Since __len__ is called quite frequently (for example, as part of
-        list(qs) we populate the result cache and cache the length.
+        list(qs)), we populate the result cache and cache the length.
         """
         if self._len is not None:
             return self._len
+
+        # Populate the result cache with *all* of the docs in the cursor
         if self._has_more:
-            # populate the cache
             list(self._iter_results())
 
+        # Cache the length of the complete result cache and return it
         self._len = len(self._result_cache)
         return self._len
 
@@ -61,22 +64,36 @@ class QuerySet(BaseQuerySet):
             data[-1] = "...(remaining elements truncated)..."
         return repr(data)
 
-
     def _iter_results(self):
         """A generator for iterating over the result cache.
 
-        Also populates the cache if there are more possible results to yield.
+        Also populates the cache if there are more possible results to
-        Raises StopIteration when there are no more results"""
+        yield. Raises StopIteration when there are no more results.
+        """
         if self._result_cache is None:
             self._result_cache = []
+
         pos = 0
         while True:
-            upper = len(self._result_cache)
+
-            while pos < upper:
+            # For all positions lower than the length of the current result
+            # cache, serve the docs straight from the cache w/o hitting the
+            # database.
+            # XXX it's VERY important to compute the len within the `while`
+            # condition because the result cache might expand mid-iteration
+            # (e.g. if we call len(qs) inside a loop that iterates over the
+            # queryset). Fortunately len(list) is O(1) in Python, so this
+            # doesn't cause performance issues.
+            while pos < len(self._result_cache):
                 yield self._result_cache[pos]
-                pos = pos + 1
+                pos += 1
+
+            # Raise StopIteration if we already established there were no more
+            # docs in the db cursor.
             if not self._has_more:
                 raise StopIteration
+
+            # Otherwise, populate more of the cache and repeat.
             if len(self._result_cache) <= pos:
                 self._populate_cache()
 
@@ -87,12 +104,22 @@ class QuerySet(BaseQuerySet):
         """
         if self._result_cache is None:
             self._result_cache = []
-        if self._has_more:
+
-            try:
+        # Skip populating the cache if we already established there are no
-                for i in xrange(ITER_CHUNK_SIZE):
+        # more docs to pull from the database.
-                    self._result_cache.append(self.next())
+        if not self._has_more:
-            except StopIteration:
+            return
-                self._has_more = False
+
+        # Pull in ITER_CHUNK_SIZE docs from the database and store them in
+        # the result cache.
+        try:
+            for i in xrange(ITER_CHUNK_SIZE):
+                self._result_cache.append(self.next())
+        except StopIteration:
+            # Getting this exception means there are no more docs in the
+            # db cursor. Set _has_more to False so that we can use that
+            # information in other places.
+            self._has_more = False
 
     def count(self, with_limit_and_skip=False):
         """Count the selected elements in the query.

mongoengine/queryset/transform.py

@@ -1,20 +1,22 @@
 from collections import defaultdict
 
+from bson import ObjectId, SON
+from bson.dbref import DBRef
 import pymongo
-from bson import SON
 
-from mongoengine.connection import get_connection
+from mongoengine.base.fields import UPDATE_OPERATORS
 from mongoengine.common import _import_class
-from mongoengine.errors import InvalidQueryError, LookUpError
+from mongoengine.connection import get_connection
+from mongoengine.errors import InvalidQueryError
+from mongoengine.python_support import IS_PYMONGO_3
 
 __all__ = ('query', 'update')
 
-
 COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod',
                         'all', 'size', 'exists', 'not', 'elemMatch', 'type')
 GEO_OPERATORS = ('within_distance', 'within_spherical_distance',
                  'within_box', 'within_polygon', 'near', 'near_sphere',
-                 'max_distance', 'geo_within', 'geo_within_box',
+                 'max_distance', 'min_distance', 'geo_within', 'geo_within_box',
                  'geo_within_polygon', 'geo_within_center',
                  'geo_within_sphere', 'geo_intersects')
 STRING_OPERATORS = ('contains', 'icontains', 'startswith',
@@ -24,17 +26,14 @@ CUSTOM_OPERATORS = ('match',)
 MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS +
                    STRING_OPERATORS + CUSTOM_OPERATORS)
 
-UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push',
-                    'push_all', 'pull', 'pull_all', 'add_to_set',
-                    'set_on_insert', 'min', 'max')
 
-
+# TODO make this less complex
-def query(_doc_cls=None, _field_operation=False, **query):
+def query(_doc_cls=None, **kwargs):
     """Transform a query from Django-style format to Mongo format.
     """
     mongo_query = {}
     merge_query = defaultdict(list)
-    for key, value in sorted(query.items()):
+    for key, value in sorted(kwargs.items()):
         if key == "__raw__":
             mongo_query.update(value)
             continue
@@ -47,6 +46,10 @@ def query(_doc_cls=None, _field_operation=False, **query):
         if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
             op = parts.pop()
 
+        # Allow to escape operator-like field name by __
+        if len(parts) > 1 and parts[-1] == "":
+            parts.pop()
+
         negate = False
         if len(parts) > 1 and parts[-1] == 'not':
             parts.pop()
@@ -61,6 +64,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
             parts = []
 
             CachedReferenceField = _import_class('CachedReferenceField')
+            GenericReferenceField = _import_class('GenericReferenceField')
 
             cleaned_fields = []
             for field in fields:
@@ -100,17 +104,35 @@ def query(_doc_cls=None, _field_operation=False, **query):
                 # 'in', 'nin' and 'all' require a list of values
                 value = [field.prepare_query_value(op, v) for v in value]
 
+            # If we're querying a GenericReferenceField, we need to alter the
+            # key depending on the value:
+            # * If the value is a DBRef, the key should be "field_name._ref".
+            # * If the value is an ObjectId, the key should be "field_name._ref.$id".
+            if isinstance(field, GenericReferenceField):
+                if isinstance(value, DBRef):
+                    parts[-1] += '._ref'
+                elif isinstance(value, ObjectId):
+                    parts[-1] += '._ref.$id'
+
         # if op and op not in COMPARISON_OPERATORS:
         if op:
             if op in GEO_OPERATORS:
                 value = _geo_operator(field, op, value)
-            elif op in CUSTOM_OPERATORS:
+            elif op in ('match', 'elemMatch'):
-                if op in ('elem_match', 'match'):
+                ListField = _import_class('ListField')
-                    value = field.prepare_query_value(op, value)
+                EmbeddedDocumentField = _import_class('EmbeddedDocumentField')
-                    value = {"$elemMatch": value}
+                if (
+                    isinstance(value, dict) and
+                    isinstance(field, ListField) and
+                    isinstance(field.field, EmbeddedDocumentField)
+                ):
+                    value = query(field.field.document_type, **value)
                 else:
-                    NotImplementedError("Custom method '%s' has not "
+                    value = field.prepare_query_value(op, value)
-                                        "been implemented" % op)
+                value = {"$elemMatch": value}
+            elif op in CUSTOM_OPERATORS:
+                NotImplementedError("Custom method '%s' has not "
+                                    "been implemented" % op)
             elif op not in STRING_OPERATORS:
                 value = {'$' + op: value}
 
@@ -119,35 +141,42 @@ def query(_doc_cls=None, _field_operation=False, **query):
 
         for i, part in indices:
             parts.insert(i, part)
+
         key = '.'.join(parts)
+
         if op is None or key not in mongo_query:
             mongo_query[key] = value
         elif key in mongo_query:
-            if key in mongo_query and isinstance(mongo_query[key], dict):
+            if isinstance(mongo_query[key], dict):
                 mongo_query[key].update(value)
-                # $maxDistance needs to come last - convert to SON
+                # $max/minDistance needs to come last - convert to SON
                 value_dict = mongo_query[key]
-                if ('$maxDistance' in value_dict and '$near' in value_dict):
+                if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \
+                        ('$near' in value_dict or '$nearSphere' in value_dict):
                     value_son = SON()
-                    if isinstance(value_dict['$near'], dict):
+                    for k, v in value_dict.iteritems():
-                        for k, v in value_dict.iteritems():
+                        if k == '$maxDistance' or k == '$minDistance':
-                            if k == '$maxDistance':
+                            continue
-                                continue
+                        value_son[k] = v
-                            value_son[k] = v
+                    # Required for MongoDB >= 2.6, may fail when combining
-                        if (get_connection().max_wire_version <= 1):
+                    # PyMongo 3+ and MongoDB < 2.6
-                            value_son['$maxDistance'] = value_dict[
+                    near_embedded = False
-                                '$maxDistance']
+                    for near_op in ('$near', '$nearSphere'):
-                        else:
+                        if isinstance(value_dict.get(near_op), dict) and (
-                            value_son['$near'] = SON(value_son['$near'])
+                                IS_PYMONGO_3 or get_connection().max_wire_version > 1):
-                            value_son['$near'][
+                            value_son[near_op] = SON(value_son[near_op])
-                                '$maxDistance'] = value_dict['$maxDistance']
+                            if '$maxDistance' in value_dict:
-                    else:
+                                value_son[near_op][
-                        for k, v in value_dict.iteritems():
+                                    '$maxDistance'] = value_dict['$maxDistance']
-                            if k == '$maxDistance':
+                            if '$minDistance' in value_dict:
-                                continue
+                                value_son[near_op][
-                            value_son[k] = v
+                                    '$minDistance'] = value_dict['$minDistance']
-                        value_son['$maxDistance'] = value_dict['$maxDistance']
+                            near_embedded = True
-
+                    if not near_embedded:
+                        if '$maxDistance' in value_dict:
+                            value_son['$maxDistance'] = value_dict['$maxDistance']
+                        if '$minDistance' in value_dict:
+                            value_son['$minDistance'] = value_dict['$minDistance']
                     mongo_query[key] = value_son
             else:
                 # Store for manually merging later
@@ -201,6 +230,10 @@ def update(_doc_cls=None, **update):
         if parts[-1] in COMPARISON_OPERATORS:
             match = parts.pop()
 
+        # Allow to escape operator-like field name by __
+        if len(parts) > 1 and parts[-1] == "":
+            parts.pop()
+
         if _doc_cls:
             # Switch field names to proper names [set in Field(name='foo')]
             try:
@@ -301,7 +334,11 @@ def update(_doc_cls=None, **update):
 
 def _geo_operator(field, op, value):
     """Helper to return the query for a given geo query"""
-    if field._geo_index == pymongo.GEO2D:
+    if op == "max_distance":
+        value = {'$maxDistance': value}
+    elif op == "min_distance":
+        value = {'$minDistance': value}
+    elif field._geo_index == pymongo.GEO2D:
         if op == "within_distance":
             value = {'$within': {'$center': value}}
         elif op == "within_spherical_distance":
@@ -314,8 +351,6 @@ def _geo_operator(field, op, value):
             value = {'$nearSphere': value}
         elif op == 'within_box':
             value = {'$within': {'$box': value}}
-        elif op == "max_distance":
-            value = {'$maxDistance': value}
         else:
             raise NotImplementedError("Geo method '%s' has not "
                                       "been implemented for a GeoPointField" % op)
@@ -334,8 +369,6 @@ def _geo_operator(field, op, value):
             value = {"$geoIntersects": _infer_geometry(value)}
         elif op == "near":
             value = {'$near': _infer_geometry(value)}
-        elif op == "max_distance":
-            value = {'$maxDistance': value}
         else:
             raise NotImplementedError("Geo method '%s' has not "
                                       "been implemented for a %s " % (op, field._name))
@@ -352,20 +385,25 @@ def _infer_geometry(value):
         raise InvalidQueryError("Invalid $geometry dictionary should have "
                                 "type and coordinates keys")
     elif isinstance(value, (list, set)):
+        # TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon?
+        # TODO: should both TypeError and IndexError be alike interpreted?
+
         try:
             value[0][0][0]
             return {"$geometry": {"type": "Polygon", "coordinates": value}}
-        except:
+        except (TypeError, IndexError):
             pass
+
         try:
             value[0][0]
             return {"$geometry": {"type": "LineString", "coordinates": value}}
-        except:
+        except (TypeError, IndexError):
             pass
+
         try:
             value[0]
             return {"$geometry": {"type": "Point", "coordinates": value}}
-        except:
+        except (TypeError, IndexError):
             pass
 
     raise InvalidQueryError("Invalid $geometry data. Can be either a dictionary "

mongoengine/queryset/visitor.py

@@ -1,8 +1,5 @@
 import copy
 
-from itertools import product
-from functools import reduce
-
 from mongoengine.errors import InvalidQueryError
 from mongoengine.queryset import transform
 

mongoengine/signals.py

@@ -6,6 +6,7 @@ __all__ = ['pre_init', 'post_init', 'pre_save', 'pre_save_post_validation',
 signals_available = False
 try:
     from blinker import Namespace
+
     signals_available = True
 except ImportError:
     class Namespace(object):
@@ -27,7 +28,8 @@ except ImportError:
             raise RuntimeError('signalling support is unavailable '
                                'because the blinker library is '
                                'not installed.')
-        send = lambda *a, **kw: None
+
+        send = lambda *a, **kw: None  # noqa
         connect = disconnect = has_receivers_for = receivers_for = \
             temporarily_connected_to = _fail
         del _fail

requirements.txt

@@ -1,2 +1,5 @@
-pymongo>=2.7.1
 nose
+pymongo>=2.7.1
+six==1.10.0
+flake8
+flake8-import-order

setup.cfg

@@ -1,11 +1,13 @@
 [nosetests]
-verbosity = 3
+verbosity = 2
 detailed-errors = 1
-#with-coverage = 1
+cover-erase = 1
-#cover-erase = 1
+cover-branches = 1
-#cover-html = 1
+cover-package = mongoengine
-#cover-html-dir = ../htmlcov
+tests = tests
-#cover-package = mongoengine
+
-py3where = build
+[flake8]
-where = tests
+ignore=E501,F401,F403,F405,I201
-#tests =  document/__init__.py
+exclude=build,dist,docs,venv,.tox,.eggs,tests
+max-complexity=45
+application-import-names=mongoengine,tests

setup.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 # Hack to silence atexit traceback in newer python versions
 try:
@@ -8,13 +8,16 @@ try:
 except ImportError:
     pass
 
-DESCRIPTION = 'MongoEngine is a Python Object-Document ' + \
+DESCRIPTION = (
-'Mapper for working with MongoDB.'
+    'MongoEngine is a Python Object-Document '
-LONG_DESCRIPTION = None
+    'Mapper for working with MongoDB.'
+)
+
 try:
-    LONG_DESCRIPTION = open('README.rst').read()
+    with open('README.rst') as fin:
-except:
+        LONG_DESCRIPTION = fin.read()
-    pass
+except Exception:
+    LONG_DESCRIPTION = None
 
 
 def get_version(version_tuple):
@@ -22,6 +25,7 @@ def get_version(version_tuple):
         return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1]
     return '.'.join(map(str, version_tuple))
 
+
 # Dirty hack to get version number from monogengine/__init__.py - we can't
 # import it as it depends on PyMongo and PyMongo isn't installed until this
 # file is read
@@ -29,7 +33,6 @@ init = os.path.join(os.path.dirname(__file__), 'mongoengine', '__init__.py')
 version_line = list(filter(lambda l: l.startswith('VERSION'), open(init)))[0]
 
 VERSION = get_version(eval(version_line.split('=')[-1]))
-print(VERSION)
 
 CLASSIFIERS = [
     'Development Status :: 4 - Beta',
@@ -53,31 +56,33 @@ CLASSIFIERS = [
 extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])}
 if sys.version_info[0] == 3:
     extra_opts['use_2to3'] = True
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'Pillow>=2.0.0', 'django>=1.5.1']
+    extra_opts['tests_require'] = ['nose', 'coverage==3.7.1', 'blinker', 'Pillow>=2.0.0']
     if "test" in sys.argv or "nosetests" in sys.argv:
         extra_opts['packages'] = find_packages()
         extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'Pillow>=2.0.0', 'jinja2>=2.6', 'python-dateutil']
+    # coverage 4 does not support Python 3.2 anymore
+    extra_opts['tests_require'] = ['nose', 'coverage==3.7.1', 'blinker', 'Pillow>=2.0.0', 'python-dateutil']
 
     if sys.version_info[0] == 2 and sys.version_info[1] == 6:
         extra_opts['tests_require'].append('unittest2')
 
-setup(name='mongoengine',
+setup(
-      version=VERSION,
+    name='mongoengine',
-      author='Harry Marr',
+    version=VERSION,
-      author_email='harry.marr@{nospam}gmail.com',
+    author='Harry Marr',
-      maintainer="Ross Lawley",
+    author_email='harry.marr@{nospam}gmail.com',
-      maintainer_email="ross.lawley@{nospam}gmail.com",
+    maintainer="Ross Lawley",
-      url='http://mongoengine.org/',
+    maintainer_email="ross.lawley@{nospam}gmail.com",
-      download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
+    url='http://mongoengine.org/',
-      license='MIT',
+    download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
-      include_package_data=True,
+    license='MIT',
-      description=DESCRIPTION,
+    include_package_data=True,
-      long_description=LONG_DESCRIPTION,
+    description=DESCRIPTION,
-      platforms=['any'],
+    long_description=LONG_DESCRIPTION,
-      classifiers=CLASSIFIERS,
+    platforms=['any'],
-      install_requires=['pymongo>=2.7.1'],
+    classifiers=CLASSIFIERS,
-      test_suite='nose.collector',
+    install_requires=['pymongo>=2.7.1', 'six'],
-      **extra_opts
+    test_suite='nose.collector',
+    **extra_opts
 )

tests/document/delta.py

@@ -93,6 +93,7 @@ class DeltaTest(unittest.TestCase):
     def delta_recursive(self, DocClass, EmbeddedClass):
 
         class Embedded(EmbeddedClass):
+            id = StringField()
             string_field = StringField()
             int_field = IntField()
             dict_field = DictField()
@@ -114,6 +115,7 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc._delta(), ({}, {}))
 
         embedded_1 = Embedded()
+        embedded_1.id = "010101"
         embedded_1.string_field = 'hello'
         embedded_1.int_field = 1
         embedded_1.dict_field = {'hello': 'world'}
@@ -123,6 +125,7 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc._get_changed_fields(), ['embedded_field'])
 
         embedded_delta = {
+            'id': "010101",
             'string_field': 'hello',
             'int_field': 1,
             'dict_field': {'hello': 'world'},
@@ -250,13 +253,13 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc.embedded_field.list_field[2].list_field,
                          [1, 2, {'hello': 'world'}])
 
-        del(doc.embedded_field.list_field[2].list_field[2]['hello'])
+        del doc.embedded_field.list_field[2].list_field[2]['hello']
         self.assertEqual(doc._delta(),
                          ({}, {'embedded_field.list_field.2.list_field.2.hello': 1}))
         doc.save()
         doc = doc.reload(10)
 
-        del(doc.embedded_field.list_field[2].list_field)
+        del doc.embedded_field.list_field[2].list_field
         self.assertEqual(doc._delta(),
                          ({}, {'embedded_field.list_field.2.list_field': 1}))
 
@@ -590,13 +593,13 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc.embedded_field.list_field[2].list_field,
             [1, 2, {'hello': 'world'}])
 
-        del(doc.embedded_field.list_field[2].list_field[2]['hello'])
+        del doc.embedded_field.list_field[2].list_field[2]['hello']
         self.assertEqual(doc._delta(),
             ({}, {'db_embedded_field.db_list_field.2.db_list_field.2.hello': 1}))
         doc.save()
         doc = doc.reload(10)
 
-        del(doc.embedded_field.list_field[2].list_field)
+        del doc.embedded_field.list_field[2].list_field
         self.assertEqual(doc._delta(), ({},
             {'db_embedded_field.db_list_field.2.db_list_field': 1}))
 
@@ -612,7 +615,7 @@ class DeltaTest(unittest.TestCase):
             SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {}))
 
         p.doc = 123
-        del(p.doc)
+        del p.doc
         self.assertEqual(p._delta(), (
             SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {}))
 
@@ -732,6 +735,56 @@ class DeltaTest(unittest.TestCase):
         mydoc._clear_changed_fields()
         self.assertEqual([], mydoc._get_changed_fields())
 
+    def test_lower_level_mark_as_changed(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            name = StringField()
+
+        class MyDoc(Document):
+            subs = MapField(EmbeddedDocumentField(EmbeddedDoc))
+
+        MyDoc.drop_collection()
+
+        MyDoc().save()
+
+        mydoc = MyDoc.objects.first()
+        mydoc.subs['a'] = EmbeddedDoc()
+        self.assertEqual(["subs.a"], mydoc._get_changed_fields())
+
+        subdoc = mydoc.subs['a']
+        subdoc.name = 'bar'
+
+        self.assertEqual(["name"], subdoc._get_changed_fields())
+        self.assertEqual(["subs.a"], mydoc._get_changed_fields())
+        mydoc.save()
+
+        mydoc._clear_changed_fields()
+        self.assertEqual([], mydoc._get_changed_fields())
+
+    def test_upper_level_mark_as_changed(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            name = StringField()
+
+        class MyDoc(Document):
+            subs = MapField(EmbeddedDocumentField(EmbeddedDoc))
+
+        MyDoc.drop_collection()
+
+        MyDoc(subs={'a': EmbeddedDoc(name='foo')}).save()
+
+        mydoc = MyDoc.objects.first()
+        subdoc = mydoc.subs['a']
+        subdoc.name = 'bar'
+
+        self.assertEqual(["name"], subdoc._get_changed_fields())
+        self.assertEqual(["subs.a.name"], mydoc._get_changed_fields())
+
+        mydoc.subs['a'] = EmbeddedDoc()
+        self.assertEqual(["subs.a"], mydoc._get_changed_fields())
+        mydoc.save()
+
+        mydoc._clear_changed_fields()
+        self.assertEqual([], mydoc._get_changed_fields())
+
     def test_referenced_object_changed_attributes(self):
         """Ensures that when you save a new reference to a field, the referenced object isn't altered"""
 
@@ -774,5 +827,43 @@ class DeltaTest(unittest.TestCase):
         org2.reload()
         self.assertEqual(org2.name, 'New Org 2')
 
+    def test_delta_for_nested_map_fields(self):
+        class UInfoDocument(Document):
+            phone = StringField()
+
+        class EmbeddedRole(EmbeddedDocument):
+            type = StringField()
+
+        class EmbeddedUser(EmbeddedDocument):
+            name = StringField()
+            roles = MapField(field=EmbeddedDocumentField(EmbeddedRole))
+            rolist = ListField(field=EmbeddedDocumentField(EmbeddedRole))
+            info = ReferenceField(UInfoDocument)
+
+        class Doc(Document):
+            users = MapField(field=EmbeddedDocumentField(EmbeddedUser))
+            num = IntField(default=-1)
+
+        Doc.drop_collection()
+
+        doc = Doc(num=1)
+        doc.users["007"] = EmbeddedUser(name="Agent007")
+        doc.save()
+
+        uinfo = UInfoDocument(phone="79089269066")
+        uinfo.save()
+
+        d = Doc.objects(num=1).first()
+        d.users["007"]["roles"]["666"] = EmbeddedRole(type="superadmin")
+        d.users["007"]["rolist"].append(EmbeddedRole(type="oops"))
+        d.users["007"]["info"] = uinfo
+        delta = d._delta()
+        self.assertEqual(True, "users.007.roles.666" in delta[0])
+        self.assertEqual(True, "users.007.rolist" in delta[0])
+        self.assertEqual(True, "users.007.info" in delta[0])
+        self.assertEqual('superadmin', delta[0]["users.007.roles.666"]["type"])
+        self.assertEqual('oops', delta[0]["users.007.rolist"][0]["type"])
+        self.assertEqual(uinfo.id, delta[0]["users.007.info"])
+
 if __name__ == '__main__':
     unittest.main()

tests/document/dynamic.py

@@ -72,7 +72,7 @@ class DynamicTest(unittest.TestCase):
         obj = collection.find_one()
         self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'misc', 'name'])
 
-        del(p.misc)
+        del p.misc
         p.save()
 
         p = self.Person.objects.get()
@@ -88,6 +88,18 @@ class DynamicTest(unittest.TestCase):
         p.update(unset__misc=1)
         p.reload()
 
+    def test_reload_dynamic_field(self):
+        self.Person.objects.delete()
+        p = self.Person.objects.create()
+        p.update(age=1)
+
+        self.assertEqual(len(p._data), 3)
+        self.assertEqual(sorted(p._data.keys()), ['_cls', 'id', 'name'])
+
+        p.reload()
+        self.assertEqual(len(p._data), 4)
+        self.assertEqual(sorted(p._data.keys()), ['_cls', 'age', 'id', 'name'])
+
     def test_dynamic_document_queries(self):
         """Ensure we can query dynamic fields"""
         p = self.Person()
@@ -129,6 +141,15 @@ class DynamicTest(unittest.TestCase):
 
         self.assertEqual(1, self.Person.objects(misc__hello='world').count())
 
+    def test_three_level_complex_data_lookups(self):
+        """Ensure you can query three level document dynamic fields"""
+        p = self.Person()
+        p.misc = {'hello': {'hello2': 'world'}}
+        p.save()
+        # from pprint import pprint as pp; import pdb; pdb.set_trace();
+        print self.Person.objects(misc__hello__hello2='world')
+        self.assertEqual(1, self.Person.objects(misc__hello__hello2='world').count())
+
     def test_complex_embedded_document_validation(self):
         """Ensure embedded dynamic documents may be validated"""
         class Embedded(DynamicEmbeddedDocument):
@@ -331,7 +352,7 @@ class DynamicTest(unittest.TestCase):
         person = Person.objects.first()
         person.attrval = "This works"
 
-        person["phone"] = "555-1212" # but this should too
+        person["phone"] = "555-1212"  # but this should too
 
         # Same thing two levels deep
         person["address"]["city"] = "Lundenne"
@@ -347,7 +368,6 @@ class DynamicTest(unittest.TestCase):
 
         self.assertEqual(Person.objects.first().address.city, "Londinium")
 
-
         person = Person.objects.first()
         person["age"] = 35
         person.save()

tests/document/indexes.py

@@ -1,9 +1,8 @@
 # -*- coding: utf-8 -*-
 import unittest
 import sys
-sys.path[0:0] = [""]
 
-import os
+
 import pymongo
 
 from nose.plugins.skip import SkipTest
@@ -32,10 +31,7 @@ class IndexesTest(unittest.TestCase):
         self.Person = Person
 
     def tearDown(self):
-        for collection in self.db.collection_names():
+        self.connection.drop_database(self.db)
-            if 'system.' in collection:
-                continue
-            self.db.drop_collection(collection)
 
     def test_indexes_document(self):
         """Ensure that indexes are used when meta[indexes] is specified for
@@ -143,7 +139,7 @@ class IndexesTest(unittest.TestCase):
             meta = {
                 'indexes': [
                         {
-                        'fields': ('title',),
+                            'fields': ('title',),
                         },
                 ],
                 'allow_inheritance': True,
@@ -275,6 +271,60 @@ class IndexesTest(unittest.TestCase):
         info = [value['key'] for key, value in info.iteritems()]
         self.assertTrue([('current.location.point', '2d')] in info)
 
+    def test_explicit_geosphere_index(self):
+        """Ensure that geosphere indexes work when created via meta[indexes]
+        """
+        class Place(Document):
+            location = DictField()
+            meta = {
+                'allow_inheritance': True,
+                'indexes': [
+                    '(location.point',
+                ]
+            }
+
+        self.assertEqual([{'fields': [('location.point', '2dsphere')]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', '2dsphere')] in info)
+
+    def test_explicit_geohaystack_index(self):
+        """Ensure that geohaystack indexes work when created via meta[indexes]
+        """
+        raise SkipTest('GeoHaystack index creation is not supported for now'
+                       'from meta, as it requires a bucketSize parameter.')
+
+        class Place(Document):
+            location = DictField()
+            name = StringField()
+            meta = {
+                'indexes': [
+                    (')location.point', 'name')
+                ]
+            }
+        self.assertEqual([{'fields': [('location.point', 'geoHaystack'), ('name', 1)]}],
+                         Place._meta['index_specs'])
+
+        Place.ensure_indexes()
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', 'geoHaystack')] in info)
+
+    def test_create_geohaystack_index(self):
+        """Ensure that geohaystack indexes can be created
+        """
+        class Place(Document):
+            location = DictField()
+            name = StringField()
+
+        Place.create_index({'fields': (')location.point', 'name')}, bucketSize=10)
+        info = Place._get_collection().index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('location.point', 'geoHaystack'), ('name', 1)] in info)
+
     def test_dictionary_indexes(self):
         """Ensure that indexes are used when meta[indexes] contains
         dictionaries instead of lists.
@@ -432,6 +482,7 @@ class IndexesTest(unittest.TestCase):
 
         class Test(Document):
             a = IntField()
+            b = IntField()
 
             meta = {
                 'indexes': ['a'],
@@ -443,16 +494,36 @@ class IndexesTest(unittest.TestCase):
         obj = Test(a=1)
         obj.save()
 
+        connection = get_connection()
+        IS_MONGODB_3 = connection.server_info()['versionArray'][0] >= 3
+
         # Need to be explicit about covered indexes as mongoDB doesn't know if
         # the documents returned might have more keys in that here.
         query_plan = Test.objects(id=obj.id).exclude('a').explain()
-        self.assertFalse(query_plan['indexOnly'])
+        if not IS_MONGODB_3:
+            self.assertFalse(query_plan['indexOnly'])
+        else:
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('inputStage').get('stage'), 'IDHACK')
 
         query_plan = Test.objects(id=obj.id).only('id').explain()
-        self.assertTrue(query_plan['indexOnly'])
+        if not IS_MONGODB_3:
+            self.assertTrue(query_plan['indexOnly'])
+        else:
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('inputStage').get('stage'), 'IDHACK')
 
         query_plan = Test.objects(a=1).only('a').exclude('id').explain()
-        self.assertTrue(query_plan['indexOnly'])
+        if not IS_MONGODB_3:
+            self.assertTrue(query_plan['indexOnly'])
+        else:
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('inputStage').get('stage'), 'IXSCAN')
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('stage'), 'PROJECTION')
+
+        query_plan = Test.objects(a=1).explain()
+        if not IS_MONGODB_3:
+            self.assertFalse(query_plan['indexOnly'])
+        else:
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('inputStage').get('stage'), 'IXSCAN')
+            self.assertEqual(query_plan.get('queryPlanner').get('winningPlan').get('stage'), 'FETCH')
 
     def test_index_on_id(self):
 
@@ -491,19 +562,22 @@ class IndexesTest(unittest.TestCase):
 
         self.assertEqual(BlogPost.objects.count(), 10)
         self.assertEqual(BlogPost.objects.hint().count(), 10)
-        self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10)
 
-        self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10)
+        # PyMongo 3.0 bug only, works correctly with 2.X and 3.0.1+ versions
+        if pymongo.version != '3.0':
+            self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10)
+
+            self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10)
 
         if pymongo.version >= '2.8':
             self.assertEqual(BlogPost.objects.hint('tags').count(), 10)
         else:
             def invalid_index():
-                BlogPost.objects.hint('tags')
+                BlogPost.objects.hint('tags').next()
             self.assertRaises(TypeError, invalid_index)
 
         def invalid_index_2():
-            return BlogPost.objects.hint(('tags', 1))
+            return BlogPost.objects.hint(('tags', 1)).next()
         self.assertRaises(Exception, invalid_index_2)
 
     def test_unique(self):
@@ -744,33 +818,34 @@ class IndexesTest(unittest.TestCase):
             name = StringField(required=True)
             term = StringField(required=True)
 
-        class Report(Document):
+        class ReportEmbedded(Document):
             key = EmbeddedDocumentField(CompoundKey, primary_key=True)
             text = StringField()
 
-        Report.drop_collection()
-
         my_key = CompoundKey(name="n", term="ok")
-        report = Report(text="OK", key=my_key).save()
+        report = ReportEmbedded(text="OK", key=my_key).save()
 
         self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}},
                          report.to_mongo())
-        self.assertEqual(report, Report.objects.get(pk=my_key))
+        self.assertEqual(report, ReportEmbedded.objects.get(pk=my_key))
 
     def test_compound_key_dictfield(self):
 
-        class Report(Document):
+        class ReportDictField(Document):
             key = DictField(primary_key=True)
             text = StringField()
 
-        Report.drop_collection()
-
         my_key = {"name": "n", "term": "ok"}
-        report = Report(text="OK", key=my_key).save()
+        report = ReportDictField(text="OK", key=my_key).save()
 
         self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}},
                          report.to_mongo())
-        self.assertEqual(report, Report.objects.get(pk=my_key))
+
+        # We can't directly call ReportDictField.objects.get(pk=my_key),
+        # because dicts are unordered, and if the order in MongoDB is
+        # different than the one in `my_key`, this test will fail.
+        self.assertEqual(report, ReportDictField.objects.get(pk__name=my_key['name']))
+        self.assertEqual(report, ReportDictField.objects.get(pk__term=my_key['term']))
 
     def test_string_indexes(self):
 
@@ -785,6 +860,20 @@ class IndexesTest(unittest.TestCase):
         self.assertTrue([('provider_ids.foo', 1)] in info)
         self.assertTrue([('provider_ids.bar', 1)] in info)
 
+    def test_sparse_compound_indexes(self):
+
+        class MyDoc(Document):
+            provider_ids = DictField()
+            meta = {
+                "indexes": [{'fields': ("provider_ids.foo", "provider_ids.bar"),
+                             'sparse': True}],
+            }
+
+        info = MyDoc.objects._collection.index_information()
+        self.assertEqual([('provider_ids.foo', 1), ('provider_ids.bar', 1)],
+                         info['provider_ids.foo_1_provider_ids.bar_1']['key'])
+        self.assertTrue(info['provider_ids.foo_1_provider_ids.bar_1']['sparse'])
+
     def test_text_indexes(self):
 
         class Book(Document):
@@ -798,6 +887,18 @@ class IndexesTest(unittest.TestCase):
         key = indexes["title_text"]["key"]
         self.assertTrue(('_fts', 'text') in key)
 
+    def test_hashed_indexes(self):
+
+        class Book(Document):
+            ref_id = StringField()
+            meta = {
+                "indexes": ["#ref_id"],
+            }
+
+        indexes = Book.objects._collection.index_information()
+        self.assertTrue("ref_id_hashed" in indexes)
+        self.assertTrue(('ref_id', 'hashed') in indexes["ref_id_hashed"]["key"])
+
     def test_indexes_after_database_drop(self):
         """
         Test to ensure that indexes are re-created on a collection even
@@ -805,26 +906,122 @@ class IndexesTest(unittest.TestCase):
 
         Issue #812
         """
+        # Use a new connection and database since dropping the database could
+        # cause concurrent tests to fail.
+        connection = connect(db='tempdatabase',
+                             alias='test_indexes_after_database_drop')
+
         class BlogPost(Document):
             title = StringField()
             slug = StringField(unique=True)
 
-        BlogPost.drop_collection()
+            meta = {'db_alias': 'test_indexes_after_database_drop'}
 
-        # Create Post #1
+        try:
-        post1 = BlogPost(title='test1', slug='test')
+            BlogPost.drop_collection()
-        post1.save()
 
-        # Drop the Database
+            # Create Post #1
-        self.connection.drop_database(BlogPost._get_db().name)
+            post1 = BlogPost(title='test1', slug='test')
+            post1.save()
 
-        # Re-create Post #1
+            # Drop the Database
-        post1 = BlogPost(title='test1', slug='test')
+            connection.drop_database('tempdatabase')
-        post1.save()
+
+            # Re-create Post #1
+            post1 = BlogPost(title='test1', slug='test')
+            post1.save()
+
+            # Create Post #2
+            post2 = BlogPost(title='test2', slug='test')
+            self.assertRaises(NotUniqueError, post2.save)
+        finally:
+            # Drop the temporary database at the end
+            connection.drop_database('tempdatabase')
+
+
+    def test_index_dont_send_cls_option(self):
+        """
+        Ensure that 'cls' option is not sent through ensureIndex. We shouldn't
+        send internal MongoEngine arguments that are not a part of the index
+        spec.
+
+        This is directly related to the fact that MongoDB doesn't validate the
+        options that are passed to ensureIndex. For more details, see:
+        https://jira.mongodb.org/browse/SERVER-769
+        """
+        class TestDoc(Document):
+            txt = StringField()
+
+            meta = {
+                'allow_inheritance': True,
+                'indexes': [
+                    {'fields': ('txt',), 'cls': False}
+                ]
+            }
+
+        class TestChildDoc(TestDoc):
+            txt2 = StringField()
+
+            meta = {
+                'indexes': [
+                    {'fields': ('txt2',), 'cls': False}
+                ]
+            }
+
+        TestDoc.drop_collection()
+        TestDoc.ensure_indexes()
+        TestChildDoc.ensure_indexes()
+
+        index_info = TestDoc._get_collection().index_information()
+        for key in index_info:
+            del index_info[key]['v']  # drop the index version - we don't care about that here
+            if 'ns' in index_info[key]:
+                del index_info[key]['ns']  # drop the index namespace - we don't care about that here, MongoDB 3+
+            if 'dropDups' in index_info[key]:
+                del index_info[key]['dropDups']  # drop the index dropDups - it is deprecated in MongoDB 3+
+
+        self.assertEqual(index_info, {
+            'txt_1': {
+                'key': [('txt', 1)],
+                'background': False
+            },
+            '_id_': {
+                'key': [('_id', 1)],
+            },
+            'txt2_1': {
+                'key': [('txt2', 1)],
+                'background': False
+            },
+            '_cls_1': {
+                'key': [('_cls', 1)],
+                'background': False,
+            }
+        })
+
+    def test_compound_index_underscore_cls_not_overwritten(self):
+        """
+        Test that the compound index doesn't get another _cls when it is specified
+        """
+        class TestDoc(Document):
+            shard_1 = StringField()
+            txt_1 = StringField()
+
+            meta = {
+                'collection': 'test',
+                'allow_inheritance': True,
+                'sparse': True,
+                'shard_key': 'shard_1',
+                'indexes': [
+                    ('shard_1', '_cls', 'txt_1'),
+                ]
+            }
+
+        TestDoc.drop_collection()
+        TestDoc.ensure_indexes()
+
+        index_info = TestDoc._get_collection().index_information()
+        self.assertTrue('shard_1_1__cls_1_txt_1_1' in index_info)
 
-        # Create Post #2
-        post2 = BlogPost(title='test2', slug='test')
-        self.assertRaises(NotUniqueError, post2.save)
 
 if __name__ == '__main__':
     unittest.main()

tests/document/inheritance.py

@@ -307,6 +307,69 @@ class InheritanceTest(unittest.TestCase):
         doc = Animal(name='dog')
         self.assertFalse('_cls' in doc.to_mongo())
 
+    def test_abstract_handle_ids_in_metaclass_properly(self):
+
+        class City(Document):
+            continent = StringField()
+            meta = {'abstract': True,
+                    'allow_inheritance': False}
+
+        class EuropeanCity(City):
+            name = StringField()
+
+        berlin = EuropeanCity(name='Berlin', continent='Europe')
+        self.assertEqual(len(berlin._db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._reverse_db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._fields_ordered), 3)
+        self.assertEqual(berlin._fields_ordered[0], 'id')
+
+    def test_auto_id_not_set_if_specific_in_parent_class(self):
+
+        class City(Document):
+            continent = StringField()
+            city_id = IntField(primary_key=True)
+            meta = {'abstract': True,
+                    'allow_inheritance': False}
+
+        class EuropeanCity(City):
+            name = StringField()
+
+        berlin = EuropeanCity(name='Berlin', continent='Europe')
+        self.assertEqual(len(berlin._db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._reverse_db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._fields_ordered), 3)
+        self.assertEqual(berlin._fields_ordered[0], 'city_id')
+
+    def test_auto_id_vs_non_pk_id_field(self):
+
+        class City(Document):
+            continent = StringField()
+            id = IntField()
+            meta = {'abstract': True,
+                    'allow_inheritance': False}
+
+        class EuropeanCity(City):
+            name = StringField()
+
+        berlin = EuropeanCity(name='Berlin', continent='Europe')
+        self.assertEqual(len(berlin._db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._reverse_db_field_map), len(berlin._fields_ordered))
+        self.assertEqual(len(berlin._fields_ordered), 4)
+        self.assertEqual(berlin._fields_ordered[0], 'auto_id_0')
+        berlin.save()
+        self.assertEqual(berlin.pk, berlin.auto_id_0)
+
+    def test_abstract_document_creation_does_not_fail(self):
+
+        class City(Document):
+            continent = StringField()
+            meta = {'abstract': True,
+                    'allow_inheritance': False}
+        bkk = City(continent='asia')
+        self.assertEqual(None, bkk.pk)
+        # TODO: expected error? Shouldn't we create a new error type?
+        self.assertRaises(KeyError, lambda: setattr(bkk, 'pk', 1))
+
     def test_allow_inheritance_embedded_document(self):
         """Ensure embedded documents respect inheritance
         """
@@ -348,7 +411,7 @@ class InheritanceTest(unittest.TestCase):
         try:
             class MyDocument(DateCreatedDocument, DateUpdatedDocument):
                 pass
-        except:
+        except Exception:
             self.assertTrue(False, "Couldn't create MyDocument class")
 
     def test_abstract_documents(self):

tests/document/instance.py

@@ -7,15 +7,18 @@ import os
 import pickle
 import unittest
 import uuid
+import weakref
 
 from datetime import datetime
 from bson import DBRef, ObjectId
+from tests import fixtures
 from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest,
-                            PickleDyanmicEmbedded, PickleDynamicTest)
+                            PickleDynamicEmbedded, PickleDynamicTest)
 
 from mongoengine import *
 from mongoengine.errors import (NotRegistered, InvalidDocumentError,
-                                InvalidQueryError, NotUniqueError)
+                                InvalidQueryError, NotUniqueError,
+                                FieldDoesNotExist, SaveConditionError)
 from mongoengine.queryset import NULLIFY, Q
 from mongoengine.connection import get_db
 from mongoengine.base import get_document
@@ -28,6 +31,8 @@ TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
 __all__ = ("InstanceTest",)
 
 
+
+
 class InstanceTest(unittest.TestCase):
 
     def setUp(self):
@@ -61,6 +66,14 @@ class InstanceTest(unittest.TestCase):
             list(self.Person._get_collection().find().sort("id")),
             sorted(docs, key=lambda doc: doc["_id"]))
 
+    def assertHasInstance(self, field, instance):
+        self.assertTrue(hasattr(field, "_instance"))
+        self.assertTrue(field._instance is not None)
+        if isinstance(field._instance, weakref.ProxyType):
+            self.assertTrue(field._instance.__eq__(instance))
+        else:
+            self.assertEqual(field._instance, instance)
+
     def test_capped_collection(self):
         """Ensure that capped collections work properly.
         """
@@ -86,7 +99,7 @@ class InstanceTest(unittest.TestCase):
         options = Log.objects._collection.options()
         self.assertEqual(options['capped'], True)
         self.assertEqual(options['max'], 10)
-        self.assertTrue(options['size'] >= 4096)
+        self.assertEqual(options['size'], 4096)
 
         # Check that the document cannot be redefined with different options
         def recreate_log_document():
@@ -101,6 +114,69 @@ class InstanceTest(unittest.TestCase):
 
         Log.drop_collection()
 
+    def test_capped_collection_default(self):
+        """Ensure that capped collections defaults work properly.
+        """
+        class Log(Document):
+            date = DateTimeField(default=datetime.now)
+            meta = {
+                'max_documents': 10,
+            }
+
+        Log.drop_collection()
+
+        # Create a doc to create the collection
+        Log().save()
+
+        options = Log.objects._collection.options()
+        self.assertEqual(options['capped'], True)
+        self.assertEqual(options['max'], 10)
+        self.assertEqual(options['size'], 10 * 2**20)
+
+        # Check that the document with default value can be recreated
+        def recreate_log_document():
+            class Log(Document):
+                date = DateTimeField(default=datetime.now)
+                meta = {
+                    'max_documents': 10,
+                }
+            # Create the collection by accessing Document.objects
+            Log.objects
+        recreate_log_document()
+        Log.drop_collection()
+
+    def test_capped_collection_no_max_size_problems(self):
+        """Ensure that capped collections with odd max_size work properly.
+        MongoDB rounds up max_size to next multiple of 256, recreating a doc
+        with the same spec failed in mongoengine <0.10
+        """
+        class Log(Document):
+            date = DateTimeField(default=datetime.now)
+            meta = {
+                'max_size': 10000,
+            }
+
+        Log.drop_collection()
+
+        # Create a doc to create the collection
+        Log().save()
+
+        options = Log.objects._collection.options()
+        self.assertEqual(options['capped'], True)
+        self.assertTrue(options['size'] >= 10000)
+
+        # Check that the document with odd max_size value can be recreated
+        def recreate_log_document():
+            class Log(Document):
+                date = DateTimeField(default=datetime.now)
+                meta = {
+                    'max_size': 10000,
+                }
+            # Create the collection by accessing Document.objects
+            Log.objects
+        recreate_log_document()
+        Log.drop_collection()
+
     def test_repr(self):
         """Ensure that unicode representation works
         """
@@ -408,6 +484,20 @@ class InstanceTest(unittest.TestCase):
         doc.reload()
         Animal.drop_collection()
 
+    def test_reload_sharded_nested(self):
+        class SuperPhylum(EmbeddedDocument):
+            name = StringField()
+
+        class Animal(Document):
+            superphylum = EmbeddedDocumentField(SuperPhylum)
+            meta = {'shard_key': ('superphylum.name',)}
+
+        Animal.drop_collection()
+        doc = Animal(superphylum=SuperPhylum(name='Deuterostomia'))
+        doc.save()
+        doc.reload()
+        Animal.drop_collection()
+
     def test_reload_referencing(self):
         """Ensures reloading updates weakrefs correctly
         """
@@ -481,6 +571,28 @@ class InstanceTest(unittest.TestCase):
         except Exception:
             self.assertFalse("Threw wrong exception")
 
+    def test_reload_of_non_strict_with_special_field_name(self):
+        """Ensures reloading works for documents with meta strict == False
+        """
+        class Post(Document):
+            meta = {
+                'strict': False
+            }
+            title = StringField()
+            items = ListField()
+
+        Post.drop_collection()
+
+        Post._get_collection().insert({
+            "title": "Items eclipse",
+            "items": ["more lorem", "even more ipsum"]
+        })
+
+        post = Post.objects.first()
+        post.reload()
+        self.assertEqual(post.title, "Items eclipse")
+        self.assertEqual(post.items, ["more lorem", "even more ipsum"])
+
     def test_dictionary_access(self):
         """Ensure that dictionary-style field access works properly.
         """
@@ -543,10 +655,12 @@ class InstanceTest(unittest.TestCase):
             embedded_field = EmbeddedDocumentField(Embedded)
 
         Doc.drop_collection()
-        Doc(embedded_field=Embedded(string="Hi")).save()
+        doc = Doc(embedded_field=Embedded(string="Hi"))
+        self.assertHasInstance(doc.embedded_field, doc)
 
+        doc.save()
         doc = Doc.objects.get()
-        self.assertEqual(doc, doc.embedded_field._instance)
+        self.assertHasInstance(doc.embedded_field, doc)
 
     def test_embedded_document_complex_instance(self):
         """Ensure that embedded documents in complex fields can reference
@@ -558,10 +672,25 @@ class InstanceTest(unittest.TestCase):
             embedded_field = ListField(EmbeddedDocumentField(Embedded))
 
         Doc.drop_collection()
-        Doc(embedded_field=[Embedded(string="Hi")]).save()
+        doc = Doc(embedded_field=[Embedded(string="Hi")])
+        self.assertHasInstance(doc.embedded_field[0], doc)
 
+        doc.save()
         doc = Doc.objects.get()
-        self.assertEqual(doc, doc.embedded_field[0]._instance)
+        self.assertHasInstance(doc.embedded_field[0], doc)
+
+    def test_embedded_document_complex_instance_no_use_db_field(self):
+        """Ensure that use_db_field is propagated to list of Emb Docs
+        """
+        class Embedded(EmbeddedDocument):
+            string = StringField(db_field='s')
+
+        class Doc(Document):
+            embedded_field = ListField(EmbeddedDocumentField(Embedded))
+
+        d = Doc(embedded_field=[Embedded(string="Hi")]).to_mongo(
+            use_db_field=False).to_dict()
+        self.assertEqual(d['embedded_field'], [{'string': 'Hi'}])
 
     def test_instance_is_set_on_setattr(self):
 
@@ -574,11 +703,28 @@ class InstanceTest(unittest.TestCase):
         Account.drop_collection()
         acc = Account()
         acc.email = Email(email='test@example.com')
-        self.assertTrue(hasattr(acc._data["email"], "_instance"))
+        self.assertHasInstance(acc._data["email"], acc)
         acc.save()
 
         acc1 = Account.objects.first()
-        self.assertTrue(hasattr(acc1._data["email"], "_instance"))
+        self.assertHasInstance(acc1._data["email"], acc1)
+
+    def test_instance_is_set_on_setattr_on_embedded_document_list(self):
+
+        class Email(EmbeddedDocument):
+            email = EmailField()
+
+        class Account(Document):
+            emails = EmbeddedDocumentListField(Email)
+
+        Account.drop_collection()
+        acc = Account()
+        acc.emails = [Email(email='test@example.com')]
+        self.assertHasInstance(acc._data["emails"][0], acc)
+        acc.save()
+
+        acc1 = Account.objects.first()
+        self.assertHasInstance(acc1._data["emails"][0], acc1)
 
     def test_document_clean(self):
         class TestDocument(Document):
@@ -952,11 +1098,12 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(w1.save_id, UUID(1))
         self.assertEqual(w1.count, 0)
 
-        # mismatch in save_condition prevents save
+        # mismatch in save_condition prevents save and raise exception
         flip(w1)
         self.assertTrue(w1.toggle)
         self.assertEqual(w1.count, 1)
-        w1.save(save_condition={'save_id': UUID(42)})
+        self.assertRaises(SaveConditionError,
+            w1.save, save_condition={'save_id': UUID(42)})
         w1.reload()
         self.assertFalse(w1.toggle)
         self.assertEqual(w1.count, 0)
@@ -984,7 +1131,8 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(w1.count, 2)
         flip(w2)
         flip(w2)
-        w2.save(save_condition={'save_id': old_id})
+        self.assertRaises(SaveConditionError,
+            w2.save, save_condition={'save_id': old_id})
         w2.reload()
         self.assertFalse(w2.toggle)
         self.assertEqual(w2.count, 2)
@@ -996,7 +1144,8 @@ class InstanceTest(unittest.TestCase):
         self.assertTrue(w1.toggle)
         self.assertEqual(w1.count, 3)
         flip(w1)
-        w1.save(save_condition={'count__gte': w1.count})
+        self.assertRaises(SaveConditionError,
+            w1.save, save_condition={'count__gte': w1.count})
         w1.reload()
         self.assertTrue(w1.toggle)
         self.assertEqual(w1.count, 3)
@@ -1757,6 +1906,62 @@ class InstanceTest(unittest.TestCase):
         author.delete()
         self.assertEqual(BlogPost.objects.count(), 0)
 
+    def test_reverse_delete_rule_with_custom_id_field(self):
+        """Ensure that a referenced document with custom primary key
+        is also deleted upon deletion.
+        """
+        class User(Document):
+            name = StringField(primary_key=True)
+
+        class Book(Document):
+            author = ReferenceField(User, reverse_delete_rule=CASCADE)
+            reviewer = ReferenceField(User, reverse_delete_rule=NULLIFY)
+
+        User.drop_collection()
+        Book.drop_collection()
+
+        user = User(name='Mike').save()
+        reviewer = User(name='John').save()
+        book = Book(author=user, reviewer=reviewer).save()
+
+        reviewer.delete()
+        self.assertEqual(Book.objects.count(), 1)
+        self.assertEqual(Book.objects.get().reviewer, None)
+
+        user.delete()
+        self.assertEqual(Book.objects.count(), 0)
+
+    def test_reverse_delete_rule_with_shared_id_among_collections(self):
+        """Ensure that cascade delete rule doesn't mix id among collections.
+        """
+        class User(Document):
+            id = IntField(primary_key=True)
+
+        class Book(Document):
+            id = IntField(primary_key=True)
+            author = ReferenceField(User, reverse_delete_rule=CASCADE)
+
+        User.drop_collection()
+        Book.drop_collection()
+
+        user_1 = User(id=1).save()
+        user_2 = User(id=2).save()
+        book_1 = Book(id=1, author=user_2).save()
+        book_2 = Book(id=2, author=user_1).save()
+
+        user_2.delete()
+        # Deleting user_2 should also delete book_1 but not book_2
+        self.assertEqual(Book.objects.count(), 1)
+        self.assertEqual(Book.objects.get(), book_2)
+
+        user_3 = User(id=3).save()
+        book_3 = Book(id=3, author=user_3).save()
+
+        user_3.delete()
+        # Deleting user_3 should also delete book_3
+        self.assertEqual(Book.objects.count(), 1)
+        self.assertEqual(Book.objects.get(), book_2)
+
     def test_reverse_delete_rule_with_document_inheritance(self):
         """Ensure that a referenced document is also deleted upon deletion
         of a child document.
@@ -1829,11 +2034,11 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(BlogPost.objects.count(), 0)
 
     def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self):
-        ''' ensure the pre_delete signal is triggered upon a cascading deletion
+        """ ensure the pre_delete signal is triggered upon a cascading deletion
         setup a blog post with content, an author and editor
         delete the author which triggers deletion of blogpost via cascade
         blog post's pre_delete signal alters an editor attribute
-        '''
+        """
         class Editor(self.Person):
             review_queue = IntField(default=0)
 
@@ -2085,11 +2290,34 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(pickle_doc.string, "Two")
         self.assertEqual(pickle_doc.lists, ["1", "2", "3"])
 
+    def test_regular_document_pickle(self):
+
+        pickle_doc = PickleTest(number=1, string="One", lists=['1', '2'])
+        pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
+        pickle_doc.save()
+
+        pickled_doc = pickle.dumps(pickle_doc)
+
+        # Test that when a document's definition changes the new
+        # definition is used
+        fixtures.PickleTest = fixtures.NewDocumentPickleTest
+
+        resurrected = pickle.loads(pickled_doc)
+        self.assertEqual(resurrected.__class__,
+                         fixtures.NewDocumentPickleTest)
+        self.assertEqual(resurrected._fields_ordered,
+                         fixtures.NewDocumentPickleTest._fields_ordered)
+        self.assertNotEqual(resurrected._fields_ordered,
+                            pickle_doc._fields_ordered)
+
+        # The local PickleTest is still a ref to the original
+        fixtures.PickleTest = PickleTest
+
     def test_dynamic_document_pickle(self):
 
         pickle_doc = PickleDynamicTest(
             name="test", number=1, string="One", lists=['1', '2'])
-        pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar")
+        pickle_doc.embedded = PickleDynamicEmbedded(foo="Bar")
         pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
 
         pickle_doc.save()
@@ -2443,6 +2671,114 @@ class InstanceTest(unittest.TestCase):
         group = Group.objects.first()
         self.assertEqual("hello - default", group.name)
 
+    def test_load_undefined_fields(self):
+        class User(Document):
+            name = StringField()
+
+        User.drop_collection()
+
+        User._get_collection().save({
+            'name': 'John',
+            'foo': 'Bar',
+            'data': [1, 2, 3]
+        })
+
+        self.assertRaises(FieldDoesNotExist, User.objects.first)
+
+    def test_load_undefined_fields_with_strict_false(self):
+        class User(Document):
+            name = StringField()
+
+            meta = {'strict': False}
+
+        User.drop_collection()
+
+        User._get_collection().save({
+            'name': 'John',
+            'foo': 'Bar',
+            'data': [1, 2, 3]
+        })
+
+        user = User.objects.first()
+        self.assertEqual(user.name, 'John')
+        self.assertFalse(hasattr(user, 'foo'))
+        self.assertEqual(user._data['foo'], 'Bar')
+        self.assertFalse(hasattr(user, 'data'))
+        self.assertEqual(user._data['data'], [1, 2, 3])
+
+    def test_load_undefined_fields_on_embedded_document(self):
+        class Thing(EmbeddedDocument):
+            name = StringField()
+
+        class User(Document):
+            name = StringField()
+            thing = EmbeddedDocumentField(Thing)
+
+        User.drop_collection()
+
+        User._get_collection().save({
+            'name': 'John',
+            'thing': {
+                'name': 'My thing',
+                'foo': 'Bar',
+                'data': [1, 2, 3]
+            }
+        })
+
+        self.assertRaises(FieldDoesNotExist, User.objects.first)
+
+    def test_load_undefined_fields_on_embedded_document_with_strict_false_on_doc(self):
+        class Thing(EmbeddedDocument):
+            name = StringField()
+
+        class User(Document):
+            name = StringField()
+            thing = EmbeddedDocumentField(Thing)
+
+            meta = {'strict': False}
+
+        User.drop_collection()
+
+        User._get_collection().save({
+            'name': 'John',
+            'thing': {
+                'name': 'My thing',
+                'foo': 'Bar',
+                'data': [1, 2, 3]
+            }
+        })
+
+        self.assertRaises(FieldDoesNotExist, User.objects.first)
+
+    def test_load_undefined_fields_on_embedded_document_with_strict_false(self):
+        class Thing(EmbeddedDocument):
+            name = StringField()
+
+            meta = {'strict': False}
+
+        class User(Document):
+            name = StringField()
+            thing = EmbeddedDocumentField(Thing)
+
+        User.drop_collection()
+
+        User._get_collection().save({
+            'name': 'John',
+            'thing': {
+                'name': 'My thing',
+                'foo': 'Bar',
+                'data': [1, 2, 3]
+            }
+        })
+
+        user = User.objects.first()
+        self.assertEqual(user.name, 'John')
+        self.assertEqual(user.thing.name, 'My thing')
+        self.assertFalse(hasattr(user.thing, 'foo'))
+        self.assertEqual(user.thing._data['foo'], 'Bar')
+        self.assertFalse(hasattr(user.thing, 'data'))
+        self.assertEqual(user.thing._data['data'], [1, 2, 3])
+
     def test_spaces_in_keys(self):
 
         class Embedded(DynamicEmbeddedDocument):
@@ -2484,6 +2820,32 @@ class InstanceTest(unittest.TestCase):
 
         self.assertRaises(OperationError, change_shard_key)
 
+    def test_shard_key_in_embedded_document(self):
+        class Foo(EmbeddedDocument):
+            foo = StringField()
+
+        class Bar(Document):
+            meta = {
+                'shard_key': ('foo.foo',)
+            }
+            foo = EmbeddedDocumentField(Foo)
+            bar = StringField()
+
+        foo_doc = Foo(foo='hello')
+        bar_doc = Bar(foo=foo_doc, bar='world')
+        bar_doc.save()
+
+        self.assertTrue(bar_doc.id is not None)
+
+        bar_doc.bar = 'baz'
+        bar_doc.save()
+
+        def change_shard_key():
+            bar_doc.foo.foo = 'something'
+            bar_doc.save()
+
+        self.assertRaises(OperationError, change_shard_key)
+
     def test_shard_key_primary(self):
         class LogEntry(Document):
             machine = StringField(primary_key=True)
@@ -2566,6 +2928,20 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 42)
 
+    def test_positional_creation_embedded(self):
+        """Ensure that embedded document may be created using positional arguments.
+        """
+        job = self.Job("Test Job", 4)
+        self.assertEqual(job.name, "Test Job")
+        self.assertEqual(job.years, 4)
+
+    def test_mixed_creation_embedded(self):
+        """Ensure that embedded document may be created using mixed arguments.
+        """
+        job = self.Job("Test Job", years=4)
+        self.assertEqual(job.name, "Test Job")
+        self.assertEqual(job.years, 4)
+
     def test_mixed_creation_dynamic(self):
         """Ensure that document may be created using mixed arguments.
         """
@@ -2742,6 +3118,17 @@ class InstanceTest(unittest.TestCase):
         p4 = Person.objects()[0]
         p4.save()
         self.assertEquals(p4.height, 189)
+        
+        # However the default will not be fixed in DB
+        self.assertEquals(Person.objects(height=189).count(), 0)
+        
+        # alter DB for the new default
+        coll = Person._get_collection()
+        for person in Person.objects.as_pymongo():
+            if 'height' not in person:
+                person['height'] = 189
+                coll.save(person)
+                
         self.assertEquals(Person.objects(height=189).count(), 1)
 
     def test_from_son(self):
@@ -2799,5 +3186,36 @@ class InstanceTest(unittest.TestCase):
         self.assertNotEqual(p, p1)
         self.assertEqual(p, p)
 
+    def test_list_iter(self):
+        # 914
+        class B(EmbeddedDocument):
+            v = StringField()
+
+        class A(Document):
+            l = ListField(EmbeddedDocumentField(B))
+
+        A.objects.delete()
+        A(l=[B(v='1'), B(v='2'), B(v='3')]).save()
+        a = A.objects.get()
+        self.assertEqual(a.l._instance, a)
+        for idx, b in enumerate(a.l):
+            self.assertEqual(b._instance, a)
+        self.assertEqual(idx, 2)
+
+    def test_falsey_pk(self):
+        """Ensure that we can create and update a document with Falsey PK.
+        """
+        class Person(Document):
+            age = IntField(primary_key=True)
+            height = FloatField()
+
+        person = Person()
+        person.age = 0
+        person.height = 1.89
+        person.save()
+
+        person.update(set__height=2.0)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/document/validation.py

@@ -165,6 +165,53 @@ class ValidatorErrorTest(unittest.TestCase):
 
         self.assertRaises(ValidationError, lambda: d2.validate())
 
+    def test_parent_reference_in_child_document(self):
+        """
+        Test to ensure a ReferenceField can store a reference to a parent
+        class when inherited. Issue #954.
+        """
+        class Parent(Document):
+            meta = {'allow_inheritance': True}
+            reference = ReferenceField('self')
+
+        class Child(Parent):
+            pass
+
+        parent = Parent()
+        parent.save()
+
+        child = Child(reference=parent)
+
+        # Saving child should not raise a ValidationError
+        try:
+            child.save()
+        except ValidationError as e:
+            self.fail("ValidationError raised: %s" % e.message)
+
+    def test_parent_reference_set_as_attribute_in_child_document(self):
+        """
+        Test to ensure a ReferenceField can store a reference to a parent
+        class when inherited and when set via attribute. Issue #954.
+        """
+        class Parent(Document):
+            meta = {'allow_inheritance': True}
+            reference = ReferenceField('self')
+
+        class Child(Parent):
+            pass
+
+        parent = Parent()
+        parent.save()
+
+        child = Child()
+        child.reference = parent
+
+        # Saving the child should not raise a ValidationError
+        try:
+            child.save()
+        except ValidationError as e:
+            self.fail("ValidationError raised: %s" % e.message)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/fields/fields.py

@@ -1,10 +1,18 @@
 # -*- coding: utf-8 -*-
 import sys
+
+import six
+from nose.plugins.skip import SkipTest
+
 sys.path[0:0] = [""]
 
 import datetime
 import unittest
 import uuid
+import math
+import itertools
+import re
+import six
 
 try:
     import dateutil
@@ -14,12 +22,16 @@ except ImportError:
 from decimal import Decimal
 
 from bson import Binary, DBRef, ObjectId
+try:
+    from bson.int64 import Int64
+except ImportError:
+    Int64 = long
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.base import _document_registry
 from mongoengine.base.datastructures import BaseDict, EmbeddedDocumentList
-from mongoengine.errors import NotRegistered
+from mongoengine.errors import NotRegistered, DoesNotExist
 from mongoengine.python_support import PY3, b, bin_type
 
 __all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
@@ -34,6 +46,7 @@ class FieldTest(unittest.TestCase):
     def tearDown(self):
         self.db.drop_collection('fs.files')
         self.db.drop_collection('fs.chunks')
+        self.db.drop_collection('mongoengine.counters')
 
     def test_default_values_nothing_set(self):
         """Ensure that default field values are used when creating a document.
@@ -336,6 +349,23 @@ class FieldTest(unittest.TestCase):
         link.url = 'http://www.google.com:8080'
         link.validate()
 
+    def test_url_scheme_validation(self):
+        """Ensure that URLFields validate urls with specific schemes properly.
+        """
+        class Link(Document):
+            url = URLField()
+
+        class SchemeLink(Document):
+            url = URLField(schemes=['ws', 'irc'])
+
+        link = Link()
+        link.url = 'ws://google.com'
+        self.assertRaises(ValidationError, link.validate)
+
+        scheme_link = SchemeLink()
+        scheme_link.url = 'ws://google.com'
+        scheme_link.validate()
+
     def test_int_validation(self):
         """Ensure that invalid values cannot be assigned to int fields.
         """
@@ -376,20 +406,37 @@ class FieldTest(unittest.TestCase):
         class Person(Document):
             height = FloatField(min_value=0.1, max_value=3.5)
 
+        class BigPerson(Document):
+            height = FloatField()
+
         person = Person()
         person.height = 1.89
         person.validate()
 
         person.height = '2.0'
         self.assertRaises(ValidationError, person.validate)
+
         person.height = 0.01
         self.assertRaises(ValidationError, person.validate)
+
         person.height = 4.0
         self.assertRaises(ValidationError, person.validate)
 
         person_2 = Person(height='something invalid')
         self.assertRaises(ValidationError, person_2.validate)
 
+        big_person = BigPerson()
+
+        for value, value_type in enumerate(six.integer_types):
+            big_person.height = value_type(value)
+            big_person.validate()
+
+        big_person.height = 2 ** 500
+        big_person.validate()
+
+        big_person.height = 2 ** 100000  # Too big for a float value
+        self.assertRaises(ValidationError, big_person.validate)
+
     def test_decimal_validation(self):
         """Ensure that invalid values cannot be assigned to decimal fields.
         """
@@ -689,6 +736,7 @@ class FieldTest(unittest.TestCase):
         """
         class LogEntry(Document):
             date = ComplexDateTimeField()
+            date_with_dots = ComplexDateTimeField(separator='.')
 
         LogEntry.drop_collection()
 
@@ -729,6 +777,18 @@ class FieldTest(unittest.TestCase):
             log1 = LogEntry.objects.get(date=d1)
             self.assertEqual(log, log1)
 
+        # Test string padding
+        microsecond = map(int, [math.pow(10, x) for x in xrange(6)])
+        mm = dd = hh = ii = ss = [1, 10]
+
+        for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond):
+            stored = LogEntry(date=datetime.datetime(*values)).to_mongo()['date']
+            self.assertTrue(re.match('^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$', stored) is not None)
+
+        # Test separator
+        stored = LogEntry(date_with_dots=datetime.datetime(2014, 1, 1)).to_mongo()['date_with_dots']
+        self.assertTrue(re.match('^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$', stored) is not None)
+
         LogEntry.drop_collection()
 
     def test_complexdatetime_usage(self):
@@ -787,6 +847,25 @@ class FieldTest(unittest.TestCase):
 
         LogEntry.drop_collection()
 
+        # Test microsecond-level ordering/filtering
+        for microsecond in (99, 999, 9999, 10000):
+            LogEntry(date=datetime.datetime(2015, 1, 1, 0, 0, 0, microsecond)).save()
+
+        logs = list(LogEntry.objects.order_by('date'))
+        for next_idx, log in enumerate(logs[:-1], start=1):
+            next_log = logs[next_idx]
+            self.assertTrue(log.date < next_log.date)
+
+        logs = list(LogEntry.objects.order_by('-date'))
+        for next_idx, log in enumerate(logs[:-1], start=1):
+            next_log = logs[next_idx]
+            self.assertTrue(log.date > next_log.date)
+
+        logs = LogEntry.objects.filter(date__lte=datetime.datetime(2015, 1, 1, 0, 0, 0, 10000))
+        self.assertEqual(logs.count(), 4)
+
+        LogEntry.drop_collection()
+
     def test_list_validation(self):
         """Ensure that a list field only accepts lists with valid elements.
         """
@@ -881,10 +960,17 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(post.comments[0].content, comment2.content)
         self.assertEqual(post.comments[1].content, comment1.content)
 
+        post.comments[0].order = 2
+        post.save()
+        post.reload()
+
+        self.assertEqual(post.comments[0].content, comment1.content)
+        self.assertEqual(post.comments[1].content, comment2.content)
+
         BlogPost.drop_collection()
 
     def test_reverse_list_sorting(self):
-        '''Ensure that a reverse sorted list field properly sorts values'''
+        """Ensure that a reverse sorted list field properly sorts values"""
 
         class Category(EmbeddedDocument):
             count = IntField()
@@ -946,8 +1032,68 @@ class FieldTest(unittest.TestCase):
             BlogPost.objects.filter(info__0__test__exact='5').count(), 0)
         self.assertEqual(
             BlogPost.objects.filter(info__100__test__exact='test').count(), 0)
+
+        post = BlogPost()
+        post.info = ['1', '2']
+        post.save()
+        post = BlogPost.objects(info=['1', '2']).get()
+        post.info += ['3', '4']
+        post.save()
+        self.assertEqual(BlogPost.objects(info=['1', '2', '3', '4']).count(), 1)
+        post = BlogPost.objects(info=['1', '2', '3', '4']).get()
+        post.info *= 2
+        post.save()
+        self.assertEqual(BlogPost.objects(info=['1', '2', '3', '4', '1', '2', '3', '4']).count(), 1)
         BlogPost.drop_collection()
 
+    def test_list_assignment(self):
+        """Ensure that list field element assignment and slicing work
+        """
+        class BlogPost(Document):
+            info = ListField()
+
+        BlogPost.drop_collection()
+
+        post = BlogPost()
+        post.info = ['e1', 'e2', 3, '4', 5]
+        post.save()
+
+        post.info[0] = 1
+        post.save()
+        post.reload()
+        self.assertEqual(post.info[0], 1)
+
+        post.info[1:3] = ['n2', 'n3']
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', '4', 5])
+
+        post.info[-1] = 'n5'
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', '4', 'n5'])
+
+        post.info[-2] = 4
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', 4, 'n5'])
+
+        post.info[1:-1] = [2]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 2, 'n5'])
+
+        post.info[:-1] = [1, 'n2', 'n3', 4]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 'n2', 'n3', 4, 'n5'])
+
+        post.info[-4:3] = [2, 3]
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, [1, 2, 3, 4, 'n5'])
+
+
     def test_list_field_passed_in_value(self):
         class Foo(Document):
             bars = ListField(ReferenceField("Bar"))
@@ -1062,6 +1208,19 @@ class FieldTest(unittest.TestCase):
         simple = simple.reload()
         self.assertEqual(simple.widgets, [4])
 
+    def test_list_field_with_negative_indices(self):
+
+        class Simple(Document):
+            widgets = ListField()
+
+        simple = Simple(widgets=[1, 2, 3, 4]).save()
+        simple.widgets[-1] = 5
+        self.assertEqual(['widgets.3'], simple._changed_fields)
+        simple.save()
+
+        simple = simple.reload()
+        self.assertEqual(simple.widgets, [1, 2, 3, 5])
+
     def test_list_field_complex(self):
         """Ensure that the list fields can handle the complex types."""
 
@@ -1183,6 +1342,44 @@ class FieldTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_dictfield_dump_document(self):
+        """Ensure a DictField can handle another document's dump
+        """
+        class Doc(Document):
+            field = DictField()
+
+        class ToEmbed(Document):
+            id = IntField(primary_key=True, default=1)
+            recursive = DictField()
+
+        class ToEmbedParent(Document):
+            id = IntField(primary_key=True, default=1)
+            recursive = DictField()
+
+            meta = {'allow_inheritance': True}
+
+        class ToEmbedChild(ToEmbedParent):
+            pass
+
+        to_embed_recursive = ToEmbed(id=1).save()
+        to_embed = ToEmbed(
+            id=2, recursive=to_embed_recursive.to_mongo().to_dict()).save()
+        doc = Doc(field=to_embed.to_mongo().to_dict())
+        doc.save()
+        assert isinstance(doc.field, dict)
+        assert doc.field == {'_id': 2, 'recursive': {'_id': 1, 'recursive': {}}}
+        # Same thing with a Document with a _cls field
+        to_embed_recursive = ToEmbedChild(id=1).save()
+        to_embed_child = ToEmbedChild(
+            id=2, recursive=to_embed_recursive.to_mongo().to_dict()).save()
+        doc = Doc(field=to_embed_child.to_mongo().to_dict())
+        doc.save()
+        assert isinstance(doc.field, dict)
+        assert doc.field == {
+            '_id': 2, '_cls': 'ToEmbedParent.ToEmbedChild',
+            'recursive': {'_id': 1, '_cls': 'ToEmbedParent.ToEmbedChild', 'recursive': {}}
+        }
+
     def test_dictfield_strict(self):
         """Ensure that dict field handles validation if provided a strict field type."""
 
@@ -1260,7 +1457,6 @@ class FieldTest(unittest.TestCase):
     def test_atomic_update_dict_field(self):
         """Ensure that the entire DictField can be atomically updated."""
 
-
         class Simple(Document):
             mapping = DictField(field=ListField(IntField(required=True)))
 
@@ -1275,7 +1471,7 @@ class FieldTest(unittest.TestCase):
         self.assertEqual({"ints": [3, 4]}, e.mapping)
 
         def create_invalid_mapping():
-            e.update(set__mapping={"somestrings": ["foo", "bar",]})
+            e.update(set__mapping={"somestrings": ["foo", "bar", ]})
 
         self.assertRaises(ValueError, create_invalid_mapping)
 
@@ -1384,16 +1580,49 @@ class FieldTest(unittest.TestCase):
     def test_map_field_lookup(self):
         """Ensure MapField lookups succeed on Fields without a lookup method"""
 
+        class Action(EmbeddedDocument):
+            operation = StringField()
+            object = StringField()
+
         class Log(Document):
             name = StringField()
             visited = MapField(DateTimeField())
+            actions = MapField(EmbeddedDocumentField(Action))
 
         Log.drop_collection()
-        Log(name="wilson", visited={'friends': datetime.datetime.now()}).save()
+        Log(name="wilson", visited={'friends': datetime.datetime.now()},
+            actions={'friends': Action(operation='drink', object='beer')}).save()
 
         self.assertEqual(1, Log.objects(
             visited__friends__exists=True).count())
 
+        self.assertEqual(1, Log.objects(
+            actions__friends__operation='drink',
+            actions__friends__object='beer').count())
+
+    def test_map_field_unicode(self):
+
+        class Info(EmbeddedDocument):
+            description = StringField()
+            value_list = ListField(field=StringField())
+
+        class BlogPost(Document):
+            info_dict = MapField(field=EmbeddedDocumentField(Info))
+
+        BlogPost.drop_collection()
+
+        tree = BlogPost(info_dict={
+            u"éééé": {
+                'description': u"VALUE: éééé"
+            }
+        })
+
+        tree.save()
+
+        self.assertEqual(BlogPost.objects.get(id=tree.id).info_dict[u"éééé"].description, u"VALUE: éééé")
+
+        BlogPost.drop_collection()
+
     def test_embedded_db_field(self):
 
         class Embedded(EmbeddedDocument):
@@ -1430,6 +1659,8 @@ class FieldTest(unittest.TestCase):
             name = StringField()
             preferences = EmbeddedDocumentField(PersonPreferences)
 
+        Person.drop_collection()
+
         person = Person(name='Test User')
         person.preferences = 'My Preferences'
         self.assertRaises(ValidationError, person.validate)
@@ -1462,12 +1693,70 @@ class FieldTest(unittest.TestCase):
             content = StringField()
             author = EmbeddedDocumentField(User)
 
+        BlogPost.drop_collection()
+
         post = BlogPost(content='What I did today...')
         post.author = PowerUser(name='Test User', power=47)
         post.save()
 
         self.assertEqual(47, BlogPost.objects.first().author.power)
 
+    def test_embedded_document_inheritance_with_list(self):
+        """Ensure that nested list of subclassed embedded documents is
+        handled correctly.
+        """
+
+        class Group(EmbeddedDocument):
+            name = StringField()
+            content = ListField(StringField())
+
+        class Basedoc(Document):
+            groups = ListField(EmbeddedDocumentField(Group))
+            meta = {'abstract': True}
+
+        class User(Basedoc):
+            doctype = StringField(require=True, default='userdata')
+
+        User.drop_collection()
+
+        content = ['la', 'le', 'lu']
+        group = Group(name='foo', content=content)
+        foobar = User(groups=[group])
+        foobar.save()
+
+        self.assertEqual(content, User.objects.first().groups[0].content)
+
+    def test_reference_miss(self):
+        """Ensure an exception is raised when dereferencing unknow document
+        """
+
+        class Foo(Document):
+            pass
+
+        class Bar(Document):
+            ref = ReferenceField(Foo)
+            generic_ref = GenericReferenceField()
+
+        Foo.drop_collection()
+        Bar.drop_collection()
+
+        foo = Foo().save()
+        bar = Bar(ref=foo, generic_ref=foo).save()
+
+        # Reference is no longer valid
+        foo.delete()
+        bar = Bar.objects.get()
+        self.assertRaises(DoesNotExist, lambda: getattr(bar, 'ref'))
+        self.assertRaises(DoesNotExist, lambda: getattr(bar, 'generic_ref'))
+
+        # When auto_dereference is disabled, there is no trouble returning DBRef
+        bar = Bar.objects.get()
+        expected = foo.to_dbref()
+        bar._fields['ref']._auto_dereference = False
+        self.assertEqual(bar.ref, expected)
+        bar._fields['generic_ref']._auto_dereference = False
+        self.assertEqual(bar.generic_ref, {'_ref': expected, '_cls': 'Foo'})
+
     def test_reference_validation(self):
         """Ensure that invalid docment objects cannot be assigned to reference
         fields.
@@ -1535,6 +1824,27 @@ class FieldTest(unittest.TestCase):
         mongoed = p1.to_mongo()
         self.assertTrue(isinstance(mongoed['parent'], ObjectId))
 
+    def test_cached_reference_field_get_and_save(self):
+        """
+        Tests #1047: CachedReferenceField creates DBRefs on to_python, but can't save them on to_mongo
+        """
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class Ocorrence(Document):
+            person = StringField()
+            animal = CachedReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocorrence.drop_collection()
+
+        Ocorrence(person="testte",
+                  animal=Animal(name="Leopard", tag="heavy").save()).save()
+        p = Ocorrence.objects.get()
+        p.person = 'new_testte'
+        p.save()
+
     def test_cached_reference_fields(self):
         class Animal(Document):
             name = StringField()
@@ -2068,9 +2378,7 @@ class FieldTest(unittest.TestCase):
         obj = Product.objects(company=None).first()
         self.assertEqual(obj, me)
 
-        obj, created = Product.objects.get_or_create(company=None)
+        obj = Product.objects.get(company=None)
-
-        self.assertEqual(created, False)
         self.assertEqual(obj, me)
 
     def test_reference_query_conversion(self):
@@ -2141,6 +2449,91 @@ class FieldTest(unittest.TestCase):
         Member.drop_collection()
         BlogPost.drop_collection()
 
+    def test_drop_abstract_document(self):
+        """Ensure that an abstract document cannot be dropped given it
+        has no underlying collection.
+        """
+        class AbstractDoc(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        self.assertRaises(OperationError, AbstractDoc.drop_collection)
+
+    def test_reference_class_with_abstract_parent(self):
+        """Ensure that a class with an abstract parent can be referenced.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Sister(Sibling):
+            pass
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+        sister = Sister(name="Alice")
+        sister.save()
+        brother = Brother(name="Bob", sibling=sister)
+        brother.save()
+
+        self.assertEquals(Brother.objects[0].sibling.name, sister.name)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+    def test_reference_abstract_class(self):
+        """Ensure that an abstract class instance cannot be used in the
+        reference of that abstract class.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Sister(Sibling):
+            pass
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+        sister = Sibling(name="Alice")
+        brother = Brother(name="Bob", sibling=sister)
+        self.assertRaises(ValidationError, brother.save)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+    def test_abstract_reference_base_type(self):
+        """Ensure that an an abstract reference fails validation when given a
+        Document that does not inherit from the abstract type.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        class Mother(Document):
+            name = StringField()
+
+        Brother.drop_collection()
+        Mother.drop_collection()
+
+        mother = Mother(name="Carol")
+        mother.save()
+        brother = Brother(name="Bob", sibling=mother)
+        self.assertRaises(ValidationError, brother.save)
+
+        Brother.drop_collection()
+        Mother.drop_collection()
+
     def test_generic_reference(self):
         """Ensure that a GenericReferenceField properly dereferences items.
         """
@@ -2294,6 +2687,62 @@ class FieldTest(unittest.TestCase):
         bm = Bookmark.objects.first()
         self.assertEqual(bm.bookmark_object, post_1)
 
+    def test_generic_reference_string_choices(self):
+        """Ensure that a GenericReferenceField can handle choices as strings
+        """
+        class Link(Document):
+            title = StringField()
+
+        class Post(Document):
+            title = StringField()
+
+        class Bookmark(Document):
+            bookmark_object = GenericReferenceField(choices=('Post', Link))
+
+        Link.drop_collection()
+        Post.drop_collection()
+        Bookmark.drop_collection()
+
+        link_1 = Link(title="Pitchfork")
+        link_1.save()
+
+        post_1 = Post(title="Behind the Scenes of the Pavement Reunion")
+        post_1.save()
+
+        bm = Bookmark(bookmark_object=link_1)
+        bm.save()
+
+        bm = Bookmark(bookmark_object=post_1)
+        bm.save()
+
+        bm = Bookmark(bookmark_object=bm)
+        self.assertRaises(ValidationError, bm.validate)
+
+    def test_generic_reference_choices_no_dereference(self):
+        """Ensure that a GenericReferenceField can handle choices on
+        non-derefenreced (i.e. DBRef) elements
+        """
+        class Post(Document):
+            title = StringField()
+
+        class Bookmark(Document):
+            bookmark_object = GenericReferenceField(choices=(Post, ))
+            other_field = StringField()
+
+        Post.drop_collection()
+        Bookmark.drop_collection()
+
+        post_1 = Post(title="Behind the Scenes of the Pavement Reunion")
+        post_1.save()
+
+        bm = Bookmark(bookmark_object=post_1)
+        bm.save()
+
+        bm = Bookmark.objects.get(id=bm.id)
+        # bookmark_object is now a DBRef
+        bm.other_field = 'dummy_change'
+        bm.save()
+
     def test_generic_reference_list_choices(self):
         """Ensure that a ListField properly dereferences generic references and
         respects choices.
@@ -2361,6 +2810,38 @@ class FieldTest(unittest.TestCase):
         Post.drop_collection()
         User.drop_collection()
 
+    def test_generic_reference_filter_by_dbref(self):
+        """Ensure we can search for a specific generic reference by
+        providing its ObjectId.
+        """
+        class Doc(Document):
+            ref = GenericReferenceField()
+
+        Doc.drop_collection()
+
+        doc1 = Doc.objects.create()
+        doc2 = Doc.objects.create(ref=doc1)
+
+        doc = Doc.objects.get(ref=DBRef('doc', doc1.pk))
+        self.assertEqual(doc, doc2)
+
+    def test_generic_reference_filter_by_objectid(self):
+        """Ensure we can search for a specific generic reference by
+        providing its DBRef.
+        """
+        class Doc(Document):
+            ref = GenericReferenceField()
+
+        Doc.drop_collection()
+
+        doc1 = Doc.objects.create()
+        doc2 = Doc.objects.create(ref=doc1)
+
+        self.assertTrue(isinstance(doc1.pk, ObjectId))
+
+        doc = Doc.objects.get(ref=doc1.pk)
+        self.assertEqual(doc, doc2)
+
     def test_binary_fields(self):
         """Ensure that binary fields can be stored and retrieved.
         """
@@ -2424,10 +2905,29 @@ class FieldTest(unittest.TestCase):
             id = BinaryField(primary_key=True)
 
         Attachment.drop_collection()
-
+        binary_id = uuid.uuid4().bytes
-        att = Attachment(id=uuid.uuid4().bytes).save()
+        att = Attachment(id=binary_id).save()
+        self.assertEqual(1, Attachment.objects.count())
+        self.assertEqual(1, Attachment.objects.filter(id=att.id).count())
+        # TODO use assertIsNotNone once Python 2.6 support is dropped
+        self.assertTrue(Attachment.objects.filter(id=att.id).first() is not None)
         att.delete()
+        self.assertEqual(0, Attachment.objects.count())
 
+    def test_binary_field_primary_filter_by_binary_pk_as_str(self):
+
+        raise SkipTest("Querying by id as string is not currently supported")
+
+        class Attachment(Document):
+            id = BinaryField(primary_key=True)
+
+        Attachment.drop_collection()
+        binary_id = uuid.uuid4().bytes
+        att = Attachment(id=binary_id).save()
+        self.assertEqual(1, Attachment.objects.filter(id=binary_id).count())
+        # TODO use assertIsNotNone once Python 2.6 support is dropped
+        self.assertTrue(Attachment.objects.filter(id=binary_id).first() is not None)
+        att.delete()
         self.assertEqual(0, Attachment.objects.count())
 
     def test_choices_validation(self):
@@ -2533,28 +3033,32 @@ class FieldTest(unittest.TestCase):
                 ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'),
                 ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large')))
             style = StringField(max_length=3, choices=(
-                ('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S')
+                ('S', 'Small'), ('B', 'Baggy'), ('W', 'Wide')), default='W')
 
         Shirt.drop_collection()
 
-        shirt = Shirt()
+        shirt1 = Shirt()
+        shirt2 = Shirt()
 
-        self.assertEqual(shirt.get_size_display(), None)
+        # Make sure get_<field>_display returns the default value (or None)
-        self.assertEqual(shirt.get_style_display(), 'Small')
+        self.assertEqual(shirt1.get_size_display(), None)
+        self.assertEqual(shirt1.get_style_display(), 'Wide')
 
-        shirt.size = "XXL"
+        shirt1.size = 'XXL'
-        shirt.style = "B"
+        shirt1.style = 'B'
-        self.assertEqual(shirt.get_size_display(), 'Extra Extra Large')
+        shirt2.size = 'M'
-        self.assertEqual(shirt.get_style_display(), 'Baggy')
+        shirt2.style = 'S'
+        self.assertEqual(shirt1.get_size_display(), 'Extra Extra Large')
+        self.assertEqual(shirt1.get_style_display(), 'Baggy')
+        self.assertEqual(shirt2.get_size_display(), 'Medium')
+        self.assertEqual(shirt2.get_style_display(), 'Small')
 
         # Set as Z - an invalid choice
-        shirt.size = "Z"
+        shirt1.size = 'Z'
-        shirt.style = "Z"
+        shirt1.style = 'Z'
-        self.assertEqual(shirt.get_size_display(), 'Z')
+        self.assertEqual(shirt1.get_size_display(), 'Z')
-        self.assertEqual(shirt.get_style_display(), 'Z')
+        self.assertEqual(shirt1.get_style_display(), 'Z')
-        self.assertRaises(ValidationError, shirt.validate)
+        self.assertRaises(ValidationError, shirt1.validate)
-
-        Shirt.drop_collection()
 
     def test_simple_choices_validation(self):
         """Ensure that value is in a container of allowed values.
@@ -2869,6 +3373,57 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(1, post.comments[0].id)
         self.assertEqual(2, post.comments[1].id)
 
+    def test_inherited_sequencefield(self):
+        class Base(Document):
+            name = StringField()
+            counter = SequenceField()
+            meta = {'abstract': True}
+
+        class Foo(Base):
+            pass
+
+        class Bar(Base):
+            pass
+
+        bar = Bar(name='Bar')
+        bar.save()
+
+        foo = Foo(name='Foo')
+        foo.save()
+
+        self.assertTrue('base.counter' in
+                        self.db['mongoengine.counters'].find().distinct('_id'))
+        self.assertFalse(('foo.counter' or 'bar.counter') in
+                         self.db['mongoengine.counters'].find().distinct('_id'))
+        self.assertNotEqual(foo.counter, bar.counter)
+        self.assertEqual(foo._fields['counter'].owner_document, Base)
+        self.assertEqual(bar._fields['counter'].owner_document, Base)
+
+    def test_no_inherited_sequencefield(self):
+        class Base(Document):
+            name = StringField()
+            meta = {'abstract': True}
+
+        class Foo(Base):
+            counter = SequenceField()
+
+        class Bar(Base):
+            counter = SequenceField()
+
+        bar = Bar(name='Bar')
+        bar.save()
+
+        foo = Foo(name='Foo')
+        foo.save()
+
+        self.assertFalse('base.counter' in
+                         self.db['mongoengine.counters'].find().distinct('_id'))
+        self.assertTrue(('foo.counter' and 'bar.counter') in
+                         self.db['mongoengine.counters'].find().distinct('_id'))
+        self.assertEqual(foo.counter, bar.counter)
+        self.assertEqual(foo._fields['counter'].owner_document, Foo)
+        self.assertEqual(bar._fields['counter'].owner_document, Bar)
+
     def test_generic_embedded_document(self):
         class Car(EmbeddedDocument):
             name = StringField()
@@ -3003,7 +3558,6 @@ class FieldTest(unittest.TestCase):
         self.assertTrue(user.validate() is None)
 
         user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
-                           "ucictfqpdkK9iS1zeFw8sg7s7cwAF7suIfUfeyueLpfosjn3"
                            "aJIazqqWkm7.net"))
         self.assertTrue(user.validate() is None)
 
@@ -3100,6 +3654,39 @@ class FieldTest(unittest.TestCase):
         doc = Doc.objects.get()
         self.assertEqual(doc.embed_me.field_1, "hello")
 
+    def test_dynamicfield_dump_document(self):
+        """Ensure a DynamicField can handle another document's dump
+        """
+        class Doc(Document):
+            field = DynamicField()
+
+        class ToEmbed(Document):
+            id = IntField(primary_key=True, default=1)
+            recursive = DynamicField()
+
+        class ToEmbedParent(Document):
+            id = IntField(primary_key=True, default=1)
+            recursive = DynamicField()
+
+            meta = {'allow_inheritance': True}
+
+        class ToEmbedChild(ToEmbedParent):
+            pass
+
+        to_embed_recursive = ToEmbed(id=1).save()
+        to_embed = ToEmbed(id=2, recursive=to_embed_recursive).save()
+        doc = Doc(field=to_embed)
+        doc.save()
+        assert isinstance(doc.field, ToEmbed)
+        assert doc.field == to_embed
+        # Same thing with a Document with a _cls field
+        to_embed_recursive = ToEmbedChild(id=1).save()
+        to_embed_child = ToEmbedChild(id=2, recursive=to_embed_recursive).save()
+        doc = Doc(field=to_embed_child)
+        doc.save()
+        assert isinstance(doc.field, ToEmbedChild)
+        assert doc.field == to_embed_child
+
     def test_invalid_dict_value(self):
         class DictFieldTest(Document):
             dictionary = DictField(required=True)
@@ -3148,7 +3735,7 @@ class FieldTest(unittest.TestCase):
 
     def test_undefined_field_exception(self):
         """Tests if a `FieldDoesNotExist` exception is raised when trying to
-        set a value to a field that's not defined.
+        instanciate a document with a field that's not defined.
         """
 
         class Doc(Document):
@@ -3159,6 +3746,34 @@ class FieldTest(unittest.TestCase):
 
         self.assertRaises(FieldDoesNotExist, test)
 
+    def test_undefined_field_exception_with_strict(self):
+        """Tests if a `FieldDoesNotExist` exception is raised when trying to
+        instanciate a document with a field that's not defined,
+        even when strict is set to False.
+        """
+
+        class Doc(Document):
+            foo = StringField(db_field='f')
+            meta = {'strict': False}
+
+        def test():
+            Doc(bar='test')
+
+        self.assertRaises(FieldDoesNotExist, test)
+
+    def test_long_field_is_considered_as_int64(self):
+        """
+        Tests that long fields are stored as long in mongo, even if long value
+        is small enough to be an int.
+        """
+        class TestLongFieldConsideredAsInt64(Document):
+            some_long = LongField()
+
+        doc = TestLongFieldConsideredAsInt64(some_long=42).save()
+        db = get_db()
+        self.assertTrue(isinstance(db.test_long_field_considered_as_int64.find()[0]['some_long'], Int64))
+        self.assertTrue(isinstance(doc.some_long, six.integer_types))
+
 
 class EmbeddedDocumentListFieldTestCase(unittest.TestCase):
 
@@ -3546,6 +4161,17 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase):
         # modified
         self.assertEqual(number, 2)
 
+    def test_unicode(self):
+        """
+        Tests that unicode strings handled correctly
+        """
+        post = self.BlogPost(comments=[
+            self.Comments(author='user1', message=u'сообщение'),
+            self.Comments(author='user2', message=u'хабарлама')
+        ]).save()
+        self.assertEqual(post.comments.get(message=u'сообщение').author,
+                         'user1')
+
     def test_save(self):
         """
         Tests the save method of a List of Embedded Documents.
@@ -3596,6 +4222,30 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase):
         # deleted from the database
         self.assertEqual(number, 2)
 
+    def test_empty_list_embedded_documents_with_unique_field(self):
+        """
+        Tests that only one document with an empty list of embedded documents
+        that have a unique field can be saved, but if the unique field is
+        also sparse than multiple documents with an empty list can be saved.
+        """
+        class EmbeddedWithUnique(EmbeddedDocument):
+            number = IntField(unique=True)
+
+        class A(Document):
+            my_list = ListField(EmbeddedDocumentField(EmbeddedWithUnique))
+
+        A(my_list=[]).save()
+        self.assertRaises(NotUniqueError, lambda: A(my_list=[]).save())
+
+        class EmbeddedWithSparseUnique(EmbeddedDocument):
+            number = IntField(unique=True, sparse=True)
+
+        class B(Document):
+            my_list = ListField(EmbeddedDocumentField(EmbeddedWithSparseUnique))
+
+        B(my_list=[]).save()
+        B(my_list=[]).save()
+
     def test_filtered_delete(self):
         """
         Tests the delete method of a List of Embedded Documents
@@ -3627,5 +4277,23 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase):
         # deleted from the database
         self.assertEqual(number, 1)
 
+    def test_custom_data(self):
+        """
+        Tests that custom data is saved in the field object
+        and doesn't interfere with the rest of field functionalities.
+        """
+        custom_data = {'a': 'a_value', 'b': [1, 2]}
+
+        class CustomData(Document):
+            a_field = IntField()
+            c_field = IntField(custom_data=custom_data)
+
+        a1 = CustomData(a_field=1, c_field=2).save()
+        self.assertEqual(2, a1.c_field)
+        self.assertFalse(hasattr(a1.c_field, 'custom_data'))
+        self.assertTrue(hasattr(CustomData.c_field, 'custom_data'))
+        self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a'])
+
+
 if __name__ == '__main__':
     unittest.main()

tests/fields/file_tests.py

@@ -12,7 +12,7 @@ import gridfs
 from nose.plugins.skip import SkipTest
 from mongoengine import *
 from mongoengine.connection import get_db
-from mongoengine.python_support import PY3, b, StringIO
+from mongoengine.python_support import b, StringIO
 
 try:
     from PIL import Image
@@ -112,7 +112,7 @@ class FileTest(unittest.TestCase):
         result.the_file.delete()
 
         # Ensure deleted file returns None
-        self.assertTrue(result.the_file.read() == None)
+        self.assertTrue(result.the_file.read() is None)
 
     def test_file_fields_stream_after_none(self):
         """Ensure that a file field can be written to after it has been saved as
@@ -138,7 +138,7 @@ class FileTest(unittest.TestCase):
         result = StreamFile.objects.first()
         self.assertTrue(streamfile == result)
         self.assertEqual(result.the_file.read(), text + more_text)
-        #self.assertEqual(result.the_file.content_type, content_type)
+        # self.assertEqual(result.the_file.content_type, content_type)
         result.the_file.seek(0)
         self.assertEqual(result.the_file.tell(), 0)
         self.assertEqual(result.the_file.read(len(text)), text)
@@ -148,7 +148,7 @@ class FileTest(unittest.TestCase):
         result.the_file.delete()
 
         # Ensure deleted file returns None
-        self.assertTrue(result.the_file.read() == None)
+        self.assertTrue(result.the_file.read() is None)
 
     def test_file_fields_set(self):
 
@@ -297,6 +297,71 @@ class FileTest(unittest.TestCase):
         test_file = TestFile()
         self.assertFalse(test_file.the_file in [{"test": 1}])
 
+    def test_file_disk_space(self): 
+        """ Test disk space usage when we delete/replace a file """ 
+        class TestFile(Document):
+            the_file = FileField()
+            
+        text = b('Hello, World!')
+        content_type = 'text/plain'
+
+        testfile = TestFile()
+        testfile.the_file.put(text, content_type=content_type, filename="hello")
+        testfile.save()
+        
+        # Now check fs.files and fs.chunks 
+        db = TestFile._get_db()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 1)
+        self.assertEquals(len(list(chunks)), 1)
+
+        # Deleting the docoument should delete the files 
+        testfile.delete()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 0)
+        self.assertEquals(len(list(chunks)), 0)
+        
+        # Test case where we don't store a file in the first place 
+        testfile = TestFile()
+        testfile.save()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 0)
+        self.assertEquals(len(list(chunks)), 0)
+        
+        testfile.delete()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 0)
+        self.assertEquals(len(list(chunks)), 0)
+        
+        # Test case where we overwrite the file 
+        testfile = TestFile()
+        testfile.the_file.put(text, content_type=content_type, filename="hello")
+        testfile.save()
+        
+        text = b('Bonjour, World!')
+        testfile.the_file.replace(text, content_type=content_type, filename="hello")
+        testfile.save()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 1)
+        self.assertEquals(len(list(chunks)), 1)
+        
+        testfile.delete()
+        
+        files = db.fs.files.find()
+        chunks = db.fs.chunks.find()
+        self.assertEquals(len(list(files)), 0)
+        self.assertEquals(len(list(chunks)), 0)
+
     def test_image_field(self):
         if not HAS_PIL:
             raise SkipTest('PIL not installed')

tests/fields/geo.py

@@ -115,7 +115,7 @@ class GeoFieldTest(unittest.TestCase):
             expected = "Invalid LineString:\nBoth values (%s) in point must be float or int" % repr(coord[0])
             self._test_for_expected_error(Location, coord, expected)
 
-        Location(loc=[[1, 2], [3, 4], [5, 6], [1,2]]).validate()
+        Location(loc=[[1, 2], [3, 4], [5, 6], [1, 2]]).validate()
 
     def test_polygon_validation(self):
         class Location(Document):
@@ -226,7 +226,7 @@ class GeoFieldTest(unittest.TestCase):
             expected = "Invalid MultiLineString:\nBoth values (%s) in point must be float or int" % repr(coord[0][0])
             self._test_for_expected_error(Location, coord, expected)
 
-        Location(loc=[[[1, 2], [3, 4], [5, 6], [1,2]]]).validate()
+        Location(loc=[[[1, 2], [3, 4], [5, 6], [1, 2]]]).validate()
 
     def test_multipolygon_validation(self):
         class Location(Document):
@@ -336,12 +336,11 @@ class GeoFieldTest(unittest.TestCase):
         Location.drop_collection()
         Parent.drop_collection()
 
-        list(Parent.objects)
+        Parent(name='Berlin').save()
-
+        info = Parent._get_collection().index_information()
-        collection = Parent._get_collection()
-        info = collection.index_information()
-
         self.assertFalse('location_2d' in info)
+        info = Location._get_collection().index_information()
+        self.assertTrue('location_2d' in info)
 
         self.assertEqual(len(Parent._geo_indices()), 0)
         self.assertEqual(len(Location._geo_indices()), 1)

tests/fixtures.py

@@ -17,7 +17,16 @@ class PickleTest(Document):
     photo = FileField()
 
 
-class PickleDyanmicEmbedded(DynamicEmbeddedDocument):
+class NewDocumentPickleTest(Document):
+    number = IntField()
+    string = StringField(choices=(('One', '1'), ('Two', '2')))
+    embedded = EmbeddedDocumentField(PickleEmbedded)
+    lists = ListField(StringField())
+    photo = FileField()
+    new_field = StringField()
+
+
+class PickleDynamicEmbedded(DynamicEmbeddedDocument):
     date = DateTimeField(default=datetime.now)
 
 

tests/migration/__init__.py

@@ -1,8 +1,11 @@
+import unittest
+
 from convert_to_new_inheritance_model import *
 from decimalfield_as_float import *
-from refrencefield_dbref_to_object_id import *
+from referencefield_dbref_to_object_id import *
 from turn_off_inheritance import *
 from uuidfield_to_binary import *
 
+
 if __name__ == '__main__':
     unittest.main()

tests/queryset/geo.py

@@ -1,12 +1,16 @@
 import sys
+
 sys.path[0:0] = [""]
 
 import unittest
 from datetime import datetime, timedelta
-from mongoengine import *
 
+from pymongo.errors import OperationFailure
+from mongoengine import *
+from mongoengine.connection import get_connection
 from nose.plugins.skip import SkipTest
 
+
 __all__ = ("GeoQueriesTest",)
 
 
@@ -66,6 +70,16 @@ class GeoQueriesTest(unittest.TestCase):
         self.assertEqual(events.count(), 1)
         self.assertEqual(events[0], event2)
 
+        # find events at least 10 degrees away of san francisco
+        point = [-122.415579, 37.7566023]
+        events = Event.objects(location__near=point, location__min_distance=10)
+        # The following real test passes on MongoDB 3 but minDistance seems
+        # buggy on older MongoDB versions
+        if get_connection().server_info()['versionArray'][0] > 2:
+            self.assertEqual(events.count(), 2)
+        else:
+            self.assertTrue(events.count() >= 2)
+
         # find events within 10 degrees of san francisco
         point_and_distance = [[-122.415579, 37.7566023], 10]
         events = Event.objects(location__within_distance=point_and_distance)
@@ -141,7 +155,13 @@ class GeoQueriesTest(unittest.TestCase):
     def test_spherical_geospatial_operators(self):
         """Ensure that spherical geospatial queries are working
         """
-        raise SkipTest("https://jira.mongodb.org/browse/SERVER-14039")
+        # Needs MongoDB > 2.6.4 https://jira.mongodb.org/browse/SERVER-14039
+        connection = get_connection()
+        info = connection.test.command('buildInfo')
+        mongodb_version = tuple([int(i) for i in info['version'].split('.')])
+        if mongodb_version < (2, 6, 4):
+            raise SkipTest("Need MongoDB version 2.6.4+")
+
         class Point(Document):
             location = GeoPointField()
 
@@ -161,7 +181,7 @@ class GeoQueriesTest(unittest.TestCase):
 
         # Same behavior for _within_spherical_distance
         points = Point.objects(
-            location__within_spherical_distance=[[-122, 37.5], 60/earth_radius]
+            location__within_spherical_distance=[[-122, 37.5], 60 / earth_radius]
         )
         self.assertEqual(points.count(), 2)
 
@@ -169,6 +189,24 @@ class GeoQueriesTest(unittest.TestCase):
                                location__max_distance=60 / earth_radius)
         self.assertEqual(points.count(), 2)
 
+        # Test query works with max_distance, being farer from one point
+        points = Point.objects(location__near_sphere=[-122, 37.8],
+                               location__max_distance=60 / earth_radius)
+        close_point = points.first()
+        self.assertEqual(points.count(), 1)
+
+        # Test query works with min_distance, being farer from one point
+        points = Point.objects(location__near_sphere=[-122, 37.8],
+                               location__min_distance=60 / earth_radius)
+        # The following real test passes on MongoDB 3 but minDistance seems
+        # buggy on older MongoDB versions
+        if get_connection().server_info()['versionArray'][0] > 2:
+            self.assertEqual(points.count(), 1)
+            far_point = points.first()
+            self.assertNotEqual(close_point, far_point)
+        else:
+            self.assertTrue(points.count() >= 1)
+
         # Finds both points, but orders the north point first because it's
         # closer to the reference point to the north.
         points = Point.objects(location__near_sphere=[-122, 38.5])
@@ -251,6 +289,20 @@ class GeoQueriesTest(unittest.TestCase):
         self.assertEqual(events.count(), 2)
         self.assertEqual(events[0], event3)
 
+        # ensure min_distance and max_distance combine well
+        events = Event.objects(location__near=[-87.67892, 41.9120459],
+                               location__min_distance=1000,
+                               location__max_distance=10000).order_by("-date")
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event3)
+
+        # ensure ordering is respected by "near"
+        events = Event.objects(location__near=[-87.67892, 41.9120459],
+                               # location__min_distance=10000
+                               location__min_distance=10000).order_by("-date")
+        self.assertEqual(events.count(), 1)
+        self.assertEqual(events[0], event2)
+
         # check that within_box works
         box = [(-125.0, 35.0), (-100.0, 40.0)]
         events = Event.objects(location__geo_within_box=box)

tests/queryset/pickable.py

@@ -0,0 +1,78 @@
+import pickle
+import unittest
+from pymongo.mongo_client import MongoClient
+from mongoengine import Document, StringField, IntField
+from mongoengine.connection import connect
+
+__author__ = 'stas'
+
+class Person(Document):
+    name = StringField()
+    age = IntField()
+
+class TestQuerysetPickable(unittest.TestCase):
+    """
+    Test for adding pickling support for QuerySet instances
+    See issue https://github.com/MongoEngine/mongoengine/issues/442
+    """
+    def setUp(self):
+        super(TestQuerysetPickable, self).setUp()
+
+        connection = connect(db="test") #type: pymongo.mongo_client.MongoClient
+
+        connection.drop_database("test")
+
+        self.john = Person.objects.create(
+            name="John",
+            age=21
+        )
+
+
+    def test_picke_simple_qs(self):
+
+        qs = Person.objects.all()
+
+        pickle.dumps(qs)
+
+    def _get_loaded(self, qs):
+        s = pickle.dumps(qs)
+
+        return pickle.loads(s)
+
+    def test_unpickle(self):
+        qs = Person.objects.all()
+
+        loadedQs = self._get_loaded(qs)
+
+        self.assertEqual(qs.count(), loadedQs.count())
+
+        #can update loadedQs
+        loadedQs.update(age=23)
+
+        #check
+        self.assertEqual(Person.objects.first().age, 23)
+
+    def test_pickle_support_filtration(self):
+        Person.objects.create(
+            name="Alice",
+            age=22
+        )
+
+        Person.objects.create(
+            name="Bob",
+            age=23
+        )
+
+        qs = Person.objects.filter(age__gte=22)
+        self.assertEqual(qs.count(), 2)
+
+        loaded = self._get_loaded(qs)
+
+        self.assertEqual(loaded.count(), 2)
+        self.assertEqual(loaded.filter(name="Bob").first().age, 23)
+    
+
+
+
+
+

tests/queryset/transform.py

@@ -1,11 +1,7 @@
-import sys
-sys.path[0:0] = [""]
-
 import unittest
 
 from mongoengine import *
-from mongoengine.queryset import Q
+from mongoengine.queryset import Q, transform
-from mongoengine.queryset import transform
 
 __all__ = ("TransformTest",)
 
@@ -41,8 +37,8 @@ class TransformTest(unittest.TestCase):
         DicDoc.drop_collection()
         Doc.drop_collection()
 
+        DicDoc().save()
         doc = Doc().save()
-        dic_doc = DicDoc().save()
 
         for k, v in (("set", "$set"), ("set_on_insert", "$setOnInsert"), ("push", "$push")):
             update = transform.update(DicDoc, **{"%s__dictField__test" % k: doc})
@@ -55,7 +51,6 @@ class TransformTest(unittest.TestCase):
         update = transform.update(DicDoc, pull__dictField__test=doc)
         self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict))
 
-
     def test_query_field_name(self):
         """Ensure that the correct field name is used when querying.
         """
@@ -156,26 +151,33 @@ class TransformTest(unittest.TestCase):
         class Doc(Document):
             meta = {'allow_inheritance': False}
 
-        raw_query = Doc.objects(__raw__={'deleted': False,
+        raw_query = Doc.objects(__raw__={
-                                'scraped': 'yes',
+            'deleted': False,
-                                '$nor': [{'views.extracted': 'no'},
+            'scraped': 'yes',
-                                         {'attachments.views.extracted':'no'}]
+            '$nor': [
-                                })._query
+                {'views.extracted': 'no'},
+                {'attachments.views.extracted': 'no'}
+            ]
+        })._query
 
-        expected = {'deleted': False, 'scraped': 'yes',
+        self.assertEqual(raw_query, {
-                    '$nor': [{'views.extracted': 'no'},
+            'deleted': False,
-                             {'attachments.views.extracted': 'no'}]}
+            'scraped': 'yes',
-        self.assertEqual(expected, raw_query)
+            '$nor': [
+                {'views.extracted': 'no'},
+                {'attachments.views.extracted': 'no'}
+            ]
+        })
 
     def test_geojson_PointField(self):
         class Location(Document):
             loc = PointField()
 
         update = transform.update(Location, set__loc=[1, 2])
-        self.assertEqual(update, {'$set': {'loc': {"type": "Point", "coordinates": [1,2]}}})
+        self.assertEqual(update, {'$set': {'loc': {"type": "Point", "coordinates": [1, 2]}}})
 
-        update = transform.update(Location, set__loc={"type": "Point", "coordinates": [1,2]})
+        update = transform.update(Location, set__loc={"type": "Point", "coordinates": [1, 2]})
-        self.assertEqual(update, {'$set': {'loc': {"type": "Point", "coordinates": [1,2]}}})
+        self.assertEqual(update, {'$set': {'loc': {"type": "Point", "coordinates": [1, 2]}}})
 
     def test_geojson_LineStringField(self):
         class Location(Document):
@@ -208,6 +210,36 @@ class TransformTest(unittest.TestCase):
         self.assertEqual(Doc.objects(df__type=2).count(), 1)  # str
         self.assertEqual(Doc.objects(df__type=16).count(), 1)  # int
 
+    def test_last_field_name_like_operator(self):
+        class EmbeddedItem(EmbeddedDocument):
+            type = StringField()
+            name = StringField()
+
+        class Doc(Document):
+            item = EmbeddedDocumentField(EmbeddedItem)
+
+        Doc.drop_collection()
+
+        doc = Doc(item=EmbeddedItem(type="axe", name="Heroic axe"))
+        doc.save()
+
+        self.assertEqual(1, Doc.objects(item__type__="axe").count())
+        self.assertEqual(1, Doc.objects(item__name__="Heroic axe").count())
+
+        Doc.objects(id=doc.id).update(set__item__type__='sword')
+        self.assertEqual(1, Doc.objects(item__type__="sword").count())
+        self.assertEqual(0, Doc.objects(item__type__="axe").count())
+
+    def test_understandable_error_raised(self):
+        class Event(Document):
+            title = StringField()
+            location = GeoPointField()
+
+        box = [(35.0, -125.0), (40.0, -100.0)]
+        # I *meant* to execute location__within_box=box
+        events = Event.objects(location__within=box)
+        self.assertRaises(InvalidQueryError, lambda: events.count())
+
 
 if __name__ == '__main__':
     unittest.main()

tests/queryset/visitor.py

@@ -1,14 +1,12 @@
-import sys
+import datetime
-sys.path[0:0] = [""]
+import re
-
 import unittest
 
 from bson import ObjectId
-from datetime import datetime
 
 from mongoengine import *
-from mongoengine.queryset import Q
 from mongoengine.errors import InvalidQueryError
+from mongoengine.queryset import Q
 
 __all__ = ("QTest",)
 
@@ -132,12 +130,12 @@ class QTest(unittest.TestCase):
         TestDoc(x=10).save()
         TestDoc(y=True).save()
 
-        self.assertEqual(query,
+        self.assertEqual(query, {
-        {'$and': [
+            '$and': [
-            {'$or': [{'x': {'$gt': 0}}, {'x': {'$exists': False}}]},
+                {'$or': [{'x': {'$gt': 0}}, {'x': {'$exists': False}}]},
-            {'$or': [{'x': {'$lt': 100}}, {'y': True}]}
+                {'$or': [{'x': {'$lt': 100}}, {'y': True}]}
-        ]})
+            ]
-
+        })
         self.assertEqual(2, TestDoc.objects(q1 & q2).count())
 
     def test_or_and_or_combination(self):
@@ -157,15 +155,14 @@ class QTest(unittest.TestCase):
         q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False)))
         query = (q1 | q2).to_query(TestDoc)
 
-        self.assertEqual(query,
+        self.assertEqual(query, {
-            {'$or': [
+            '$or': [
                 {'$and': [{'x': {'$gt': 0}},
                           {'$or': [{'y': True}, {'y': {'$exists': False}}]}]},
                 {'$and': [{'x': {'$lt': 100}},
                           {'$or': [{'y': False}, {'y': {'$exists': False}}]}]}
-            ]}
+            ]
-        )
+        })
-
         self.assertEqual(2, TestDoc.objects(q1 | q2).count())
 
     def test_multiple_occurence_in_field(self):
@@ -215,19 +212,19 @@ class QTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-        post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False)
+        post1 = BlogPost(title='Test 1', publish_date=datetime.datetime(2010, 1, 8), published=False)
         post1.save()
 
-        post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True)
+        post2 = BlogPost(title='Test 2', publish_date=datetime.datetime(2010, 1, 15), published=True)
         post2.save()
 
         post3 = BlogPost(title='Test 3', published=True)
         post3.save()
 
-        post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8))
+        post4 = BlogPost(title='Test 4', publish_date=datetime.datetime(2010, 1, 8))
         post4.save()
 
-        post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15))
+        post5 = BlogPost(title='Test 1', publish_date=datetime.datetime(2010, 1, 15))
         post5.save()
 
         post6 = BlogPost(title='Test 1', published=False)
@@ -250,7 +247,7 @@ class QTest(unittest.TestCase):
         self.assertTrue(all(obj.id in posts for obj in published_posts))
 
         # Check Q object combination
-        date = datetime(2010, 1, 10)
+        date = datetime.datetime(2010, 1, 10)
         q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True))
         posts = [post.id for post in q]
 
@@ -273,8 +270,10 @@ class QTest(unittest.TestCase):
         # Test invalid query objs
         def wrong_query_objs():
             self.Person.objects('user1')
+
         def wrong_query_objs_filter():
             self.Person.objects('user1')
+
         self.assertRaises(InvalidQueryError, wrong_query_objs)
         self.assertRaises(InvalidQueryError, wrong_query_objs_filter)
 
@@ -284,7 +283,6 @@ class QTest(unittest.TestCase):
         person = self.Person(name='Guido van Rossum')
         person.save()
 
-        import re
         obj = self.Person.objects(Q(name=re.compile('^Gui'))).first()
         self.assertEqual(obj, person)
         obj = self.Person.objects(Q(name=re.compile('^gui'))).first()

tests/test_connection.py

@@ -1,21 +1,34 @@
 import sys
+import datetime
+from pymongo.errors import OperationFailure
+
 sys.path[0:0] = [""]
 
 try:
     import unittest2 as unittest
 except ImportError:
     import unittest
-
+from nose.plugins.skip import SkipTest
-import datetime
 
 import pymongo
 from bson.tz_util import utc
 
-from mongoengine import *
+from mongoengine import (
+    connect, register_connection,
+    Document, DateTimeField
+)
+from mongoengine.python_support import IS_PYMONGO_3
 import mongoengine.connection
 from mongoengine.connection import get_db, get_connection, ConnectionError
 
 
+def get_tz_awareness(connection):
+    if not IS_PYMONGO_3:
+        return connection.tz_aware
+    else:
+        return connection.codec_options.tz_aware
+
+
 class ConnectionTest(unittest.TestCase):
 
     def tearDown(self):
@@ -39,15 +52,99 @@ class ConnectionTest(unittest.TestCase):
         conn = get_connection('testdb')
         self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
 
+    def test_connect_in_mocking(self):
+        """Ensure that the connect() method works properly in mocking.
+        """
+        try:
+            import mongomock
+        except ImportError:
+            raise SkipTest('you need mongomock installed to run this testcase')
+
+        connect('mongoenginetest', host='mongomock://localhost')
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect('mongoenginetest2', host='mongomock://localhost', alias='testdb2')
+        conn = get_connection('testdb2')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect('mongoenginetest3', host='mongodb://localhost', is_mock=True, alias='testdb3')
+        conn = get_connection('testdb3')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect('mongoenginetest4', is_mock=True, alias='testdb4')
+        conn = get_connection('testdb4')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host='mongodb://localhost:27017/mongoenginetest5', is_mock=True, alias='testdb5')
+        conn = get_connection('testdb5')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host='mongomock://localhost:27017/mongoenginetest6', alias='testdb6')
+        conn = get_connection('testdb6')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host='mongomock://localhost:27017/mongoenginetest7', is_mock=True, alias='testdb7')
+        conn = get_connection('testdb7')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+    def test_connect_with_host_list(self):
+        """Ensure that the connect() method works when host is a list
+
+        Uses mongomock to test w/o needing multiple mongod/mongos processes
+        """
+        try:
+            import mongomock
+        except ImportError:
+            raise SkipTest('you need mongomock installed to run this testcase')
+
+        connect(host=['mongomock://localhost'])
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host=['mongodb://localhost'], is_mock=True,  alias='testdb2')
+        conn = get_connection('testdb2')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host=['localhost'], is_mock=True,  alias='testdb3')
+        conn = get_connection('testdb3')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host=['mongomock://localhost:27017', 'mongomock://localhost:27018'], alias='testdb4')
+        conn = get_connection('testdb4')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host=['mongodb://localhost:27017', 'mongodb://localhost:27018'], is_mock=True,  alias='testdb5')
+        conn = get_connection('testdb5')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+        connect(host=['localhost:27017', 'localhost:27018'], is_mock=True,  alias='testdb6')
+        conn = get_connection('testdb6')
+        self.assertTrue(isinstance(conn, mongomock.MongoClient))
+
+    def test_disconnect(self):
+        """Ensure that the disconnect() method works properly
+        """
+        conn1 = connect('mongoenginetest')
+        mongoengine.connection.disconnect()
+        conn2 = connect('mongoenginetest')
+        self.assertTrue(conn1 is not conn2)
+
     def test_sharing_connections(self):
         """Ensure that connections are shared when the connection settings are exactly the same
         """
-        connect('mongoenginetest', alias='testdb1')
+        connect('mongoenginetests', alias='testdb1')
-
         expected_connection = get_connection('testdb1')
 
-        connect('mongoenginetest', alias='testdb2')
+        connect('mongoenginetests', alias='testdb2')
         actual_connection = get_connection('testdb2')
+
+        # Handle PyMongo 3+ Async Connection
+        if IS_PYMONGO_3:
+            # Ensure we are connected, throws ServerSelectionTimeoutError otherwise.
+            # Purposely not catching exception to fail test if thrown.
+            expected_connection.server_info()
+
         self.assertEqual(expected_connection, actual_connection)
 
     def test_connect_uri(self):
@@ -61,7 +158,8 @@ class ConnectionTest(unittest.TestCase):
         c.admin.authenticate("admin", "password")
         c.mongoenginetest.add_user("username", "password")
 
-        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
+        if not IS_PYMONGO_3:
+            self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
 
         connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest')
 
@@ -76,19 +174,9 @@ class ConnectionTest(unittest.TestCase):
         c.mongoenginetest.system.users.remove({})
 
     def test_connect_uri_without_db(self):
-        """Ensure that the connect() method works properly with uri's
+        """Ensure connect() method works properly if the URI doesn't
-        without database_name
+        include a database name.
         """
-        c = connect(db='mongoenginetest', alias='admin')
-        c.admin.system.users.remove({})
-        c.mongoenginetest.system.users.remove({})
-
-        c.admin.add_user("admin", "password")
-        c.admin.authenticate("admin", "password")
-        c.mongoenginetest.add_user("username", "password")
-
-        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
-
         connect("mongoenginetest", host='mongodb://localhost/')
 
         conn = get_connection()
@@ -98,8 +186,67 @@ class ConnectionTest(unittest.TestCase):
         self.assertTrue(isinstance(db, pymongo.database.Database))
         self.assertEqual(db.name, 'mongoenginetest')
 
+    def test_connect_uri_default_db(self):
+        """Ensure connect() defaults to the right database name if
+        the URI and the database_name don't explicitly specify it.
+        """
+        connect(host='mongodb://localhost/')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'test')
+
+    def test_uri_without_credentials_doesnt_override_conn_settings(self):
+        """Ensure connect() uses the username & password params if the URI
+        doesn't explicitly specify them.
+        """
+        c = connect(host='mongodb://localhost/mongoenginetest',
+                    username='user',
+                    password='pass')
+
+        # OperationFailure means that mongoengine attempted authentication
+        # w/ the provided username/password and failed - that's the desired
+        # behavior. If the MongoDB URI would override the credentials
+        self.assertRaises(OperationFailure, get_db)
+
+    def test_connect_uri_with_authsource(self):
+        """Ensure that the connect() method works well with
+        the option `authSource` in URI.
+        This feature was introduced in MongoDB 2.4 and removed in 2.6
+        """
+        # Create users
+        c = connect('mongoenginetest')
+        c.admin.system.users.remove({})
+        c.admin.add_user('username2', 'password')
+
+        # Authentication fails without "authSource"
+        if IS_PYMONGO_3:
+            test_conn = connect('mongoenginetest', alias='test1',
+                                host='mongodb://username2:password@localhost/mongoenginetest')
+            self.assertRaises(OperationFailure, test_conn.server_info)
+        else:
+            self.assertRaises(
+                ConnectionError, connect, 'mongoenginetest', alias='test1',
+                host='mongodb://username2:password@localhost/mongoenginetest'
+            )
+            self.assertRaises(ConnectionError, get_db, 'test1')
+
+        # Authentication succeeds with "authSource"
+        connect(
+            'mongoenginetest', alias='test2',
+            host=('mongodb://username2:password@localhost/'
+                  'mongoenginetest?authSource=admin')
+        )
+        # This will fail starting from MongoDB 2.6+
+        db = get_db('test2')
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+        # Clear all users
         c.admin.system.users.remove({})
-        c.mongoenginetest.system.users.remove({})
 
     def test_register_connection(self):
         """Ensure that connections with different aliases may be registered.
@@ -128,11 +275,11 @@ class ConnectionTest(unittest.TestCase):
         connect('mongoenginetest', alias='t1', tz_aware=True)
         conn = get_connection('t1')
 
-        self.assertTrue(conn.tz_aware)
+        self.assertTrue(get_tz_awareness(conn))
 
         connect('mongoenginetest2', alias='t2')
         conn = get_connection('t2')
-        self.assertFalse(conn.tz_aware)
+        self.assertFalse(get_tz_awareness(conn))
 
     def test_datetime(self):
         connect('mongoenginetest', tz_aware=True)
@@ -156,8 +303,17 @@ class ConnectionTest(unittest.TestCase):
         self.assertEqual(len(mongo_connections.items()), 2)
         self.assertTrue('t1' in mongo_connections.keys())
         self.assertTrue('t2' in mongo_connections.keys())
-        self.assertEqual(mongo_connections['t1'].host, 'localhost')
+        if not IS_PYMONGO_3:
-        self.assertEqual(mongo_connections['t2'].host, '127.0.0.1')
+            self.assertEqual(mongo_connections['t1'].host, 'localhost')
+            self.assertEqual(mongo_connections['t2'].host, '127.0.0.1')
+        else:
+            # Handle PyMongo 3+ Async Connection
+            # Ensure we are connected, throws ServerSelectionTimeoutError otherwise.
+            # Purposely not catching exception to fail test if thrown.
+            mongo_connections['t1'].server_info()
+            mongo_connections['t2'].server_info()
+            self.assertEqual(mongo_connections['t1'].address[0], 'localhost')
+            self.assertEqual(mongo_connections['t2'].address[0], '127.0.0.1')
 
 
 if __name__ == '__main__':

tests/test_datastructures.py

@@ -1,15 +1,27 @@
 import unittest
+
 from mongoengine.base.datastructures import StrictDict, SemiStrictDict
 
+
 class TestStrictDict(unittest.TestCase):
     def strict_dict_class(self, *args, **kwargs):
         return StrictDict.create(*args, **kwargs)
+
     def setUp(self):
         self.dtype = self.strict_dict_class(("a", "b", "c"))
+
     def test_init(self):
         d = self.dtype(a=1, b=1, c=1)
         self.assertEqual((d.a, d.b, d.c), (1, 1, 1))
 
+    def test_repr(self):
+        d = self.dtype(a=1, b=2, c=3)
+        self.assertEqual(repr(d), '{"a": 1, "b": 2, "c": 3}')
+
+        # make sure quotes are escaped properly
+        d = self.dtype(a='"', b="'", c="")
+        self.assertEqual(repr(d), '{"a": \'"\', "b": "\'", "c": \'\'}')
+
     def test_init_fails_on_nonexisting_attrs(self):
         self.assertRaises(AttributeError, lambda: self.dtype(a=1, b=2, d=3))
 
@@ -38,8 +50,9 @@ class TestStrictDict(unittest.TestCase):
 
     def test_setattr_raises_on_nonexisting_attr(self):
         d = self.dtype()
+
         def _f():
-            d.x=1
+            d.x = 1
         self.assertRaises(AttributeError, _f)
 
     def test_setattr_getattr_special(self):

tests/test_dereference.py

@@ -12,9 +12,13 @@ from mongoengine.context_managers import query_counter
 
 class FieldTest(unittest.TestCase):
 
-    def setUp(self):
+    @classmethod
-        connect(db='mongoenginetest')
+    def setUpClass(cls):
-        self.db = get_db()
+        cls.db = connect(db='mongoenginetest')
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.db.drop_database('mongoenginetest')
 
     def test_list_item_dereference(self):
         """Ensure that DBRef items in ListFields are dereferenced.
@@ -304,6 +308,7 @@ class FieldTest(unittest.TestCase):
 
         User.drop_collection()
         Post.drop_collection()
+        SimpleList.drop_collection()
 
         u1 = User.objects.create(name='u1')
         u2 = User.objects.create(name='u2')
@@ -1026,6 +1031,43 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(type(foo.bar), Bar)
         self.assertEqual(type(foo.baz), Baz)
 
+
+    def test_document_reload_reference_integrity(self):
+        """
+        Ensure reloading a document with multiple similar id
+        in different collections doesn't mix them.
+        """
+        class Topic(Document):
+            id = IntField(primary_key=True)
+        class User(Document):
+            id = IntField(primary_key=True)
+            name = StringField()
+        class Message(Document):
+            id = IntField(primary_key=True)
+            topic = ReferenceField(Topic)
+            author = ReferenceField(User)
+
+        Topic.drop_collection()
+        User.drop_collection()
+        Message.drop_collection()
+
+        # All objects share the same id, but each in a different collection
+        topic = Topic(id=1).save()
+        user = User(id=1, name='user-name').save()
+        Message(id=1, topic=topic, author=user).save()
+
+        concurrent_change_user = User.objects.get(id=1)
+        concurrent_change_user.name = 'new-name'
+        concurrent_change_user.save()
+        self.assertNotEqual(user.name, 'new-name')
+
+        msg = Message.objects.get(id=1)
+        msg.reload()
+        self.assertEqual(msg.topic, topic)
+        self.assertEqual(msg.author, user)
+        self.assertEqual(msg.author.name, 'new-name')
+
+
     def test_list_lookup_not_checked_in_map(self):
         """Ensure we dereference list data correctly
         """

tests/test_django.py

@@ -1,330 +0,0 @@
-import sys
-sys.path[0:0] = [""]
-import unittest
-from nose.plugins.skip import SkipTest
-
-from mongoengine import *
-from mongoengine.django.shortcuts import get_document_or_404
-
-import django
-from django.http import Http404
-from django.template import Context, Template
-from django.conf import settings
-from django.core.paginator import Paginator
-
-settings.configure(
-    USE_TZ=True,
-    INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'),
-    AUTH_USER_MODEL=('mongo_auth.MongoUser'),
-    AUTHENTICATION_BACKENDS = ('mongoengine.django.auth.MongoEngineBackend',)
-)
-
-try:
-    # For Django >= 1.7
-    if hasattr(django, 'setup'):
-        django.setup()
-except RuntimeError:
-    pass
-
-try:
-    from django.contrib.auth import authenticate, get_user_model
-    from mongoengine.django.auth import User
-    from mongoengine.django.mongo_auth.models import (
-        MongoUser,
-        MongoUserManager,
-        get_user_document,
-    )
-    DJ15 = True
-except Exception:
-    DJ15 = False
-from mongoengine.django.sessions import SessionStore, MongoSession
-from mongoengine.django.tests import MongoTestCase
-from datetime import tzinfo, timedelta
-ZERO = timedelta(0)
-
-
-class FixedOffset(tzinfo):
-    """Fixed offset in minutes east from UTC."""
-
-    def __init__(self, offset, name):
-        self.__offset = timedelta(minutes=offset)
-        self.__name = name
-
-    def utcoffset(self, dt):
-        return self.__offset
-
-    def tzname(self, dt):
-        return self.__name
-
-    def dst(self, dt):
-        return ZERO
-
-
-def activate_timezone(tz):
-    """Activate Django timezone support if it is available.
-    """
-    try:
-        from django.utils import timezone
-        timezone.deactivate()
-        timezone.activate(tz)
-    except ImportError:
-        pass
-
-
-class QuerySetTest(unittest.TestCase):
-
-    def setUp(self):
-        connect(db='mongoenginetest')
-
-        class Person(Document):
-            name = StringField()
-            age = IntField()
-        self.Person = Person
-
-    def test_order_by_in_django_template(self):
-        """Ensure that QuerySets are properly ordered in Django template.
-        """
-        self.Person.drop_collection()
-
-        self.Person(name="A", age=20).save()
-        self.Person(name="D", age=10).save()
-        self.Person(name="B", age=40).save()
-        self.Person(name="C", age=30).save()
-
-        t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}")
-
-        d = {"ol": self.Person.objects.order_by('-name')}
-        self.assertEqual(t.render(Context(d)), u'D-10:C-30:B-40:A-20:')
-        d = {"ol": self.Person.objects.order_by('+name')}
-        self.assertEqual(t.render(Context(d)), u'A-20:B-40:C-30:D-10:')
-        d = {"ol": self.Person.objects.order_by('-age')}
-        self.assertEqual(t.render(Context(d)), u'B-40:C-30:A-20:D-10:')
-        d = {"ol": self.Person.objects.order_by('+age')}
-        self.assertEqual(t.render(Context(d)), u'D-10:A-20:C-30:B-40:')
-
-        self.Person.drop_collection()
-
-    def test_q_object_filter_in_template(self):
-
-        self.Person.drop_collection()
-
-        self.Person(name="A", age=20).save()
-        self.Person(name="D", age=10).save()
-        self.Person(name="B", age=40).save()
-        self.Person(name="C", age=30).save()
-
-        t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}")
-
-        d = {"ol": self.Person.objects.filter(Q(age=10) | Q(name="C"))}
-        self.assertEqual(t.render(Context(d)), 'D-10:C-30:')
-
-        # Check double rendering doesn't throw an error
-        self.assertEqual(t.render(Context(d)), 'D-10:C-30:')
-
-    def test_get_document_or_404(self):
-        p = self.Person(name="G404")
-        p.save()
-
-        self.assertRaises(Http404, get_document_or_404, self.Person, pk='1234')
-        self.assertEqual(p, get_document_or_404(self.Person, pk=p.pk))
-
-    def test_pagination(self):
-        """Ensure that Pagination works as expected
-        """
-        class Page(Document):
-            name = StringField()
-
-        Page.drop_collection()
-
-        for i in xrange(1, 11):
-            Page(name=str(i)).save()
-
-        paginator = Paginator(Page.objects.all(), 2)
-
-        t = Template("{% for i in page.object_list  %}{{ i.name }}:{% endfor %}")
-        for p in paginator.page_range:
-            d = {"page": paginator.page(p)}
-            end = p * 2
-            start = end - 1
-            self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end))
-
-    def test_nested_queryset_template_iterator(self):
-        # Try iterating the same queryset twice, nested, in a Django template.
-        names = ['A', 'B', 'C', 'D']
-
-        class CustomUser(Document):
-            name = StringField()
-
-            def __unicode__(self):
-                return self.name
-
-        CustomUser.drop_collection()
-
-        for name in names:
-            CustomUser(name=name).save()
-
-        users = CustomUser.objects.all().order_by('name')
-        template = Template("{% for user in users %}{{ user.name }}{% ifequal forloop.counter 2 %} {% for inner_user in users %}{{ inner_user.name }}{% endfor %} {% endifequal %}{% endfor %}")
-        rendered = template.render(Context({'users': users}))
-        self.assertEqual(rendered, 'AB ABCD CD')
-
-    def test_filter(self):
-        """Ensure that a queryset and filters work as expected
-        """
-
-        class LimitCountQuerySet(QuerySet):
-            def count(self, with_limit_and_skip=True):
-                return super(LimitCountQuerySet, self).count(with_limit_and_skip)
-
-        class Note(Document):
-            meta = dict(queryset_class=LimitCountQuerySet)
-            name = StringField()
-
-        Note.drop_collection()
-
-        for i in xrange(1, 101):
-            Note(name="Note: %s" % i).save()
-
-        # Check the count
-        self.assertEqual(Note.objects.count(), 100)
-
-        # Get the first 10 and confirm
-        notes = Note.objects[:10]
-        self.assertEqual(notes.count(), 10)
-
-        # Test djangos template filters
-        # self.assertEqual(length(notes), 10)
-        t = Template("{{ notes.count }}")
-        c = Context({"notes": notes})
-        self.assertEqual(t.render(c), "10")
-
-        # Test with skip
-        notes = Note.objects.skip(90)
-        self.assertEqual(notes.count(), 10)
-
-        # Test djangos template filters
-        self.assertEqual(notes.count(), 10)
-        t = Template("{{ notes.count }}")
-        c = Context({"notes": notes})
-        self.assertEqual(t.render(c), "10")
-
-        # Test with limit
-        notes = Note.objects.skip(90)
-        self.assertEqual(notes.count(), 10)
-
-        # Test djangos template filters
-        self.assertEqual(notes.count(), 10)
-        t = Template("{{ notes.count }}")
-        c = Context({"notes": notes})
-        self.assertEqual(t.render(c), "10")
-
-        # Test with skip and limit
-        notes = Note.objects.skip(10).limit(10)
-
-        # Test djangos template filters
-        self.assertEqual(notes.count(), 10)
-        t = Template("{{ notes.count }}")
-        c = Context({"notes": notes})
-        self.assertEqual(t.render(c), "10")
-
-
-class _BaseMongoDBSessionTest(unittest.TestCase):
-    backend = SessionStore
-
-    def setUp(self):
-        connect(db='mongoenginetest')
-        MongoSession.drop_collection()
-        super(_BaseMongoDBSessionTest, self).setUp()
-
-    def assertIn(self, first, second, msg=None):
-        self.assertTrue(first in second, msg)
-
-    def assertNotIn(self, first, second, msg=None):
-        self.assertFalse(first in second, msg)
-
-    def test_first_save(self):
-        session = SessionStore()
-        session['test'] = True
-        session.save()
-        self.assertTrue('test' in session)
-
-    def test_session_expiration_tz(self):
-        activate_timezone(FixedOffset(60, 'UTC+1'))
-        # create and save new session
-        session = SessionStore()
-        session.set_expiry(600)  # expire in 600 seconds
-        session['test_expire'] = True
-        session.save()
-        # reload session with key
-        key = session.session_key
-        session = SessionStore(key)
-        self.assertTrue('test_expire' in session, 'Session has expired before it is expected')
-
-
-try:
-    # SessionTestsMixin isn't available for import on django > 1.8a1
-    from django.contrib.sessions.tests import SessionTestsMixin
-
-    class _MongoDBSessionTest(SessionTestsMixin):
-        pass
-
-    class MongoDBSessionTest(_BaseMongoDBSessionTest):
-        pass
-
-except ImportError:
-    class MongoDBSessionTest(_BaseMongoDBSessionTest):
-        pass
-
-
-class MongoAuthTest(unittest.TestCase):
-    user_data = {
-        'username': 'user',
-        'email': 'user@example.com',
-        'password': 'test',
-    }
-
-    def setUp(self):
-        if not DJ15:
-            raise SkipTest('mongo_auth requires Django 1.5')
-        connect(db='mongoenginetest')
-        User.drop_collection()
-        super(MongoAuthTest, self).setUp()
-
-    def test_get_user_model(self):
-        self.assertEqual(get_user_model(), MongoUser)
-
-    def test_get_user_document(self):
-        self.assertEqual(get_user_document(), User)
-
-    def test_user_manager(self):
-        manager = get_user_model()._default_manager
-        self.assertTrue(isinstance(manager, MongoUserManager))
-
-    def test_user_manager_exception(self):
-        manager = get_user_model()._default_manager
-        self.assertRaises(MongoUser.DoesNotExist, manager.get,
-                          username='not found')
-
-    def test_create_user(self):
-        manager = get_user_model()._default_manager
-        user = manager.create_user(**self.user_data)
-        self.assertTrue(isinstance(user, User))
-        db_user = User.objects.get(username='user')
-        self.assertEqual(user.id, db_user.id)
-
-    def test_authenticate(self):
-        get_user_model()._default_manager.create_user(**self.user_data)
-        user = authenticate(username='user', password='fail')
-        self.assertEqual(None, user)
-        user = authenticate(username='user', password='test')
-        db_user = User.objects.get(username='user')
-        self.assertEqual(user.id, db_user.id)
-
-
-class MongoTestCaseTest(MongoTestCase):
-    def test_mongo_test_case(self):
-        self.db.dummy_collection.insert({'collection': 'will be dropped'})
-
-
-if __name__ == '__main__':
-    unittest.main()

tests/test_jinja.py

@@ -1,47 +0,0 @@
-import sys
-sys.path[0:0] = [""]
-
-import unittest
-
-from mongoengine import *
-
-import jinja2
-
-
-class TemplateFilterTest(unittest.TestCase):
-
-    def setUp(self):
-        connect(db='mongoenginetest')
-
-    def test_jinja2(self):
-        env = jinja2.Environment()
-
-        class TestData(Document):
-            title = StringField()
-            description = StringField()
-
-        TestData.drop_collection()
-
-        examples = [('A', '1'),
-                    ('B', '2'),
-                    ('C', '3')]
-
-        for title, description in examples:
-            TestData(title=title, description=description).save()
-
-        tmpl = """
-{%- for record in content -%}
-    {%- if loop.first -%}{ {%- endif -%}
-    "{{ record.title }}": "{{ record.description }}"
-    {%- if loop.last -%} }{%- else -%},{% endif -%}
-{%- endfor -%}
-"""
-        ctx = {'content': TestData.objects}
-        template = env.from_string(tmpl)
-        rendered = template.render(**ctx)
-
-        self.assertEqual('{"A": "1","B": "2","C": "3"}', rendered)
-
-
-if __name__ == '__main__':
-    unittest.main()

tests/test_replicaset_connection.py

@@ -1,17 +1,33 @@
 import sys
+
 sys.path[0:0] = [""]
 import unittest
 
-import pymongo
+from pymongo import ReadPreference
-from pymongo import ReadPreference, ReplicaSetConnection
+
+from mongoengine.python_support import IS_PYMONGO_3
+
+if IS_PYMONGO_3:
+    from pymongo import MongoClient
+    CONN_CLASS = MongoClient
+    READ_PREF = ReadPreference.SECONDARY
+else:
+    from pymongo import ReplicaSetConnection
+    CONN_CLASS = ReplicaSetConnection
+    READ_PREF = ReadPreference.SECONDARY_ONLY
 
 import mongoengine
 from mongoengine import *
-from mongoengine.connection import get_db, get_connection, ConnectionError
+from mongoengine.connection import ConnectionError
 
 
 class ConnectionTest(unittest.TestCase):
 
+    def setUp(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
     def tearDown(self):
         mongoengine.connection._connection_settings = {}
         mongoengine.connection._connections = {}
@@ -22,14 +38,17 @@ class ConnectionTest(unittest.TestCase):
         """
 
         try:
-            conn = connect(db='mongoenginetest', host="mongodb://localhost/mongoenginetest?replicaSet=rs", read_preference=ReadPreference.SECONDARY_ONLY)
+            conn = connect(db='mongoenginetest',
+                           host="mongodb://localhost/mongoenginetest?replicaSet=rs",
+                           read_preference=READ_PREF)
         except ConnectionError, e:
             return
 
-        if not isinstance(conn, ReplicaSetConnection):
+        if not isinstance(conn, CONN_CLASS):
+            # really???
             return
 
-        self.assertEqual(conn.read_preference, ReadPreference.SECONDARY_ONLY)
+        self.assertEqual(conn.read_preference, READ_PREF)
 
 if __name__ == '__main__':
     unittest.main()

tests/test_signals.py

@@ -25,6 +25,8 @@ class SignalTests(unittest.TestCase):
         connect(db='mongoenginetest')
 
         class Author(Document):
+            # Make the id deterministic for easier testing
+            id = SequenceField(primary_key=True)
             name = StringField()
 
             def __unicode__(self):
@@ -33,7 +35,7 @@ class SignalTests(unittest.TestCase):
             @classmethod
             def pre_init(cls, sender, document, *args, **kwargs):
                 signal_output.append('pre_init signal, %s' % cls.__name__)
-                signal_output.append(str(kwargs['values']))
+                signal_output.append(kwargs['values'])
 
             @classmethod
             def post_init(cls, sender, document, **kwargs):
@@ -43,48 +45,55 @@ class SignalTests(unittest.TestCase):
             @classmethod
             def pre_save(cls, sender, document, **kwargs):
                 signal_output.append('pre_save signal, %s' % document)
+                signal_output.append(kwargs)
 
             @classmethod
             def pre_save_post_validation(cls, sender, document, **kwargs):
                 signal_output.append('pre_save_post_validation signal, %s' % document)
-                if 'created' in kwargs:
+                if kwargs.pop('created', False):
-                    if kwargs['created']:
+                    signal_output.append('Is created')
-                        signal_output.append('Is created')
+                else:
-                    else:
+                    signal_output.append('Is updated')
-                        signal_output.append('Is updated')
+                signal_output.append(kwargs)
 
             @classmethod
             def post_save(cls, sender, document, **kwargs):
                 dirty_keys = document._delta()[0].keys() + document._delta()[1].keys()
                 signal_output.append('post_save signal, %s' % document)
                 signal_output.append('post_save dirty keys, %s' % dirty_keys)
-                if 'created' in kwargs:
+                if kwargs.pop('created', False):
-                    if kwargs['created']:
+                    signal_output.append('Is created')
-                        signal_output.append('Is created')
+                else:
-                    else:
+                    signal_output.append('Is updated')
-                        signal_output.append('Is updated')
+                signal_output.append(kwargs)
 
             @classmethod
             def pre_delete(cls, sender, document, **kwargs):
                 signal_output.append('pre_delete signal, %s' % document)
+                signal_output.append(kwargs)
 
             @classmethod
             def post_delete(cls, sender, document, **kwargs):
                 signal_output.append('post_delete signal, %s' % document)
+                signal_output.append(kwargs)
 
             @classmethod
             def pre_bulk_insert(cls, sender, documents, **kwargs):
                 signal_output.append('pre_bulk_insert signal, %s' % documents)
+                signal_output.append(kwargs)
 
             @classmethod
             def post_bulk_insert(cls, sender, documents, **kwargs):
                 signal_output.append('post_bulk_insert signal, %s' % documents)
-                if kwargs.get('loaded', False):
+                if kwargs.pop('loaded', False):
                     signal_output.append('Is loaded')
                 else:
                     signal_output.append('Not loaded')
+                signal_output.append(kwargs)
+
         self.Author = Author
         Author.drop_collection()
+        Author.id.set_next_value(0)
 
         class Another(Document):
 
@@ -96,10 +105,12 @@ class SignalTests(unittest.TestCase):
             @classmethod
             def pre_delete(cls, sender, document, **kwargs):
                 signal_output.append('pre_delete signal, %s' % document)
+                signal_output.append(kwargs)
 
             @classmethod
             def post_delete(cls, sender, document, **kwargs):
                 signal_output.append('post_delete signal, %s' % document)
+                signal_output.append(kwargs)
 
         self.Another = Another
         Another.drop_collection()
@@ -118,6 +129,41 @@ class SignalTests(unittest.TestCase):
         self.ExplicitId = ExplicitId
         ExplicitId.drop_collection()
 
+        class Post(Document):
+            title = StringField()
+            content = StringField()
+            active = BooleanField(default=False)
+
+            def __unicode__(self):
+                return self.title
+
+            @classmethod
+            def pre_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('pre_bulk_insert signal, %s' %
+                                     [(doc, {'active': documents[n].active})
+                                      for n, doc in enumerate(documents)])
+
+                # make changes here, this is just an example -
+                # it could be anything that needs pre-validation or looks-ups before bulk bulk inserting
+                for document in documents:
+                    if not document.active:
+                        document.active = True
+                signal_output.append(kwargs)
+
+            @classmethod
+            def post_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('post_bulk_insert signal, %s' %
+                                     [(doc, {'active': documents[n].active})
+                                      for n, doc in enumerate(documents)])
+                if kwargs.pop('loaded', False):
+                    signal_output.append('Is loaded')
+                else:
+                    signal_output.append('Not loaded')
+                signal_output.append(kwargs)
+
+        self.Post = Post
+        Post.drop_collection()
+
         # Save up the number of connected signals so that we can check at the
         # end that all the signals we register get properly unregistered
         self.pre_signals = (
@@ -147,6 +193,9 @@ class SignalTests(unittest.TestCase):
 
         signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId)
 
+        signals.pre_bulk_insert.connect(Post.pre_bulk_insert, sender=Post)
+        signals.post_bulk_insert.connect(Post.post_bulk_insert, sender=Post)
+
     def tearDown(self):
         signals.pre_init.disconnect(self.Author.pre_init)
         signals.post_init.disconnect(self.Author.post_init)
@@ -163,6 +212,9 @@ class SignalTests(unittest.TestCase):
 
         signals.post_save.disconnect(self.ExplicitId.post_save)
 
+        signals.pre_bulk_insert.disconnect(self.Post.pre_bulk_insert)
+        signals.post_bulk_insert.disconnect(self.Post.post_bulk_insert)
+
         # Check that all our signals got disconnected properly.
         post_signals = (
             len(signals.pre_init.receivers),
@@ -202,63 +254,118 @@ class SignalTests(unittest.TestCase):
 
         self.assertEqual(self.get_signal_output(create_author), [
             "pre_init signal, Author",
-            "{'name': 'Bill Shakespeare'}",
+            {'name': 'Bill Shakespeare'},
             "post_init signal, Bill Shakespeare, document._created = True",
         ])
 
         a1 = self.Author(name='Bill Shakespeare')
         self.assertEqual(self.get_signal_output(a1.save), [
             "pre_save signal, Bill Shakespeare",
+            {},
             "pre_save_post_validation signal, Bill Shakespeare",
             "Is created",
+            {},
             "post_save signal, Bill Shakespeare",
             "post_save dirty keys, ['name']",
-            "Is created"
+            "Is created",
+            {}
         ])
 
         a1.reload()
         a1.name = 'William Shakespeare'
         self.assertEqual(self.get_signal_output(a1.save), [
             "pre_save signal, William Shakespeare",
+            {},
             "pre_save_post_validation signal, William Shakespeare",
             "Is updated",
+            {},
             "post_save signal, William Shakespeare",
             "post_save dirty keys, ['name']",
-            "Is updated"
+            "Is updated",
+            {}
         ])
 
         self.assertEqual(self.get_signal_output(a1.delete), [
             'pre_delete signal, William Shakespeare',
+            {},
             'post_delete signal, William Shakespeare',
+            {}
         ])
 
-        signal_output = self.get_signal_output(load_existing_author)
+        self.assertEqual(self.get_signal_output(load_existing_author), [
-        # test signal_output lines separately, because of random ObjectID after object load
-        self.assertEqual(signal_output[0],
             "pre_init signal, Author",
-        )
+            {'id': 2, 'name': 'Bill Shakespeare'},
-        self.assertEqual(signal_output[2],
+            "post_init signal, Bill Shakespeare, document._created = False"
-            "post_init signal, Bill Shakespeare, document._created = False",
+        ])
-        )
 
-
+        self.assertEqual(self.get_signal_output(bulk_create_author_with_load), [
-        signal_output = self.get_signal_output(bulk_create_author_with_load)
+            'pre_init signal, Author',
-
+            {'name': 'Bill Shakespeare'},
-        # The output of this signal is not entirely deterministic. The reloaded
+            'post_init signal, Bill Shakespeare, document._created = True',
-        # object will have an object ID. Hence, we only check part of the output
+            'pre_bulk_insert signal, [<Author: Bill Shakespeare>]',
-        self.assertEqual(signal_output[3], "pre_bulk_insert signal, [<Author: Bill Shakespeare>]"
+            {},
-        )
+            'pre_init signal, Author',
-        self.assertEqual(signal_output[-2:],
+            {'id': 3, 'name': 'Bill Shakespeare'},
-            ["post_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            'post_init signal, Bill Shakespeare, document._created = False',
-             "Is loaded",])
+            'post_bulk_insert signal, [<Author: Bill Shakespeare>]',
+            'Is loaded',
+            {}
+        ])
 
         self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [
             "pre_init signal, Author",
-            "{'name': 'Bill Shakespeare'}",
+            {'name': 'Bill Shakespeare'},
             "post_init signal, Bill Shakespeare, document._created = True",
             "pre_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            {},
             "post_bulk_insert signal, [<Author: Bill Shakespeare>]",
             "Not loaded",
+            {}
+        ])
+
+    def test_signal_kwargs(self):
+        """ Make sure signal_kwargs is passed to signals calls. """
+
+        def live_and_let_die():
+            a = self.Author(name='Bill Shakespeare')
+            a.save(signal_kwargs={'live': True, 'die': False})
+            a.delete(signal_kwargs={'live': False, 'die': True})
+
+        self.assertEqual(self.get_signal_output(live_and_let_die), [
+            "pre_init signal, Author",
+            {'name': 'Bill Shakespeare'},
+            "post_init signal, Bill Shakespeare, document._created = True",
+            "pre_save signal, Bill Shakespeare",
+            {'die': False, 'live': True},
+            "pre_save_post_validation signal, Bill Shakespeare",
+            "Is created",
+            {'die': False, 'live': True},
+            "post_save signal, Bill Shakespeare",
+            "post_save dirty keys, ['name']",
+            "Is created",
+            {'die': False, 'live': True},
+            'pre_delete signal, Bill Shakespeare',
+            {'die': True, 'live': False},
+            'post_delete signal, Bill Shakespeare',
+            {'die': True, 'live': False}
+        ])
+
+        def bulk_create_author():
+            a1 = self.Author(name='Bill Shakespeare')
+            self.Author.objects.insert([a1], signal_kwargs={'key': True})
+
+        self.assertEqual(self.get_signal_output(bulk_create_author), [
+            'pre_init signal, Author',
+            {'name': 'Bill Shakespeare'},
+            'post_init signal, Bill Shakespeare, document._created = True',
+            'pre_bulk_insert signal, [<Author: Bill Shakespeare>]',
+            {'key': True},
+            'pre_init signal, Author',
+            {'id': 2, 'name': 'Bill Shakespeare'},
+            'post_init signal, Bill Shakespeare, document._created = False',
+            'post_bulk_insert signal, [<Author: Bill Shakespeare>]',
+            'Is loaded',
+            {'key': True}
         ])
 
     def test_queryset_delete_signals(self):
@@ -267,7 +374,9 @@ class SignalTests(unittest.TestCase):
         self.Another(name='Bill Shakespeare').save()
         self.assertEqual(self.get_signal_output(self.Another.objects.delete), [
             'pre_delete signal, Bill Shakespeare',
+            {},
             'post_delete signal, Bill Shakespeare',
+            {}
         ])
 
     def test_signals_with_explicit_doc_ids(self):
@@ -279,5 +388,50 @@ class SignalTests(unittest.TestCase):
         # second time, it must be an update
         self.assertEqual(self.get_signal_output(ei.save), ['Is updated'])
 
+    def test_signals_with_switch_collection(self):
+        ei = self.ExplicitId(id=123)
+        ei.switch_collection("explicit__1")
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+        ei.switch_collection("explicit__1")
+        self.assertEqual(self.get_signal_output(ei.save), ['Is updated'])
+
+        ei.switch_collection("explicit__1", keep_created=False)
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+        ei.switch_collection("explicit__1", keep_created=False)
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+
+    def test_signals_with_switch_db(self):
+        connect('mongoenginetest')
+        register_connection('testdb-1', 'mongoenginetest2')
+
+        ei = self.ExplicitId(id=123)
+        ei.switch_db("testdb-1")
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+        ei.switch_db("testdb-1")
+        self.assertEqual(self.get_signal_output(ei.save), ['Is updated'])
+
+        ei.switch_db("testdb-1", keep_created=False)
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+        ei.switch_db("testdb-1", keep_created=False)
+        self.assertEqual(self.get_signal_output(ei.save), ['Is created'])
+
+    def test_signals_bulk_insert(self):
+        def bulk_set_active_post():
+            posts = [
+                self.Post(title='Post 1'),
+                self.Post(title='Post 2'),
+                self.Post(title='Post 3')
+            ]
+            self.Post.objects.insert(posts)
+
+        results = self.get_signal_output(bulk_set_active_post)
+        self.assertEqual(results, [
+            "pre_bulk_insert signal, [(<Post: Post 1>, {'active': False}), (<Post: Post 2>, {'active': False}), (<Post: Post 3>, {'active': False})]",
+            {},
+            "post_bulk_insert signal, [(<Post: Post 1>, {'active': True}), (<Post: Post 2>, {'active': True}), (<Post: Post 3>, {'active': True})]",
+            'Is loaded',
+            {}
+        ])
+
 if __name__ == '__main__':
     unittest.main()

tox.ini

@@ -0,0 +1,22 @@
+[tox]
+envlist = {py26,py27,py33,py34,py35,pypy,pypy3}-{mg27,mg28},flake8
+
+[testenv]
+commands =
+    python setup.py nosetests {posargs}
+deps =
+    nose
+    mg27: PyMongo<2.8
+    mg28: PyMongo>=2.8,<3.0
+    mg30: PyMongo>=3.0
+    mgdev: https://github.com/mongodb/mongo-python-driver/tarball/master
+setenv =
+    PYTHON_EGG_CACHE = {envdir}/python-eggs
+passenv = windir
+
+[testenv:flake8]
+deps =
+    flake8
+    flake8-import-order
+commands =
+   flake8