Updated setup.py

Remove database name necessity in uri connection schema

.travis.yml

@@ -11,12 +11,14 @@ env:
   - PYMONGO=dev DJANGO=1.4.2
   - PYMONGO=2.5 DJANGO=1.5.1
   - PYMONGO=2.5 DJANGO=1.4.2
+  - PYMONGO=3.2 DJANGO=1.5.1
+  - PYMONGO=3.3 DJANGO=1.5.1
 install:
     - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
     - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
     - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
+    - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b
     - python setup.py install
 script:
     - python setup.py test

AUTHORS

@@ -16,8 +16,6 @@ Dervived from the git logs, inevitably incomplete but all of whom and others
 have submitted patches, reported bugs and generally helped make MongoEngine
 that much better:
 
- * Harry Marr
- * Ross Lawley
  * blackbrrr
  * Florian Schlachter
  * Vincent Driessen
@@ -77,7 +75,7 @@ that much better:
  * Adam Parrish
  * jpfarias
  * jonrscott
- * Alice Zoë Bevan-McGregor
+ * Alice Zoë Bevan-McGregor (https://github.com/amcgregor/)
  * Stephen Young
  * tkloc
  * aid
@@ -161,3 +159,23 @@ that much better:
  * Jin Zhang
  * Daniel Axtens
  * Leo-Naeka
+ * Ryan Witt (https://github.com/ryanwitt)
+ * Jiequan (https://github.com/Jiequan)
+ * hensom (https://github.com/hensom)
+ * zhy0216 (https://github.com/zhy0216)
+ * istinspring (https://github.com/istinspring)
+ * Massimo Santini (https://github.com/mapio)
+ * Nigel McNie (https://github.com/nigelmcnie)
+ * ygbourhis (https://github.com/ygbourhis)
+ * Bob Dickinson (https://github.com/BobDickinson)
+ * Michael Bartnett (https://github.com/michaelbartnett)
+ * Alon Horev (https://github.com/alonho)
+ * Kelvin Hammond (https://github.com/kelvinhammond)
+ * Jatin- (https://github.com/jatin-)
+ * Paul Uithol (https://github.com/PaulUithol)
+ * Thom Knowles (https://github.com/fleat)
+ * Paul (https://github.com/squamous)
+ * Olivier Cortès (https://github.com/Karmak23)
+ * crazyzubr (https://github.com/crazyzubr)
+ * FrankSomething (https://github.com/FrankSomething)
+ * Alexandr Morozov (https://github.com/LK4D4)

docs/_themes/nature/static/nature.css_t

@@ -2,11 +2,15 @@
  * Sphinx stylesheet -- default theme
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
- 
+
 @import url("basic.css");
- 
+
+#changelog p.first {margin-bottom: 0 !important;}
+#changelog p {margin-top: 0 !important;
+              margin-bottom: 0 !important;}
+
 /* -- page layout ----------------------------------------------------------- */
- 
+
 body {
     font-family: Arial, sans-serif;
     font-size: 100%;
@@ -28,18 +32,18 @@ div.bodywrapper {
 hr{
     border: 1px solid #B1B4B6;
 }
- 
+
 div.document {
     background-color: #eee;
 }
- 
+
 div.body {
     background-color: #ffffff;
     color: #3E4349;
     padding: 0 30px 30px 30px;
     font-size: 0.8em;
 }
- 
+
 div.footer {
     color: #555;
     width: 100%;
@@ -47,12 +51,12 @@ div.footer {
     text-align: center;
     font-size: 75%;
 }
- 
+
 div.footer a {
     color: #444;
     text-decoration: underline;
 }
- 
+
 div.related {
     background-color: #6BA81E;
     line-height: 32px;
@@ -60,11 +64,11 @@ div.related {
     text-shadow: 0px 1px 0 #444;
     font-size: 0.80em;
 }
- 
+
 div.related a {
     color: #E2F3CC;
 }
- 
+
 div.sphinxsidebar {
     font-size: 0.75em;
     line-height: 1.5em;
@@ -73,7 +77,7 @@ div.sphinxsidebar {
 div.sphinxsidebarwrapper{
     padding: 20px 0;
 }
- 
+
 div.sphinxsidebar h3,
 div.sphinxsidebar h4 {
     font-family: Arial, sans-serif;
@@ -89,30 +93,30 @@ div.sphinxsidebar h4 {
 div.sphinxsidebar h4{
     font-size: 1.1em;
 }
- 
+
 div.sphinxsidebar h3 a {
     color: #444;
 }
- 
+
- 
+
 div.sphinxsidebar p {
     color: #888;
     padding: 5px 20px;
 }
- 
+
 div.sphinxsidebar p.topless {
 }
- 
+
 div.sphinxsidebar ul {
     margin: 10px 20px;
     padding: 0;
     color: #000;
 }
- 
+
 div.sphinxsidebar a {
     color: #444;
 }
- 
+
 div.sphinxsidebar input {
     border: 1px solid #ccc;
     font-family: sans-serif;
@@ -122,19 +126,19 @@ div.sphinxsidebar input {
 div.sphinxsidebar input[type=text]{
     margin-left: 20px;
 }
- 
+
 /* -- body styles ----------------------------------------------------------- */
- 
+
 a {
     color: #005B81;
     text-decoration: none;
 }
- 
+
 a:hover {
     color: #E32E00;
     text-decoration: underline;
 }
- 
+
 div.body h1,
 div.body h2,
 div.body h3,
@@ -149,30 +153,30 @@ div.body h6 {
     padding: 5px 0 5px 10px;
     text-shadow: 0px 1px 0 white
 }
- 
+
 div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; }
 div.body h2 { font-size: 150%; background-color: #C8D5E3; }
 div.body h3 { font-size: 120%; background-color: #D8DEE3; }
 div.body h4 { font-size: 110%; background-color: #D8DEE3; }
 div.body h5 { font-size: 100%; background-color: #D8DEE3; }
 div.body h6 { font-size: 100%; background-color: #D8DEE3; }
- 
+
 a.headerlink {
     color: #c60f0f;
     font-size: 0.8em;
     padding: 0 4px 0 4px;
     text-decoration: none;
 }
- 
+
 a.headerlink:hover {
     background-color: #c60f0f;
     color: white;
 }
- 
+
 div.body p, div.body dd, div.body li {
     line-height: 1.5em;
 }
- 
+
 div.admonition p.admonition-title + p {
     display: inline;
 }
@@ -185,29 +189,29 @@ div.note {
     background-color: #eee;
     border: 1px solid #ccc;
 }
- 
+
 div.seealso {
     background-color: #ffc;
     border: 1px solid #ff6;
 }
- 
+
 div.topic {
     background-color: #eee;
 }
- 
+
 div.warning {
     background-color: #ffe4e4;
     border: 1px solid #f66;
 }
- 
+
 p.admonition-title {
     display: inline;
 }
- 
+
 p.admonition-title:after {
     content: ":";
 }
- 
+
 pre {
     padding: 10px;
     background-color: White;
@@ -219,7 +223,7 @@ pre {
     -webkit-box-shadow: 1px 1px 1px #d8d8d8;
     -moz-box-shadow: 1px 1px 1px #d8d8d8;
 }
- 
+
 tt {
     background-color: #ecf0f3;
     color: #222;

docs/apireference.rst

@@ -44,16 +44,26 @@ Context Managers
 Querying
 ========
 
-.. autoclass:: mongoengine.queryset.QuerySet
+.. automodule:: mongoengine.queryset
-   :members:
+    :synopsis: Queryset level operations
 
-   .. automethod:: mongoengine.queryset.QuerySet.__call__
+    .. autoclass:: mongoengine.queryset.QuerySet
+      :members:
+      :inherited-members:
 
-.. autofunction:: mongoengine.queryset.queryset_manager
+      .. automethod:: QuerySet.__call__
+
+    .. autoclass:: mongoengine.queryset.QuerySetNoCache
+      :members:
+
+       .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__
+
+    .. autofunction:: mongoengine.queryset.queryset_manager
 
 Fields
 ======
 
+.. autoclass:: mongoengine.base.fields.BaseField
 .. autoclass:: mongoengine.fields.StringField
 .. autoclass:: mongoengine.fields.URLField
 .. autoclass:: mongoengine.fields.EmailField
@@ -87,3 +97,8 @@ Fields
 .. autoclass:: mongoengine.fields.GridFSProxy
 .. autoclass:: mongoengine.fields.ImageGridFsProxy
 .. autoclass:: mongoengine.fields.ImproperlyConfigured
+
+Misc
+====
+
+.. autofunction:: mongoengine.common._import_class

docs/changelog.rst

@@ -2,6 +2,76 @@
 Changelog
 =========
 
+Changes in 0.8.4
+================
+- Remove database name necessity in uri connection schema (#452)
+- Fixed "$pull" semantics for nested ListFields (#447)
+- Allow fields to be named the same as query operators (#445)
+- Updated field filter logic - can now exclude subclass fields (#443)
+- Fixed dereference issue with embedded listfield referencefields (#439)
+- Fixed slice when using inheritance causing fields to be excluded (#437)
+- Fixed ._get_db() attribute after a Document.switch_db() (#441)
+- Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449)
+- Handle dynamic fieldnames that look like digits (#434)
+- Added get_user_document and improve mongo_auth module (#423)
+- Added str representation of GridFSProxy (#424)
+- Update transform to handle docs erroneously passed to unset (#416)
+- Fixed indexing - turn off _cls (#414)
+- Fixed dereference threading issue in ComplexField.__get__ (#412)
+- Fixed QuerySetNoCache.count() caching (#410)
+- Don't follow references in _get_changed_fields (#422, #417)
+- Allow args and kwargs to be passed through to_json (#420)
+
+Changes in 0.8.3
+================
+- Fixed EmbeddedDocuments with `id` also storing `_id` (#402)
+- Added get_proxy_object helper to filefields (#391)
+- Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365)
+- Fixed sum and average mapreduce dot notation support (#375, #376, #393)
+- Fixed as_pymongo to return the id (#386)
+- Document.select_related() now respects `db_alias` (#377)
+- Reload uses shard_key if applicable (#384)
+- Dynamic fields are ordered based on creation and stored in _fields_ordered (#396)
+
+  **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3
+
+- Fixed pickling dynamic documents `_dynamic_fields` (#387)
+- Fixed ListField setslice and delslice dirty tracking (#390)
+- Added Django 1.5 PY3 support (#392)
+- Added match ($elemMatch) support for EmbeddedDocuments (#379)
+- Fixed weakref being valid after reload (#374)
+- Fixed queryset.get() respecting no_dereference (#373)
+- Added full_result kwarg to update (#380)
+
+
+
+Changes in 0.8.2
+================
+- Added compare_indexes helper (#361)
+- Fixed cascading saves which weren't turned off as planned (#291)
+- Fixed Datastructures so instances are a Document or EmbeddedDocument (#363)
+- Improved cascading saves write performance (#361)
+- Fixed ambiguity and differing behaviour regarding field defaults (#349)
+- ImageFields now include PIL error messages if invalid error (#353)
+- Added lock when calling doc.Delete() for when signals have no sender (#350)
+- Reload forces read preference to be PRIMARY (#355)
+- Querysets are now lest restrictive when querying duplicate fields (#332, #333)
+- FileField now honouring db_alias (#341)
+- Removed customised __set__ change tracking in ComplexBaseField (#344)
+- Removed unused var in _get_changed_fields (#347)
+- Added pre_save_post_validation signal (#345)
+- DateTimeField now auto converts valid datetime isostrings into dates (#343)
+- DateTimeField now uses dateutil for parsing if available (#343)
+- Fixed Doc.objects(read_preference=X) not setting read preference (#352)
+- Django session ttl index expiry fixed (#329)
+- Fixed pickle.loads (#342)
+- Documentation fixes
+
+Changes in 0.8.1
+================
+- Fixed Python 2.6 django auth importlib issue (#326)
+- Fixed pickle unsaved document regression (#327)
+
 Changes in 0.8.0
 ================
 - Fixed querying ReferenceField custom_id (#317)

docs/django.rst

@@ -27,9 +27,9 @@ MongoEngine includes a Django authentication backend, which uses MongoDB. The
 :class:`~mongoengine.Document`, but implements most of the methods and
 attributes that the standard Django :class:`User` model does - so the two are
 moderately compatible. Using this backend will allow you to store users in
-MongoDB but still use many of the Django authentication infrastucture (such as
+MongoDB but still use many of the Django authentication infrastructure (such as
 the :func:`login_required` decorator and the :func:`authenticate` function). To
-enable the MongoEngine auth backend, add the following to you **settings.py**
+enable the MongoEngine auth backend, add the following to your **settings.py**
 file::
 
     AUTHENTICATION_BACKENDS = (
@@ -45,8 +45,8 @@ The :mod:`~mongoengine.django.auth` module also contains a
 Custom User model
 =================
 Django 1.5 introduced `Custom user Models
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/#auth-custom-user>`
+<https://docs.djangoproject.com/en/dev/topics/auth/customizing/#auth-custom-user>`_
-which can be used as an alternative the Mongoengine authentication backend.
+which can be used as an alternative to the MongoEngine authentication backend.
 
 The main advantage of this option is that other components relying on
 :mod:`django.contrib.auth` and supporting the new swappable user model are more
@@ -74,7 +74,7 @@ An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the
 The custom :class:`User` must be a :class:`~mongoengine.Document` class, but
 otherwise has the same requirements as a standard custom user model,
 as specified in the `Django Documentation
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/>`.
+<https://docs.djangoproject.com/en/dev/topics/auth/customizing/>`_.
 In particular, the custom class must define :attr:`USERNAME_FIELD` and
 :attr:`REQUIRED_FIELDS` attributes.
 
@@ -82,16 +82,16 @@ Sessions
 ========
 Django allows the use of different backend stores for its sessions. MongoEngine
 provides a MongoDB-based session backend for Django, which allows you to use
-sessions in you Django application with just MongoDB. To enable the MongoEngine
+sessions in your Django application with just MongoDB. To enable the MongoEngine
 session backend, ensure that your settings module has
 ``'django.contrib.sessions.middleware.SessionMiddleware'`` in the
 ``MIDDLEWARE_CLASSES`` field  and ``'django.contrib.sessions'`` in your
 ``INSTALLED_APPS``. From there, all you need to do is add the following line
-into you settings module::
+into your settings module::
 
     SESSION_ENGINE = 'mongoengine.django.sessions'
 
-Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports  `mongodb TTL
+Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesn't delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports  `mongodb TTL
 <http://docs.mongodb.org/manual/tutorial/expire-data/>`_.
 
 .. versionadded:: 0.2.1
@@ -128,7 +128,7 @@ appended to the filename until the generated filename doesn't exist. The
     >>> fs.listdir()
     ([], [u'hello.txt'])
 
-All files will be saved and retrieved in GridFS via the :class::`FileDocument`
+All files will be saved and retrieved in GridFS via the :class:`FileDocument`
 document, allowing easy access to the files without the GridFSStorage
 backend.::
 
@@ -137,3 +137,36 @@ backend.::
     [<FileDocument: FileDocument object>]
 
 .. versionadded:: 0.4
+
+Shortcuts
+=========
+Inspired by the `Django shortcut get_object_or_404
+<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-object-or-404>`_,
+the :func:`~mongoengine.django.shortcuts.get_document_or_404` method returns 
+a document or raises an Http404 exception if the document does not exist::
+
+    from mongoengine.django.shortcuts import get_document_or_404
+    
+    admin_user = get_document_or_404(User, username='root')
+
+The first argument may be a Document or QuerySet object. All other passed arguments
+and keyword arguments are used in the query::
+
+    foo_email = get_document_or_404(User.objects.only('email'), username='foo', is_active=True).email
+
+.. note:: Like with :func:`get`, a MultipleObjectsReturned will be raised if more than one
+    object is found.
+
+
+Also inspired by the `Django shortcut get_list_or_404
+<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-list-or-404>`_,
+the :func:`~mongoengine.django.shortcuts.get_list_or_404` method returns a list of
+documents or raises an Http404 exception if the list is empty::
+
+    from mongoengine.django.shortcuts import get_list_or_404
+    
+    active_users = get_list_or_404(User, is_active=True)
+
+The first argument may be a Document or QuerySet object. All other passed
+arguments and keyword arguments are used to filter the query.
+

docs/guide/connecting.rst

@@ -23,12 +23,15 @@ arguments should be provided::
 
     connect('project1', username='webapp', password='pwd123')
 
-Uri style connections are also supported as long as you include the database
+Uri style connections are also supported - just supply the uri as
-name - just supply the uri as the :attr:`host` to
+the :attr:`host` to
 :func:`~mongoengine.connect`::
 
     connect('project1', host='mongodb://localhost/database_name')
 
+Note that database name from uri has priority over name
+in ::func:`~mongoengine.connect`
+
 ReplicaSets
 ===========
 
@@ -36,7 +39,7 @@ MongoEngine supports :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetCl
 to use them please use a URI style connection and provide the `replicaSet` name in the
 connection kwargs.
 
-Read preferences are supported throught the connection or via individual
+Read preferences are supported through the connection or via individual
 queries by passing the read_preference ::
 
     Bar.objects().read_preference(ReadPreference.PRIMARY)
@@ -83,7 +86,7 @@ reasons.
 
 The :class:`~mongoengine.context_managers.switch_db` context manager allows
 you to change the database alias for a given class allowing quick and easy
-access to the same User document across databases.eg ::
+access to the same User document across databases::
 
         from mongoengine.context_managers import switch_db
 

docs/guide/defining-documents.rst

@@ -54,7 +54,7 @@ be saved ::
 
    There is one caveat on Dynamic Documents: fields cannot start with `_`
 
-Dynamic fields are stored in alphabetical order *after* any declared fields.
+Dynamic fields are stored in creation order *after* any declared fields.
 
 Fields
 ======
@@ -100,9 +100,6 @@ arguments can be set on all fields:
 :attr:`db_field` (Default: None)
     The MongoDB field name.
 
-:attr:`name` (Default: None)
-    The mongoengine field name.
-
 :attr:`required` (Default: False)
     If set to True and the field is not set on the document instance, a
     :class:`~mongoengine.ValidationError` will be raised when the document is
@@ -129,6 +126,7 @@ arguments can be set on all fields:
             # instead to just an object
             values = ListField(IntField(), default=[1,2,3])
 
+    .. note:: Unsetting a field with a default value will revert back to the default.
 
 :attr:`unique` (Default: False)
     When True, no documents in the collection will have the same value for this
@@ -403,7 +401,7 @@ either a single field name, or a list or tuple of field names::
 Skipping Document validation on save
 ------------------------------------
 You can also skip the whole document validation process by setting
-``validate=False`` when caling the :meth:`~mongoengine.document.Document.save`
+``validate=False`` when calling the :meth:`~mongoengine.document.Document.save`
 method::
 
     class Recipient(Document):
@@ -444,6 +442,8 @@ The following example shows a :class:`Log` document that will be limited to
         ip_address = StringField()
         meta = {'max_documents': 1000, 'max_size': 2000000}
 
+.. defining-indexes_
+
 Indexes
 =======
 
@@ -452,8 +452,8 @@ by creating a list of index specifications called :attr:`indexes` in the
 :attr:`~mongoengine.Document.meta` dictionary, where an index specification may
 either be a single field name, a tuple containing multiple field names, or a
 dictionary containing a full index definition. A direction may be specified on
-fields by prefixing the field name with a **+** or a **-** sign. Note that
+fields by prefixing the field name with a **+** (for ascending) or a **-** sign
-direction only matters on multi-field indexes. ::
+(for descending). Note that direction only matters on multi-field indexes. ::
 
     class Page(Document):
         title = StringField()
@@ -487,6 +487,35 @@ If a dictionary is passed then the following options are available:
 
     Inheritance adds extra fields indices see: :ref:`document-inheritance`.
 
+Global index default options
+----------------------------
+
+There are a few top level defaults for all indexes that can be set::
+
+    class Page(Document):
+        title = StringField()
+        rating = StringField()
+        meta = {
+            'index_options': {},
+            'index_background': True,
+            'index_drop_dups': True,
+            'index_cls': False
+        }
+
+
+:attr:`index_options` (Optional)
+    Set any default index options - see the `full options list <http://docs.mongodb.org/manual/reference/method/db.collection.ensureIndex/#db.collection.ensureIndex>`_
+
+:attr:`index_background` (Optional)
+    Set the default value for if an index should be indexed in the background
+
+:attr:`index_drop_dups` (Optional)
+    Set the default value for if an index should drop duplicates
+
+:attr:`index_cls` (Optional)
+    A way to turn off a specific index for _cls.
+
+
 Compound Indexes and Indexing sub documents
 -------------------------------------------
 
@@ -499,7 +528,6 @@ in this case use 'dot' notation to identify the value to index eg: `rank.title`
 Geospatial indexes
 ------------------
 
-
 The best geo index for mongodb is the new "2dsphere", which has an improved
 spherical model and provides better performance and more options when querying.
 The following fields will explicitly add a "2dsphere" index:
@@ -561,6 +589,19 @@ documentation for more information.  A common usecase might be session data::
             ]
         }
 
+.. warning:: TTL indexes happen on the MongoDB server and not in the application
+    code, therefore no signals will be fired on document deletion.
+    If you need signals to be fired on deletion, then you must handle the
+    deletion of Documents in your application code.
+
+Comparing Indexes
+-----------------
+
+Use :func:`mongoengine.Document.compare_indexes` to compare actual indexes in
+the database to those that your document definitions define.  This is useful
+for maintenance purposes and ensuring you have the correct indexes for your
+schema.
+
 Ordering
 ========
 A default ordering can be specified for your
@@ -648,7 +689,6 @@ document.::
 .. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults
           to False, meaning you must set it to True to use inheritance.
 
-
 Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and
@@ -668,3 +708,25 @@ defining all possible field types.
 
 If you use :class:`~mongoengine.Document` and the database contains data that
 isn't defined then that data will be stored in the `document._data` dictionary.
+
+Abstract classes
+================
+
+If you want to add some extra functionality to a group of Document classes but
+you don't need or want the overhead of inheritance you can use the
+:attr:`abstract` attribute of :attr:`-mongoengine.Document.meta`.
+This won't turn on :ref:`document-inheritance` but will allow you to keep your
+code DRY::
+
+        class BaseDocument(Document):
+            meta = {
+                'abstract': True,
+            }
+            def check_permissions(self):
+                ...
+
+        class User(BaseDocument):
+           ...
+
+Now the User class will have access to the inherited `check_permissions` method
+and won't store any of the extra `_cls` information.

docs/guide/querying.rst

@@ -15,11 +15,10 @@ fetch documents from the database::
 
 .. note::
 
-   Once the iteration finishes (when :class:`StopIteration` is raised),
+    As of MongoEngine 0.8 the querysets utilise a local cache.  So iterating
-   :meth:`~mongoengine.queryset.QuerySet.rewind` will be called so that the
+    it multiple times will only cause a single query.  If this is not the
-   :class:`~mongoengine.queryset.QuerySet` may be iterated over again. The
+    desired behavour you can call :class:`~mongoengine.QuerySet.no_cache`
-   results of the first iteration are *not* cached, so the database will be hit
+    (version **0.8.3+**) to return a non-caching queryset.
-   each time the :class:`~mongoengine.queryset.QuerySet` is iterated over.
 
 Filtering queries
 =================
@@ -498,7 +497,6 @@ that you may use with these methods:
 * ``unset`` -- delete a particular value (since MongoDB v1.3+)
 * ``inc`` -- increment a value by a given amount
 * ``dec`` -- decrement a value by a given amount
-* ``pop`` -- remove the last item from a list
 * ``push`` -- append a value to a list
 * ``push_all`` -- append several values to a list
 * ``pop`` -- remove the first or last element of a list

docs/guide/signals.rst

@@ -1,5 +1,6 @@
 .. _signals:
 
+=======
 Signals
 =======
 
@@ -7,32 +8,95 @@ Signals
 
 .. note::
 
-  Signal support is provided by the excellent `blinker`_ library and
+  Signal support is provided by the excellent `blinker`_ library. If you wish
-  will gracefully fall back if it is not available.
+  to enable signal support this library must be installed, though it is not
+  required for MongoEngine to function.
 
+Overview
+--------
 
-The following document signals exist in MongoEngine and are pretty self-explanatory:
+Signals are found within the `mongoengine.signals` module.  Unless
+specified signals receive no additional arguments beyond the `sender` class and
+`document` instance.  Post-signals are only called if there were no exceptions
+raised during the processing of their related function.
 
-  * `mongoengine.signals.pre_init`
+Available signals include:
-  * `mongoengine.signals.post_init`
-  * `mongoengine.signals.pre_save`
-  * `mongoengine.signals.post_save`
-  * `mongoengine.signals.pre_delete`
-  * `mongoengine.signals.post_delete`
-  * `mongoengine.signals.pre_bulk_insert`
-  * `mongoengine.signals.post_bulk_insert`
 
-Example usage::
+`pre_init`
+  Called during the creation of a new :class:`~mongoengine.Document` or
+  :class:`~mongoengine.EmbeddedDocument` instance, after the constructor
+  arguments have been collected but before any additional processing has been
+  done to them.  (I.e. assignment of default values.)  Handlers for this signal
+  are passed the dictionary of arguments using the `values` keyword argument
+  and may modify this dictionary prior to returning.
+
+`post_init`
+  Called after all processing of a new :class:`~mongoengine.Document` or
+  :class:`~mongoengine.EmbeddedDocument` instance has been completed.
+
+`pre_save`
+  Called within :meth:`~mongoengine.document.Document.save` prior to performing
+  any actions.
+
+`pre_save_post_validation`
+  Called within :meth:`~mongoengine.document.Document.save` after validation
+  has taken place but before saving.
+
+`post_save`
+  Called within :meth:`~mongoengine.document.Document.save` after all actions
+  (validation, insert/update, cascades, clearing dirty flags) have completed
+  successfully.  Passed the additional boolean keyword argument `created` to
+  indicate if the save was an insert or an update.
+
+`pre_delete`
+  Called within :meth:`~mongoengine.document.Document.delete` prior to
+  attempting the delete operation.
+
+`post_delete`
+  Called within :meth:`~mongoengine.document.Document.delete` upon successful
+  deletion of the record.
+
+`pre_bulk_insert`
+  Called after validation of the documents to insert, but prior to any data
+  being written. In this case, the `document` argument is replaced by a
+  `documents` argument representing the list of documents being inserted.
+
+`post_bulk_insert`
+  Called after a successful bulk insert operation.  As per `pre_bulk_insert`,
+  the `document` argument is omitted and replaced with a `documents` argument.
+  An additional boolean argument, `loaded`, identifies the contents of
+  `documents` as either :class:`~mongoengine.Document` instances when `True` or
+  simply a list of primary key values for the inserted records if `False`.
+
+Attaching Events
+----------------
+
+After writing a handler function like the following::
+
+    import logging
+    from datetime import datetime
 
     from mongoengine import *
     from mongoengine import signals
 
+    def update_modified(sender, document):
+        document.modified = datetime.utcnow()
+
+You attach the event handler to your :class:`~mongoengine.Document` or
+:class:`~mongoengine.EmbeddedDocument` subclass::
+
+    class Record(Document):
+        modified = DateTimeField()
+
+    signals.pre_save.connect(update_modified)
+
+While this is not the most elaborate document model, it does demonstrate the
+concepts involved.  As a more complete demonstration you can also define your
+handlers within your subclass::
+
     class Author(Document):
         name = StringField()
 
-        def __unicode__(self):
-            return self.name
-
         @classmethod
         def pre_save(cls, sender, document, **kwargs):
             logging.debug("Pre Save: %s" % document.name)
@@ -49,12 +113,40 @@ Example usage::
     signals.pre_save.connect(Author.pre_save, sender=Author)
     signals.post_save.connect(Author.post_save, sender=Author)
 
+Finally, you can also use this small decorator to quickly create a number of
+signals and attach them to your :class:`~mongoengine.Document` or
+:class:`~mongoengine.EmbeddedDocument` subclasses as class decorators::
 
-ReferenceFields and signals
+    def handler(event):
+        """Signal decorator to allow use of callback functions as class decorators."""
+
+        def decorator(fn):
+            def apply(cls):
+                event.connect(fn, sender=cls)
+                return cls
+
+            fn.apply = apply
+            return fn
+
+        return decorator
+
+Using the first example of updating a modification time the code is now much
+cleaner looking while still allowing manual execution of the callback::
+
+    @handler(signals.pre_save)
+    def update_modified(sender, document):
+        document.modified = datetime.utcnow()
+
+    @update_modified.apply
+    class Record(Document):
+        modified = DateTimeField()
+
+
+ReferenceFields and Signals
 ---------------------------
 
 Currently `reverse_delete_rules` do not trigger signals on the other part of
-the relationship.  If this is required you must manually handled the
+the relationship.  If this is required you must manually handle the
 reverse deletion.
 
 .. _blinker: http://pypi.python.org/pypi/blinker

docs/tutorial.rst

@@ -298,5 +298,5 @@ Learning more about mongoengine
 -------------------------------
 
 If you got this far you've made a great start, so well done!  The next step on
-your mongoengine journey is the `full user guide <guide/index>`_, where you
+your mongoengine journey is the `full user guide <guide/index.html>`_, where you
-can learn indepth about how to use mongoengine and mongodb.
+can learn indepth about how to use mongoengine and mongodb.

docs/upgrade.rst

@@ -2,12 +2,22 @@
 Upgrading
 #########
 
+
+0.8.2 to 0.8.3
+**************
+
+Minor change that may impact users:
+
+DynamicDocument fields are now stored in creation order after any declared
+fields.  Previously they were stored alphabetically.
+
+
 0.7 to 0.8
 **********
 
 There have been numerous backwards breaking changes in 0.8.  The reasons for
-these are ensure that MongoEngine has sane defaults going forward and
+these are to ensure that MongoEngine has sane defaults going forward and that it
-performs the best it can out the box.  Where possible there have been
+performs the best it can out of the box.  Where possible there have been
 FutureWarnings to help get you ready for the change, but that hasn't been
 possible for the whole of the release.
 
@@ -61,7 +71,7 @@ inherited classes like so: ::
 Document Definition
 -------------------
 
-The default for inheritance has changed - its now off by default and
+The default for inheritance has changed - it is now off by default and
 :attr:`_cls` will not be stored automatically with the class.  So if you extend
 your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments`
 you will need to declare :attr:`allow_inheritance` in the meta data like so: ::
@@ -71,7 +81,7 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: ::
 
         meta = {'allow_inheritance': True}
 
-Previously, if you had data the database that wasn't defined in the Document
+Previously, if you had data in the database that wasn't defined in the Document
 definition, it would set it as an attribute on the document.  This is no longer
 the case and the data is set only in the ``document._data`` dictionary: ::
 
@@ -91,10 +101,17 @@ the case and the data is set only in the ``document._data`` dictionary: ::
       File "<stdin>", line 1, in <module>
     AttributeError: 'Animal' object has no attribute 'size'
 
+The Document class has introduced a reserved function `clean()`, which will be
+called before saving the document. If your document class happens to have a method
+with the same name, please try to rename it.
+
+    def clean(self):
+        pass
+
 ReferenceField
 --------------
 
-ReferenceFields now store ObjectId's by default - this is more efficient than
+ReferenceFields now store ObjectIds by default - this is more efficient than
 DBRefs as we already know what Document types they reference::
 
     # Old code
@@ -116,13 +133,17 @@ eg::
 
     # Mark all ReferenceFields as dirty and save
     for p in Person.objects:
-        p._mark_as_dirty('parent')
+        p._mark_as_changed('parent')
-        p._mark_as_dirty('friends')
+        p._mark_as_changed('friends')
         p.save()
 
 `An example test migration for ReferenceFields is available on github
 <https://github.com/MongoEngine/mongoengine/blob/master/tests/migration/refrencefield_dbref_to_object_id.py>`_.
 
+.. Note:: Internally mongoengine handles ReferenceFields the same, so they are
+   converted to DBRef on loading and ObjectIds or DBRefs depending on settings
+   on storage.
+
 UUIDField
 ---------
 
@@ -136,16 +157,16 @@ UUIDFields now default to storing binary values::
     class Animal(Document):
         uuid = UUIDField(binary=False)
 
-To migrate all the uuid's you need to touch each object and mark it as dirty
+To migrate all the uuids you need to touch each object and mark it as dirty
 eg::
 
     # Doc definition
     class Animal(Document):
         uuid = UUIDField()
 
-    # Mark all ReferenceFields as dirty and save
+    # Mark all UUIDFields as dirty and save
     for a in Animal.objects:
-        a._mark_as_dirty('uuid')
+        a._mark_as_changed('uuid')
         a.save()
 
 `An example test migration for UUIDFields is available on github
@@ -154,7 +175,7 @@ eg::
 DecimalField
 ------------
 
-DecimalField now store floats - previous it was storing strings and that
+DecimalFields now store floats - previously it was storing strings and that
 made it impossible to do comparisons when querying correctly.::
 
     # Old code
@@ -165,19 +186,19 @@ made it impossible to do comparisons when querying correctly.::
     class Person(Document):
         balance = DecimalField(force_string=True)
 
-To migrate all the uuid's you need to touch each object and mark it as dirty
+To migrate all the DecimalFields you need to touch each object and mark it as dirty
 eg::
 
     # Doc definition
     class Person(Document):
         balance = DecimalField()
 
-    # Mark all ReferenceFields as dirty and save
+    # Mark all DecimalField's as dirty and save
     for p in Person.objects:
-        p._mark_as_dirty('balance')
+        p._mark_as_changed('balance')
         p.save()
 
-.. note:: DecimalField's have also been improved with the addition of precision
+.. note:: DecimalFields have also been improved with the addition of precision
     and rounding.  See :class:`~mongoengine.fields.DecimalField` for more information.
 
 `An example test migration for DecimalFields is available on github
@@ -186,7 +207,7 @@ eg::
 Cascading Saves
 ---------------
 To improve performance document saves will no longer automatically cascade.
-Any changes to a Documents references will either have to be saved manually or
+Any changes to a Document's references will either have to be saved manually or
 you will have to explicitly tell it to cascade on save::
 
     # At the class level:
@@ -228,7 +249,7 @@ update your code like so: ::
 
     # Update example a) assign queryset after a change:
     mammals = Animal.objects(type="mammal")
-    carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so fitler can be applied
+    carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so filter can be applied
     [m for m in carnivores]                        # This will return all carnivores
 
     # Update example b) chain the queryset:
@@ -255,7 +276,7 @@ queryset you should upgrade to use count::
 .only() now inline with .exclude()
 ----------------------------------
 
-The behaviour of `.only()` was highly ambious, now it works in the mirror fashion
+The behaviour of `.only()` was highly ambiguous, now it works in mirror fashion
 to `.exclude()`.  Chaining `.only()` calls will increase the fields required::
 
     # Old code
@@ -419,7 +440,7 @@ main areas of changed are: choices in fields, map_reduce and collection names.
 Choice options:
 ===============
 
-Are now expected to be an iterable of tuples, with  the first element in each
+Are now expected to be an iterable of tuples, with the first element in each
 tuple being the actual value to be stored. The second element is the
 human-readable name for the option.
 
@@ -441,8 +462,8 @@ such the following have been changed:
 Default collection naming
 =========================
 
-Previously it was just lowercase, its now much more pythonic and readable as
+Previously it was just lowercase, it's now much more pythonic and readable as
-its lowercase and underscores, previously ::
+it's lowercase and underscores, previously ::
 
     class MyAceDocument(Document):
         pass
@@ -509,5 +530,5 @@ Alternatively, you can rename your collections eg ::
 mongodb 1.8 > 2.0 +
 ===================
 
-Its been reported that indexes may need to be recreated to the newer version of indexes.
+It's been reported that indexes may need to be recreated to the newer version of indexes.
 To do this drop indexes and call ``ensure_indexes`` on each model.

mongoengine/__init__.py

@@ -15,7 +15,7 @@ import django
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
 
-VERSION = (0, 8, 0)
+VERSION = (0, 8, 4)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -13,7 +13,11 @@ class BaseDict(dict):
     _name = None
 
     def __init__(self, dict_items, instance, name):
-        self._instance = weakref.proxy(instance)
+        Document = _import_class('Document')
+        EmbeddedDocument = _import_class('EmbeddedDocument')
+
+        if isinstance(instance, (Document, EmbeddedDocument)):
+            self._instance = weakref.proxy(instance)
         self._name = name
         return super(BaseDict, self).__init__(dict_items)
 
@@ -80,7 +84,11 @@ class BaseList(list):
     _name = None
 
     def __init__(self, list_items, instance, name):
-        self._instance = weakref.proxy(instance)
+        Document = _import_class('Document')
+        EmbeddedDocument = _import_class('EmbeddedDocument')
+
+        if isinstance(instance, (Document, EmbeddedDocument)):
+            self._instance = weakref.proxy(instance)
         self._name = name
         return super(BaseList, self).__init__(list_items)
 
@@ -100,6 +108,14 @@ class BaseList(list):
         self._mark_as_changed()
         return super(BaseList, self).__delitem__(*args, **kwargs)
 
+    def __setslice__(self, *args, **kwargs):
+        self._mark_as_changed()
+        return super(BaseList, self).__setslice__(*args, **kwargs)
+
+    def __delslice__(self, *args, **kwargs):
+        self._mark_as_changed()
+        return super(BaseList, self).__delslice__(*args, **kwargs)
+
     def __getstate__(self):
         self.instance = None
         self._dereferenced = False

mongoengine/base/document.py

@@ -4,7 +4,7 @@ import numbers
 from functools import partial
 
 import pymongo
-from bson import json_util
+from bson import json_util, ObjectId
 from bson.dbref import DBRef
 from bson.son import SON
 
@@ -42,6 +42,9 @@ class BaseDocument(object):
             # Combine positional arguments with named arguments.
             # We only want named arguments.
             field = iter(self._fields_ordered)
+            # If its an automatic id field then skip to the first defined field
+            if self._auto_id_field:
+                next(field)
             for value in args:
                 name = next(field)
                 if name in values:
@@ -51,6 +54,7 @@ class BaseDocument(object):
         signals.pre_init.send(self.__class__, document=self, values=values)
 
         self._data = {}
+        self._dynamic_fields = SON()
 
         # Assign default values to instance
         for key, field in self._fields.iteritems():
@@ -61,7 +65,6 @@ class BaseDocument(object):
 
         # Set passed values after initialisation
         if self._dynamic:
-            self._dynamic_fields = {}
             dynamic_data = {}
             for key, value in values.iteritems():
                 if key in self._fields or key == '_id':
@@ -116,6 +119,7 @@ class BaseDocument(object):
                 field = DynamicField(db_field=name)
                 field.name = name
                 self._dynamic_fields[name] = field
+                self._fields_ordered += (name,)
 
             if not name.startswith('_'):
                 value = self.__expand_dynamic_values(name, value)
@@ -142,28 +146,32 @@ class BaseDocument(object):
 
     def __getstate__(self):
         data = {}
-        for k in ('_changed_fields', '_initialised', '_created'):
+        for k in ('_changed_fields', '_initialised', '_created',
-            data[k] = getattr(self, k)
+                  '_dynamic_fields', '_fields_ordered'):
+            if hasattr(self, k):
+                data[k] = getattr(self, k)
         data['_data'] = self.to_mongo()
         return data
 
     def __setstate__(self, data):
         if isinstance(data["_data"], SON):
             data["_data"] = self.__class__._from_son(data["_data"])._data
-        for k in ('_changed_fields', '_initialised', '_created', '_data'):
+        for k in ('_changed_fields', '_initialised', '_created', '_data',
-            setattr(self, k, data[k])
+                  '_fields_ordered', '_dynamic_fields'):
+            if k in data:
+                setattr(self, k, data[k])
+        dynamic_fields = data.get('_dynamic_fields') or SON()
+        for k in dynamic_fields.keys():
+            setattr(self, k, data["_data"].get(k))
 
     def __iter__(self):
-        if 'id' in self._fields and 'id' not in self._fields_ordered:
-            return iter(('id', ) + self._fields_ordered)
-
         return iter(self._fields_ordered)
 
     def __getitem__(self, name):
         """Dictionary-style field access, return a field's value if present.
         """
         try:
-            if name in self._fields:
+            if name in self._fields_ordered:
                 return getattr(self, name)
         except AttributeError:
             pass
@@ -213,7 +221,7 @@ class BaseDocument(object):
         return not self.__eq__(other)
 
     def __hash__(self):
-        if self.pk is None:
+        if getattr(self, 'pk', None) is None:
             # For new object
             return super(BaseDocument, self).__hash__()
         else:
@@ -239,6 +247,8 @@ class BaseDocument(object):
         for field_name in self:
             value = self._data.get(field_name, None)
             field = self._fields.get(field_name)
+            if field is None and self._dynamic:
+                field = self._dynamic_fields.get(field_name)
 
             if value is not None:
                 value = field.to_mongo(value)
@@ -252,8 +262,10 @@ class BaseDocument(object):
                 data[field.db_field] = value
 
         # If "_id" has not been set, then try and set it
-        if data["_id"] is None:
+        Document = _import_class("Document")
-            data["_id"] = self._data.get("id", None)
+        if isinstance(self, Document):
+            if data["_id"] is None:
+                data["_id"] = self._data.get("id", None)
 
         if data['_id'] is None:
             data.pop('_id')
@@ -263,15 +275,6 @@ class BaseDocument(object):
            not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)):
             data.pop('_cls')
 
-        if not self._dynamic:
-            return data
-
-        # Sort dynamic fields by key
-        dynamic_fields = sorted(self._dynamic_fields.iteritems(),
-                                key=operator.itemgetter(0))
-        for name, field in dynamic_fields:
-            data[name] = field.to_mongo(self._data.get(name, None))
-
         return data
 
     def validate(self, clean=True):
@@ -287,11 +290,8 @@ class BaseDocument(object):
                 errors[NON_FIELD_ERRORS] = error
 
         # Get a list of tuples of field names and their current values
-        fields = [(field, self._data.get(name))
+        fields = [(self._fields.get(name, self._dynamic_fields.get(name)),
-                  for name, field in self._fields.items()]
+                   self._data.get(name)) for name in self._fields_ordered]
-        if self._dynamic:
-            fields += [(field, self._data.get(name))
-                       for name, field in self._dynamic_fields.items()]
 
         EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
         GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField")
@@ -321,9 +321,9 @@ class BaseDocument(object):
             message = "ValidationError (%s:%s) " % (self._class_name, pk)
             raise ValidationError(message, errors=errors)
 
-    def to_json(self):
+    def to_json(self, *args, **kwargs):
         """Converts a document to JSON"""
-        return json_util.dumps(self.to_mongo())
+        return json_util.dumps(self.to_mongo(),  *args, **kwargs)
 
     @classmethod
     def from_json(cls, json_data):
@@ -390,11 +390,12 @@ class BaseDocument(object):
                 if field_value:
                     field_value._clear_changed_fields()
 
-    def _get_changed_fields(self, key='', inspected=None):
+    def _get_changed_fields(self, inspected=None):
         """Returns a list of all fields that have explicitly been changed.
         """
         EmbeddedDocument = _import_class("EmbeddedDocument")
         DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument")
+        ReferenceField = _import_class("ReferenceField")
         _changed_fields = []
         _changed_fields += getattr(self, '_changed_fields', [])
 
@@ -404,38 +405,39 @@ class BaseDocument(object):
                 return _changed_fields
             inspected.add(self.id)
 
-        field_list = self._fields.copy()
+        for field_name in self._fields_ordered:
-        if self._dynamic:
-            field_list.update(self._dynamic_fields)
-
-        for field_name in field_list:
-
             db_field_name = self._db_field_map.get(field_name, field_name)
             key = '%s.' % db_field_name
-            field = self._data.get(field_name, None)
+            data = self._data.get(field_name, None)
-            if hasattr(field, 'id'):
+            field = self._fields.get(field_name)
-                if field.id in inspected:
-                    continue
-                inspected.add(field.id)
 
-            if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument))
+            if hasattr(data, 'id'):
+                if data.id in inspected:
+                    continue
+                inspected.add(data.id)
+            if isinstance(field, ReferenceField):
+                continue
+            elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
                and db_field_name not in _changed_fields):
                  # Find all embedded fields that have been changed
-                changed = field._get_changed_fields(key, inspected)
+                changed = data._get_changed_fields(inspected)
                 _changed_fields += ["%s%s" % (key, k) for k in changed if k]
-            elif (isinstance(field, (list, tuple, dict)) and
+            elif (isinstance(data, (list, tuple, dict)) and
                     db_field_name not in _changed_fields):
                 # Loop list / dict fields as they contain documents
                 # Determine the iterator to use
-                if not hasattr(field, 'items'):
+                if not hasattr(data, 'items'):
-                    iterator = enumerate(field)
+                    iterator = enumerate(data)
                 else:
-                    iterator = field.iteritems()
+                    iterator = data.iteritems()
                 for index, value in iterator:
                     if not hasattr(value, '_get_changed_fields'):
                         continue
+                    if (hasattr(field, 'field') and
+                        isinstance(field.field, ReferenceField)):
+                        continue
                     list_key = "%s%s." % (key, index)
-                    changed = value._get_changed_fields(list_key, inspected)
+                    changed = value._get_changed_fields(inspected)
                     _changed_fields += ["%s%s" % (list_key, k)
                                         for k in changed if k]
         return _changed_fields
@@ -448,7 +450,6 @@ class BaseDocument(object):
         doc = self.to_mongo()
 
         set_fields = self._get_changed_fields()
-        set_data = {}
         unset_data = {}
         parts = []
         if hasattr(self, '_changed_fields'):
@@ -459,7 +460,7 @@ class BaseDocument(object):
                 d = doc
                 new_path = []
                 for p in parts:
-                    if isinstance(d, DBRef):
+                    if isinstance(d, (ObjectId, DBRef)):
                         break
                     elif isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
@@ -628,8 +629,10 @@ class BaseDocument(object):
         # Check to see if we need to include _cls
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
-        include_cls = allow_inheritance and not spec.get('sparse', False)
+        include_cls = (allow_inheritance and not spec.get('sparse', False) and
-
+                       spec.get('cls',  True))
+        if "cls" in spec:
+            spec.pop('cls')
         for key in spec['fields']:
             # If inherited spec continue
             if isinstance(key, (list, tuple)):
@@ -759,7 +762,7 @@ class BaseDocument(object):
 
         for field_name in parts:
             # Handle ListField indexing:
-            if field_name.isdigit():
+            if field_name.isdigit() and hasattr(field, 'field'):
                 new_field = field.field
                 fields.append(field_name)
                 continue

mongoengine/base/fields.py

@@ -36,6 +36,29 @@ class BaseField(object):
                  unique=False, unique_with=None, primary_key=False,
                  validation=None, choices=None, verbose_name=None,
                  help_text=None):
+        """
+        :param db_field: The database field to store this field in
+            (defaults to the name of the field)
+        :param name: Depreciated - use db_field
+        :param required: If the field is required. Whether it has to have a
+            value or not. Defaults to False.
+        :param default: (optional) The default value for this field if no value
+            has been set (or if the value has been unset).  It Can be a
+            callable.
+        :param unique: Is the field value unique or not.  Defaults to False.
+        :param unique_with: (optional) The other field this field should be
+            unique with.
+        :param primary_key: Mark this field as the primary key. Defaults to False.
+        :param validation: (optional) A callable to validate the value of the
+            field.  Generally this is deprecated in favour of the
+            `FIELD.validate` method
+        :param choices: (optional) The valid choices
+        :param verbose_name: (optional)  The verbose name for the field.
+            Designed to be human readable and is often used when generating
+            model forms from the document model.
+        :param help_text: (optional) The help text for this field and is often
+            used when generating model forms from the document model.
+        """
         self.db_field = (db_field or name) if not primary_key else '_id'
         if name:
             msg = "Fields' 'name' attribute deprecated in favour of 'db_field'"
@@ -59,20 +82,14 @@ class BaseField(object):
             BaseField.creation_counter += 1
 
     def __get__(self, instance, owner):
-        """Descriptor for retrieving a value from a field in a document. Do
+        """Descriptor for retrieving a value from a field in a document.
-        any necessary conversion between Python and MongoDB types.
         """
         if instance is None:
             # Document class being used rather than a document object
             return self
-        # Get value from document instance if available, if not use default
-        value = instance._data.get(self.name)
 
-        if value is None:
+        # Get value from document instance if available
-            value = self.default
+        value = instance._data.get(self.name)
-            # Allow callable default values
-            if callable(value):
-                value = value()
 
         EmbeddedDocument = _import_class('EmbeddedDocument')
         if isinstance(value, EmbeddedDocument) and value._instance is None:
@@ -82,6 +99,14 @@ class BaseField(object):
     def __set__(self, instance, value):
         """Descriptor for assigning a value to a field in a document.
         """
+
+        # If setting to None and theres a default
+        # Then set the value to the default value
+        if value is None and self.default is not None:
+            value = self.default
+            if callable(value):
+                value = value()
+
         if instance._initialised:
             try:
                 if (self.name not in instance._data or
@@ -161,7 +186,6 @@ class ComplexBaseField(BaseField):
     """
 
     field = None
-    __dereference = False
 
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references.
@@ -176,9 +200,11 @@ class ComplexBaseField(BaseField):
                        (self.field is None or isinstance(self.field,
                         (GenericReferenceField, ReferenceField))))
 
+        _dereference = _import_class("DeReference")()
+
         self._auto_dereference = instance._fields[self.name]._auto_dereference
-        if not self.__dereference and instance._initialised and dereference:
+        if instance._initialised and dereference:
-            instance._data[self.name] = self._dereference(
+            instance._data[self.name] = _dereference(
                 instance._data.get(self.name), max_depth=1, instance=instance,
                 name=self.name
             )
@@ -197,7 +223,7 @@ class ComplexBaseField(BaseField):
         if (self._auto_dereference and instance._initialised and
            isinstance(value, (BaseList, BaseDict))
            and not value._dereferenced):
-            value = self._dereference(
+            value = _dereference(
                 value, max_depth=1, instance=instance, name=self.name
             )
             value._dereferenced = True
@@ -205,12 +231,6 @@ class ComplexBaseField(BaseField):
 
         return value
 
-    def __set__(self, instance, value):
-        """Descriptor for assigning a value to a field in a document.
-        """
-        instance._data[self.name] = value
-        instance._mark_as_changed(self.name)
-
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
         """
@@ -363,13 +383,6 @@ class ComplexBaseField(BaseField):
 
     owner_document = property(_get_owner_document, _set_owner_document)
 
-    @property
-    def _dereference(self,):
-        if not self.__dereference:
-            DeReference = _import_class("DeReference")
-            self.__dereference = DeReference()  # Cached
-        return self.__dereference
-
 
 class ObjectIdField(BaseField):
     """A field wrapper around MongoDB's ObjectIds.

mongoengine/base/metaclasses.py

@@ -91,11 +91,12 @@ class DocumentMetaclass(type):
         attrs['_fields'] = doc_fields
         attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k))
                                       for k, v in doc_fields.iteritems()])
+        attrs['_reverse_db_field_map'] = dict(
+            (v, k) for k, v in attrs['_db_field_map'].iteritems())
+
         attrs['_fields_ordered'] = tuple(i[1] for i in sorted(
                                          (v.creation_counter, v.name)
                                          for v in doc_fields.itervalues()))
-        attrs['_reverse_db_field_map'] = dict(
-            (v, k) for k, v in attrs['_db_field_map'].iteritems())
 
         #
         # Set document hierarchy
@@ -358,12 +359,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                     new_class.id = field
 
         # Set primary key if not defined by the document
+        new_class._auto_id_field = False
         if not new_class._meta.get('id_field'):
+            new_class._auto_id_field = True
             new_class._meta['id_field'] = 'id'
             new_class._fields['id'] = ObjectIdField(db_field='_id')
             new_class._fields['id'].name = 'id'
             new_class.id = new_class._fields['id']
 
+        # Prepend id field to _fields_ordered
+        if 'id' in new_class._fields and 'id' not in new_class._fields_ordered:
+            new_class._fields_ordered = ('id', ) + new_class._fields_ordered
+
         # Merge in exceptions with parent hierarchy
         exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned)
         module = attrs.get('__module__')

mongoengine/common.py

@@ -2,7 +2,19 @@ _class_registry_cache = {}
 
 
 def _import_class(cls_name):
-    """Cached mechanism for imports"""
+    """Cache mechanism for imports.
+
+    Due to complications of circular imports mongoengine needs to do lots of
+    inline imports in functions.  This is inefficient as classes are
+    imported repeated throughout the mongoengine code.  This is
+    compounded by some recursive functions requiring inline imports.
+
+    :mod:`mongoengine.common` provides a single point to import all these
+    classes.  Circular imports aren't an issue as it dynamically imports the
+    class when first needed.  Subsequent calls to the
+    :func:`~mongoengine.common._import_class` can then directly retrieve the
+    class from the :data:`mongoengine.common._class_registry_cache`.
+    """
     if cls_name in _class_registry_cache:
         return _class_registry_cache.get(cls_name)
 
@@ -11,8 +23,9 @@ def _import_class(cls_name):
     field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField',
                      'FileField', 'GenericReferenceField',
                      'GenericEmbeddedDocumentField', 'GeoPointField',
-                     'PointField', 'LineStringField', 'PolygonField',
+                     'PointField', 'LineStringField', 'ListField',
-                     'ReferenceField', 'StringField', 'ComplexBaseField')
+                     'PolygonField', 'ReferenceField', 'StringField',
+                     'ComplexBaseField')
     queryset_classes = ('OperationError',)
     deref_classes = ('DeReference',)
 

mongoengine/connection.py

@@ -55,12 +55,9 @@ def register_connection(alias, name, host='localhost', port=27017,
     # Handle uri style connections
     if "://" in host:
         uri_dict = uri_parser.parse_uri(host)
-        if uri_dict.get('database') is None:
-            raise ConnectionError("If using URI style connection include "\
-                                  "database name in string")
         conn_settings.update({
             'host': host,
-            'name': uri_dict.get('database'),
+            'name': uri_dict.get('database') or name,
             'username': uri_dict.get('username'),
             'password': uri_dict.get('password'),
             'read_preference': read_preference,

mongoengine/context_managers.py

@@ -189,7 +189,8 @@ class query_counter(object):
 
     def __eq__(self, value):
         """ == Compare querycounter. """
-        return value == self._get_count()
+        counter = self._get_count()
+        return value == counter
 
     def __ne__(self, value):
         """ != Compare querycounter. """
@@ -221,6 +222,7 @@ class query_counter(object):
 
     def _get_count(self):
         """ Get the number of queries. """
-        count = self.db.system.profile.find().count() - self.counter
+        ignore_query = {"ns": {"$ne": "%s.system.indexes" % self.db.name}}
+        count = self.db.system.profile.find(ignore_query).count() - self.counter
         self.counter += 1
         return count

mongoengine/dereference.py

@@ -4,7 +4,7 @@ from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document)
 from fields import (ReferenceField, ListField, DictField, MapField)
 from connection import get_db
 from queryset import QuerySet
-from document import Document
+from document import Document, EmbeddedDocument
 
 
 class DeReference(object):
@@ -33,7 +33,8 @@ class DeReference(object):
         self.max_depth = max_depth
         doc_type = None
 
-        if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)):
+        if instance and isinstance(instance, (Document, EmbeddedDocument,
+                                              TopLevelDocumentMetaclass)):
             doc_type = instance._fields.get(name)
             if hasattr(doc_type, 'field'):
                 doc_type = doc_type.field

mongoengine/django/mongo_auth/models.py

@@ -1,16 +1,34 @@
-from importlib import import_module
-
 from django.conf import settings
 from django.contrib.auth.models import UserManager
 from django.core.exceptions import ImproperlyConfigured
 from django.db import models
+from django.utils.importlib import import_module
 from django.utils.translation import ugettext_lazy as _
 
 
+__all__ = (
+    'get_user_document',
+)
+
+
 MONGOENGINE_USER_DOCUMENT = getattr(
     settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User')
 
 
+def get_user_document():
+    """Get the user document class used for authentication.
+
+    This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which
+    defaults to `mongoengine.django.auth.User`.
+
+    """
+
+    name = MONGOENGINE_USER_DOCUMENT
+    dot = name.rindex('.')
+    module = import_module(name[:dot])
+    return getattr(module, name[dot + 1:])
+
+
 class MongoUserManager(UserManager):
     """A User manager wich allows the use of MongoEngine documents in Django.
 
@@ -45,7 +63,7 @@ class MongoUserManager(UserManager):
     def contribute_to_class(self, model, name):
         super(MongoUserManager, self).contribute_to_class(model, name)
         self.dj_model = self.model
-        self.model = self._get_user_document()
+        self.model = get_user_document()
 
         self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD
         username = models.CharField(_('username'), max_length=30, unique=True)
@@ -56,16 +74,6 @@ class MongoUserManager(UserManager):
             field = models.CharField(_(name), max_length=30)
             field.contribute_to_class(self.dj_model, name)
 
-    def _get_user_document(self):
-        try:
-            name = MONGOENGINE_USER_DOCUMENT
-            dot = name.rindex('.')
-            module = import_module(name[:dot])
-            return getattr(module, name[dot + 1:])
-        except ImportError:
-            raise ImproperlyConfigured("Error importing %s, please check "
-                                       "settings.MONGOENGINE_USER_DOCUMENT"
-                                       % name)
 
     def get(self, *args, **kwargs):
         try:
@@ -86,5 +94,14 @@ class MongoUserManager(UserManager):
 
 
 class MongoUser(models.Model):
-    objects = MongoUserManager()
+    """"Dummy user model for Django.
 
+    MongoUser is used to replace Django's UserManager with MongoUserManager.
+    The actual user document class is mongoengine.django.auth.User or any
+    other document class specified in MONGOENGINE_USER_DOCUMENT.
+
+    To get the user document class, use `get_user_document()`.
+
+    """
+
+    objects = MongoUserManager()

mongoengine/django/sessions.py

@@ -1,7 +1,10 @@
 from django.conf import settings
 from django.contrib.sessions.backends.base import SessionBase, CreateError
 from django.core.exceptions import SuspiciousOperation
-from django.utils.encoding import force_unicode
+try:
+    from django.utils.encoding import force_unicode
+except ImportError:
+    from django.utils.encoding import force_text as force_unicode
 
 from mongoengine.document import Document
 from mongoengine import fields
@@ -39,7 +42,7 @@ class MongoSession(Document):
         'indexes': [
             {
                 'fields': ['expire_date'],
-                'expireAfterSeconds': settings.SESSION_COOKIE_AGE
+                'expireAfterSeconds': 0
             }
         ]
     }

mongoengine/document.py

@@ -1,10 +1,14 @@
 import warnings
 
+import hashlib
 import pymongo
 import re
 
+from pymongo.read_preferences import ReadPreference
+from bson import ObjectId
 from bson.dbref import DBRef
 from mongoengine import signals
+from mongoengine.common import _import_class
 from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass,
                               BaseDocument, BaseDict, BaseList,
                               ALLOW_INHERITANCE, get_document)
@@ -17,6 +21,19 @@ __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument',
            'InvalidCollectionError', 'NotUniqueError', 'MapReduceDocument')
 
 
+def includes_cls(fields):
+    """ Helper function used for ensuring and comparing indexes
+    """
+
+    first_field = None
+    if len(fields):
+        if isinstance(fields[0], basestring):
+            first_field = fields[0]
+        elif isinstance(fields[0], (list, tuple)) and len(fields[0]):
+            first_field = fields[0][0]
+    return first_field == '_cls'
+
+
 class InvalidCollectionError(Exception):
     pass
 
@@ -52,6 +69,9 @@ class EmbeddedDocument(BaseDocument):
             return self._data == other._data
         return False
 
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
 
 class Document(BaseDocument):
     """The base class used for defining the structure and properties of
@@ -179,8 +199,8 @@ class Document(BaseDocument):
             will force an fsync on the primary server.
         :param cascade: Sets the flag for cascading saves.  You can set a
             default by setting "cascade" in the document __meta__
-        :param cascade_kwargs: optional kwargs dictionary to be passed throw
+        :param cascade_kwargs: (optional) kwargs dictionary to be passed throw
-            to cascading saves
+            to cascading saves.  Implies ``cascade=True``.
         :param _refs: A list of processed references used in cascading saves
 
         .. versionchanged:: 0.5
@@ -189,24 +209,28 @@ class Document(BaseDocument):
             :class:`~bson.dbref.DBRef` objects that have changes are
             saved as well.
         .. versionchanged:: 0.6
-            Cascade saves are optional = defaults to True, if you want
+            Added cascading saves
+        .. versionchanged:: 0.8
+            Cascade saves are optional and default to False.  If you want
             fine grain control then you can turn off using document
-            meta['cascade'] = False  Also you can pass different kwargs to
+            meta['cascade'] = True.  Also you can pass different kwargs to
             the cascade save using cascade_kwargs which overwrites the
-            existing kwargs with custom values
+            existing kwargs with custom values.
         """
         signals.pre_save.send(self.__class__, document=self)
 
         if validate:
             self.validate(clean=clean)
 
-        if not write_concern:
+        if write_concern is None:
-            write_concern = {}
+            write_concern = {"w": 1}
 
         doc = self.to_mongo()
 
         created = ('_id' not in doc or self._created or force_insert)
 
+        signals.pre_save_post_validation.send(self.__class__, document=self, created=created)
+
         try:
             collection = self._get_collection()
             if created:
@@ -242,8 +266,9 @@ class Document(BaseDocument):
                                                    upsert=True, **write_concern)
                     created = is_new_object(last_error)
 
-            cascade = (self._meta.get('cascade', True)
+            if cascade is None:
-                       if cascade is None else cascade)
+                cascade = self._meta.get('cascade', False) or cascade_kwargs is not None
+
             if cascade:
                 kwargs = {
                     "force_insert": force_insert,
@@ -276,15 +301,17 @@ class Document(BaseDocument):
     def cascade_save(self, *args, **kwargs):
         """Recursively saves any references /
            generic references on an objects"""
-        import fields
         _refs = kwargs.get('_refs', []) or []
 
+        ReferenceField = _import_class('ReferenceField')
+        GenericReferenceField = _import_class('GenericReferenceField')
+
         for name, cls in self._fields.items():
-            if not isinstance(cls, (fields.ReferenceField,
+            if not isinstance(cls, (ReferenceField,
-                                    fields.GenericReferenceField)):
+                                    GenericReferenceField)):
                 continue
 
-            ref = getattr(self, name)
+            ref = self._data.get(name)
             if not ref or isinstance(ref, DBRef):
                 continue
 
@@ -325,7 +352,13 @@ class Document(BaseDocument):
         been saved.
         """
         if not self.pk:
-            raise OperationError('attempt to update a document not yet saved')
+            if kwargs.get('upsert', False):
+                query = self.to_mongo()
+                if "_cls" in query:
+                    del(query["_cls"])
+                return self._qs.filter(**query).update_one(**kwargs)
+            else:
+                raise OperationError('attempt to update a document not yet saved')
 
         # Need to add shard key to query, or you get an error
         return self._qs.filter(**self._object_key).update_one(**kwargs)
@@ -344,11 +377,10 @@ class Document(BaseDocument):
         signals.pre_delete.send(self.__class__, document=self)
 
         try:
-            self._qs.filter(**self._object_key).delete(write_concern=write_concern)
+            self._qs.filter(**self._object_key).delete(write_concern=write_concern, _from_doc_delete=True)
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
-
         signals.post_delete.send(self.__class__, document=self)
 
     def switch_db(self, db_alias):
@@ -368,7 +400,7 @@ class Document(BaseDocument):
         """
         with switch_db(self.__class__, db_alias) as cls:
             collection = cls._get_collection()
-            db = cls._get_db
+            db = cls._get_db()
         self._get_collection = lambda: collection
         self._get_db = lambda: db
         self._collection = collection
@@ -408,8 +440,8 @@ class Document(BaseDocument):
 
         .. versionadded:: 0.5
         """
-        import dereference
+        DeReference = _import_class('DeReference')
-        self._data = dereference.DeReference()(self._data, max_depth)
+        DeReference()([self], max_depth + 1)
         return self
 
     def reload(self, max_depth=1):
@@ -418,19 +450,16 @@ class Document(BaseDocument):
         .. versionadded:: 0.1.2
         .. versionchanged:: 0.6  Now chainable
         """
-        id_field = self._meta['id_field']
+        obj = self._qs.read_preference(ReadPreference.PRIMARY).filter(
-        obj = self._qs.filter(**{id_field: self[id_field]}
+                **self._object_key).limit(1).select_related(max_depth=max_depth)
-                              ).limit(1).select_related(max_depth=max_depth)
+
         if obj:
             obj = obj[0]
         else:
             msg = "Reloaded document has been deleted"
             raise OperationError(msg)
-        for field in self._fields:
+        for field in self._fields_ordered:
             setattr(self, field, self._reload(field, obj[field]))
-        if self._dynamic:
-            for name in self._dynamic_fields.keys():
-                setattr(self, name, self._reload(name, obj._data[name]))
         self._changed_fields = obj._changed_fields
         self._created = False
         return obj
@@ -446,6 +475,7 @@ class Document(BaseDocument):
             value = [self._reload(key, v) for v in value]
             value = BaseList(value, self, key)
         elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)):
+            value._instance = None
             value._changed_fields = []
         return value
 
@@ -506,6 +536,8 @@ class Document(BaseDocument):
     def ensure_indexes(cls):
         """Checks the document meta data and ensures all the indexes exist.
 
+        Global defaults can be set in the meta - see :doc:`guide/defining-documents`
+
         .. note:: You can disable automatic index creation by setting
                   `auto_create_index` to False in the documents meta data
         """
@@ -521,14 +553,6 @@ class Document(BaseDocument):
         # an extra index on _cls, as mongodb will use the existing
         # index to service queries against _cls
         cls_indexed = False
-        def includes_cls(fields):
-            first_field = None
-            if len(fields):
-                if isinstance(fields[0], basestring):
-                    first_field = fields[0]
-                elif isinstance(fields[0], (list, tuple)) and len(fields[0]):
-                    first_field = fields[0][0]
-            return first_field == '_cls'
 
         # Ensure document-defined indexes are created
         if cls._meta['index_specs']:
@@ -549,6 +573,90 @@ class Document(BaseDocument):
             collection.ensure_index('_cls', background=background,
                                     **index_opts)
 
+    @classmethod
+    def list_indexes(cls, go_up=True, go_down=True):
+        """ Lists all of the indexes that should be created for given
+        collection. It includes all the indexes from super- and sub-classes.
+        """
+
+        if cls._meta.get('abstract'):
+            return []
+
+        # get all the base classes, subclasses and sieblings
+        classes = []
+        def get_classes(cls):
+
+            if (cls not in classes and
+               isinstance(cls, TopLevelDocumentMetaclass)):
+                classes.append(cls)
+
+            for base_cls in cls.__bases__:
+                if (isinstance(base_cls, TopLevelDocumentMetaclass) and
+                   base_cls != Document and
+                   not base_cls._meta.get('abstract') and
+                   base_cls._get_collection().full_name == cls._get_collection().full_name and
+                   base_cls not in classes):
+                    classes.append(base_cls)
+                    get_classes(base_cls)
+            for subclass in cls.__subclasses__():
+                if (isinstance(base_cls, TopLevelDocumentMetaclass) and
+                   subclass._get_collection().full_name == cls._get_collection().full_name and
+                   subclass not in classes):
+                    classes.append(subclass)
+                    get_classes(subclass)
+
+        get_classes(cls)
+
+        # get the indexes spec for all of the gathered classes
+        def get_indexes_spec(cls):
+            indexes = []
+
+            if cls._meta['index_specs']:
+                index_spec = cls._meta['index_specs']
+                for spec in index_spec:
+                    spec = spec.copy()
+                    fields = spec.pop('fields')
+                    indexes.append(fields)
+            return indexes
+
+        indexes = []
+        for cls in classes:
+            for index in get_indexes_spec(cls):
+                if index not in indexes:
+                    indexes.append(index)
+
+        # finish up by appending { '_id': 1 } and { '_cls': 1 }, if needed
+        if [(u'_id', 1)] not in indexes:
+            indexes.append([(u'_id', 1)])
+        if (cls._meta.get('index_cls', True) and
+           cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
+             indexes.append([(u'_cls', 1)])
+
+        return indexes
+
+    @classmethod
+    def compare_indexes(cls):
+        """ Compares the indexes defined in MongoEngine with the ones existing
+        in the database. Returns any missing/extra indexes.
+        """
+
+        required = cls.list_indexes()
+        existing = [info['key'] for info in cls._get_collection().index_information().values()]
+        missing = [index for index in required if index not in existing]
+        extra = [index for index in existing if index not in required]
+
+        # if { _cls: 1 } is missing, make sure it's *really* necessary
+        if [(u'_cls', 1)] in missing:
+            cls_obsolete = False
+            for index in existing:
+                if includes_cls(index) and index not in extra:
+                    cls_obsolete = True
+                    break
+            if cls_obsolete:
+                missing.remove([(u'_cls', 1)])
+
+        return {'missing': missing, 'extra': extra}
+
 
 class DynamicDocument(Document):
     """A Dynamic Document class allowing flexible, expandable and uncontrolled

mongoengine/fields.py

@@ -8,6 +8,13 @@ import uuid
 import warnings
 from operator import itemgetter
 
+try:
+    import dateutil
+except ImportError:
+    dateutil = None
+else:
+    import dateutil.parser
+
 import pymongo
 import gridfs
 from bson import Binary, DBRef, SON, ObjectId
@@ -272,14 +279,14 @@ class DecimalField(BaseField):
         :param precision: Number of decimal places to store.
         :param rounding: The rounding rule from the python decimal libary:
 
-            - decimial.ROUND_CEILING (towards Infinity)
+            - decimal.ROUND_CEILING (towards Infinity)
-            - decimial.ROUND_DOWN (towards zero)
+            - decimal.ROUND_DOWN (towards zero)
-            - decimial.ROUND_FLOOR (towards -Infinity)
+            - decimal.ROUND_FLOOR (towards -Infinity)
-            - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero)
+            - decimal.ROUND_HALF_DOWN (to nearest with ties going towards zero)
-            - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer)
+            - decimal.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer)
-            - decimial.ROUND_HALF_UP (to nearest with ties going away from zero)
+            - decimal.ROUND_HALF_UP (to nearest with ties going away from zero)
-            - decimial.ROUND_UP (away from zero)
+            - decimal.ROUND_UP (away from zero)
-            - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
+            - decimal.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
 
             Defaults to: ``decimal.ROUND_HALF_UP``
 
@@ -347,6 +354,11 @@ class BooleanField(BaseField):
 class DateTimeField(BaseField):
     """A datetime field.
 
+    Uses the python-dateutil library if available alternatively use time.strptime
+    to parse the dates.  Note: python-dateutil's parser is fully featured and when
+    installed you can utilise it to convert varing types of date formats into valid
+    python datetime objects.
+
     Note: Microseconds are rounded to the nearest millisecond.
       Pre UTC microsecond support is effecively broken.
       Use :class:`~mongoengine.fields.ComplexDateTimeField` if you
@@ -354,13 +366,11 @@ class DateTimeField(BaseField):
     """
 
     def validate(self, value):
-        if not isinstance(value, (datetime.datetime, datetime.date)):
+        new_value = self.to_mongo(value)
+        if not isinstance(new_value, (datetime.datetime, datetime.date)):
             self.error(u'cannot parse date "%s"' % value)
 
     def to_mongo(self, value):
-        return self.prepare_query_value(None, value)
-
-    def prepare_query_value(self, op, value):
         if value is None:
             return value
         if isinstance(value, datetime.datetime):
@@ -370,8 +380,16 @@ class DateTimeField(BaseField):
         if callable(value):
             return value()
 
+        if not isinstance(value, basestring):
+            return None
+
         # Attempt to parse a datetime:
-        # value = smart_str(value)
+        if dateutil:
+            try:
+                return dateutil.parser.parse(value)
+            except ValueError:
+                return None
+
         # split usecs, because they are not recognized by strptime.
         if '.' in value:
             try:
@@ -396,6 +414,9 @@ class DateTimeField(BaseField):
                 except ValueError:
                     return None
 
+    def prepare_query_value(self, op, value):
+        return self.to_mongo(value)
+
 
 class ComplexDateTimeField(StringField):
     """
@@ -603,7 +624,9 @@ class DynamicField(BaseField):
             cls = value.__class__
             val = value.to_mongo()
             # If we its a document thats not inherited add _cls
-            if (isinstance(value, (Document, EmbeddedDocument))):
+            if (isinstance(value, Document)):
+                val = {"_ref": value.to_dbref(), "_cls": cls.__name__}
+            if (isinstance(value, EmbeddedDocument)):
                 val['_cls'] = cls.__name__
             return val
 
@@ -624,6 +647,15 @@ class DynamicField(BaseField):
             value = [v for k, v in sorted(data.iteritems(), key=itemgetter(0))]
         return value
 
+    def to_python(self, value):
+        if isinstance(value, dict) and '_cls' in value:
+            doc_cls = get_document(value['_cls'])
+            if '_ref' in value:
+                value = doc_cls._get_db().dereference(value['_ref'])
+            return doc_cls._from_son(value)
+
+        return super(DynamicField, self).to_python(value)
+
     def lookup_member(self, member_name):
         return member_name
 
@@ -748,6 +780,10 @@ class DictField(ComplexBaseField):
 
         if op in match_operators and isinstance(value, basestring):
             return StringField().prepare_query_value(op, value)
+
+        if hasattr(self.field, 'field'):
+            return self.field.prepare_query_value(op, value)
+
         return super(DictField, self).prepare_query_value(op, value)
 
 
@@ -1062,6 +1098,10 @@ class GridFSProxy(object):
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.grid_id)
 
+    def __str__(self):
+        name = getattr(self.get(), 'filename', self.grid_id) if self.get() else '(no file)'
+        return '<%s: %s>' % (self.__class__.__name__, name)
+
     def __eq__(self, other):
         if isinstance(other, GridFSProxy):
             return ((self.grid_id == other.grid_id) and
@@ -1169,9 +1209,7 @@ class FileField(BaseField):
         # Check if a file already exists for this model
         grid_file = instance._data.get(self.name)
         if not isinstance(grid_file, self.proxy_class):
-            grid_file = self.proxy_class(key=self.name, instance=instance,
+            grid_file = self.get_proxy_obj(key=self.name, instance=instance)
-                                         db_alias=self.db_alias,
-                                         collection_name=self.collection_name)
             instance._data[self.name] = grid_file
 
         if not grid_file.key:
@@ -1193,14 +1231,23 @@ class FileField(BaseField):
                     pass
 
             # Create a new proxy object as we don't already have one
-            instance._data[key] = self.proxy_class(key=key, instance=instance,
+            instance._data[key] = self.get_proxy_obj(key=key, instance=instance)
-                                                   collection_name=self.collection_name)
             instance._data[key].put(value)
         else:
             instance._data[key] = value
 
         instance._mark_as_changed(key)
 
+    def get_proxy_obj(self, key, instance, db_alias=None, collection_name=None):
+        if db_alias is None:
+            db_alias = self.db_alias
+        if collection_name is None:
+            collection_name = self.collection_name
+
+        return self.proxy_class(key=key, instance=instance,
+                                db_alias=db_alias,
+                                collection_name=collection_name)
+
     def to_mongo(self, value):
         # Store the GridFS file id in MongoDB
         if isinstance(value, self.proxy_class) and value.grid_id is not None:
@@ -1233,12 +1280,15 @@ class ImageGridFsProxy(GridFSProxy):
         applying field properties (size, thumbnail_size)
         """
         field = self.instance._fields[self.key]
+        # Handle nested fields
+        if hasattr(field, 'field') and isinstance(field.field, FileField):
+            field = field.field
 
         try:
             img = Image.open(file_obj)
             img_format = img.format
-        except:
+        except Exception, e:
-            raise ValidationError('Invalid image')
+            raise ValidationError('Invalid image: %s' % e)
 
         if (field.size and (img.size[0] > field.size['width'] or
                             img.size[1] > field.size['height'])):

mongoengine/queryset/field_list.py

@@ -55,7 +55,8 @@ class QueryFieldList(object):
 
         if self.always_include:
             if self.value is self.ONLY and self.fields:
-                self.fields = self.fields.union(self.always_include)
+                if sorted(self.slice.keys()) != sorted(self.fields):
+                    self.fields = self.fields.union(self.always_include)
             else:
                 self.fields -= self.always_include
 

mongoengine/queryset/transform.py

@@ -43,11 +43,11 @@ def query(_doc_cls=None, _field_operation=False, **query):
         parts = [part for part in parts if not part.isdigit()]
         # Check for an operator and transform to mongo-style if there is
         op = None
-        if parts[-1] in MATCH_OPERATORS:
+        if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
             op = parts.pop()
 
         negate = False
-        if parts[-1] == 'not':
+        if len(parts) > 1 and parts[-1] == 'not':
             parts.pop()
             negate = True
 
@@ -95,6 +95,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
                 value = _geo_operator(field, op, value)
             elif op in CUSTOM_OPERATORS:
                 if op == 'match':
+                    value = field.prepare_query_value(op, value)
                     value = {"$elemMatch": value}
                 else:
                     NotImplementedError("Custom method '%s' has not "
@@ -181,6 +182,7 @@ def update(_doc_cls=None, **update):
             parts = []
 
             cleaned_fields = []
+            appended_sub_field = False
             for field in fields:
                 append_field = True
                 if isinstance(field, basestring):
@@ -192,21 +194,30 @@ def update(_doc_cls=None, **update):
                 else:
                     parts.append(field.db_field)
                 if append_field:
+                    appended_sub_field = False
                     cleaned_fields.append(field)
+                    if hasattr(field, 'field'):
+                        cleaned_fields.append(field.field)
+                        appended_sub_field = True
 
             # Convert value to proper value
-            field = cleaned_fields[-1]
+            if appended_sub_field:
+                field = cleaned_fields[-2]
+            else:
+                field = cleaned_fields[-1]
 
             if op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
                 value = [field.prepare_query_value(op, v) for v in value]
-            elif op == 'addToSet':
+            elif op in ('addToSet', 'setOnInsert'):
                 if isinstance(value, (list, tuple, set)):
                     value = [field.prepare_query_value(op, v) for v in value]
                 elif field.required or value is not None:
                     value = field.prepare_query_value(op, value)
+            elif op == "unset":
+                value = 1
 
         if match:
             match = '$' + match
@@ -220,11 +231,24 @@ def update(_doc_cls=None, **update):
 
         if 'pull' in op and '.' in key:
             # Dot operators don't work on pull operations
-            # it uses nested dict syntax
+            # unless they point to a list field
+            # Otherwise it uses nested dict syntax
             if op == 'pullAll':
                 raise InvalidQueryError("pullAll operations only support "
                                         "a single field depth")
 
+            # Look for the last list field and use dot notation until there
+            field_classes = [c.__class__ for c in cleaned_fields]
+            field_classes.reverse()
+            ListField = _import_class('ListField')
+            if ListField in field_classes:
+                # Join all fields via dot notation to the last ListField
+                # Then process as normal
+                last_listField = len(cleaned_fields) - field_classes.index(ListField)
+                key = ".".join(parts[:last_listField])
+                parts = parts[last_listField:]
+                parts.insert(0, key)
+
             parts.reverse()
             for key in parts:
                 value = {key: value}

mongoengine/queryset/visitor.py

@@ -23,6 +23,10 @@ class QNodeVisitor(object):
         return query
 
 
+class DuplicateQueryConditionsError(InvalidQueryError):
+    pass
+
+
 class SimplificationVisitor(QNodeVisitor):
     """Simplifies query trees by combinging unnecessary 'and' connection nodes
     into a single Q-object.
@@ -33,7 +37,11 @@ class SimplificationVisitor(QNodeVisitor):
             # The simplification only applies to 'simple' queries
             if all(isinstance(node, Q) for node in combination.children):
                 queries = [n.query for n in combination.children]
-                return Q(**self._query_conjunction(queries))
+                try:
+                    return Q(**self._query_conjunction(queries))
+                except DuplicateQueryConditionsError:
+                    # Cannot be simplified
+                    pass
         return combination
 
     def _query_conjunction(self, queries):
@@ -47,8 +55,7 @@ class SimplificationVisitor(QNodeVisitor):
             # to a single field
             intersection = ops.intersection(query_ops)
             if intersection:
-                msg = 'Duplicate query conditions: '
+                raise DuplicateQueryConditionsError()
-                raise InvalidQueryError(msg + ', '.join(intersection))
 
             query_ops.update(ops)
             combined_query.update(copy.deepcopy(query))
@@ -122,8 +129,7 @@ class QCombination(QNode):
             # If the child is a combination of the same type, we can merge its
             # children directly into this combinations children
             if isinstance(node, QCombination) and node.operation == operation:
-                # self.children += node.children
+                self.children += node.children
-                self.children.append(node)
             else:
                 self.children.append(node)
 

mongoengine/signals.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
-__all__ = ['pre_init', 'post_init', 'pre_save', 'post_save',
+__all__ = ['pre_init', 'post_init', 'pre_save', 'pre_save_post_validation',
-           'pre_delete', 'post_delete']
+           'post_save', 'pre_delete', 'post_delete']
 
 signals_available = False
 try:
@@ -39,6 +39,7 @@ _signals = Namespace()
 pre_init = _signals.signal('pre_init')
 post_init = _signals.signal('post_init')
 pre_save = _signals.signal('pre_save')
+pre_save_post_validation = _signals.signal('pre_save_post_validation')
 post_save = _signals.signal('post_save')
 pre_delete = _signals.signal('pre_delete')
 post_delete = _signals.signal('post_delete')

python-mongoengine.spec

@@ -5,7 +5,7 @@
 %define srcname mongoengine
 
 Name:           python-%{srcname}
-Version:        0.8.0
+Version:        0.8.4
 Release:        1%{?dist}
 Summary:        A Python Document-Object Mapper for working with MongoDB
 

setup.py

@@ -48,17 +48,15 @@ CLASSIFIERS = [
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
 
-extra_opts = {}
+extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])}
 if sys.version_info[0] == 3:
     extra_opts['use_2to3'] = True
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1']
-    extra_opts['packages'] = find_packages(exclude=('tests',))
     if "test" in sys.argv or "nosetests" in sys.argv:
-        extra_opts['packages'].append("tests")
+        extra_opts['packages'] = find_packages()
         extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil']
-    extra_opts['packages'] = find_packages(exclude=('tests',))
 
 setup(name='mongoengine',
       version=VERSION,

tests/document/class_methods.py

@@ -85,6 +85,153 @@ class ClassMethodsTest(unittest.TestCase):
         self.assertEqual(self.Person._meta['delete_rules'],
                          {(Job, 'employee'): NULLIFY})
 
+    def test_compare_indexes(self):
+        """ Ensure that the indexes are properly created and that
+        compare_indexes identifies the missing/extra indexes
+        """
+
+        class BlogPost(Document):
+            author = StringField()
+            title = StringField()
+            description = StringField()
+            tags = StringField()
+
+            meta = {
+                'indexes': [('author', 'title')]
+            }
+
+        BlogPost.drop_collection()
+
+        BlogPost.ensure_indexes()
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] })
+
+        BlogPost.ensure_index(['author', 'description'])
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [[('author', 1), ('description', 1)]] })
+
+        BlogPost._get_collection().drop_index('author_1_description_1')
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] })
+
+        BlogPost._get_collection().drop_index('author_1_title_1')
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [[('author', 1), ('title', 1)]], 'extra': [] })
+
+    def test_compare_indexes_inheritance(self):
+        """ Ensure that the indexes are properly created and that
+        compare_indexes identifies the missing/extra indexes for subclassed
+        documents (_cls included)
+        """
+
+        class BlogPost(Document):
+            author = StringField()
+            title = StringField()
+            description = StringField()
+
+            meta = {
+                'allow_inheritance': True
+            }
+
+        class BlogPostWithTags(BlogPost):
+            tags = StringField()
+            tag_list = ListField(StringField())
+
+            meta = {
+                'indexes': [('author', 'tags')]
+            }
+
+        BlogPost.drop_collection()
+
+        BlogPost.ensure_indexes()
+        BlogPostWithTags.ensure_indexes()
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] })
+
+        BlogPostWithTags.ensure_index(['author', 'tag_list'])
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [[('_cls', 1), ('author', 1), ('tag_list', 1)]] })
+
+        BlogPostWithTags._get_collection().drop_index('_cls_1_author_1_tag_list_1')
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] })
+
+        BlogPostWithTags._get_collection().drop_index('_cls_1_author_1_tags_1')
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [[('_cls', 1), ('author', 1), ('tags', 1)]], 'extra': [] })
+
+    def test_compare_indexes_multiple_subclasses(self):
+        """ Ensure that compare_indexes behaves correctly if called from a
+        class, which base class has multiple subclasses
+        """
+
+        class BlogPost(Document):
+            author = StringField()
+            title = StringField()
+            description = StringField()
+
+            meta = {
+                'allow_inheritance': True
+            }
+
+        class BlogPostWithTags(BlogPost):
+            tags = StringField()
+            tag_list = ListField(StringField())
+
+            meta = {
+                'indexes': [('author', 'tags')]
+            }
+
+        class BlogPostWithCustomField(BlogPost):
+            custom = DictField()
+
+            meta = {
+                'indexes': [('author', 'custom')]
+            }
+
+        BlogPost.ensure_indexes()
+        BlogPostWithTags.ensure_indexes()
+        BlogPostWithCustomField.ensure_indexes()
+
+        self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] })
+        self.assertEqual(BlogPostWithTags.compare_indexes(), { 'missing': [], 'extra': [] })
+        self.assertEqual(BlogPostWithCustomField.compare_indexes(), { 'missing': [], 'extra': [] })
+
+    def test_list_indexes_inheritance(self):
+        """ ensure that all of the indexes are listed regardless of the super-
+        or sub-class that we call it from
+        """
+
+        class BlogPost(Document):
+            author = StringField()
+            title = StringField()
+            description = StringField()
+
+            meta = {
+                'allow_inheritance': True
+            }
+
+        class BlogPostWithTags(BlogPost):
+            tags = StringField()
+
+            meta = {
+                'indexes': [('author', 'tags')]
+            }
+
+        class BlogPostWithTagsAndExtraText(BlogPostWithTags):
+            extra_text = StringField()
+
+            meta = {
+                'indexes': [('author', 'tags', 'extra_text')]
+            }
+
+        BlogPost.drop_collection()
+
+        BlogPost.ensure_indexes()
+        BlogPostWithTags.ensure_indexes()
+        BlogPostWithTagsAndExtraText.ensure_indexes()
+
+        self.assertEqual(BlogPost.list_indexes(),
+                         BlogPostWithTags.list_indexes())
+        self.assertEqual(BlogPost.list_indexes(),
+                         BlogPostWithTagsAndExtraText.list_indexes())
+        self.assertEqual(BlogPost.list_indexes(),
+                         [[('_cls', 1), ('author', 1), ('tags', 1)],
+                         [('_cls', 1), ('author', 1), ('tags', 1), ('extra_text', 1)],
+                         [(u'_id', 1)], [('_cls', 1)]])
+
     def test_register_delete_rule_inherited(self):
 
         class Vaccine(Document):

tests/document/delta.py

@@ -3,6 +3,7 @@ import sys
 sys.path[0:0] = [""]
 import unittest
 
+from bson import SON
 from mongoengine import *
 from mongoengine.connection import get_db
 
@@ -312,29 +313,24 @@ class DeltaTest(unittest.TestCase):
         self.circular_reference_deltas_2(DynamicDocument, Document)
         self.circular_reference_deltas_2(DynamicDocument, DynamicDocument)
 
-    def circular_reference_deltas_2(self, DocClass1, DocClass2):
+    def circular_reference_deltas_2(self, DocClass1, DocClass2, dbref=True):
 
         class Person(DocClass1):
             name = StringField()
-            owns = ListField(ReferenceField('Organization'))
+            owns = ListField(ReferenceField('Organization', dbref=dbref))
-            employer = ReferenceField('Organization')
+            employer = ReferenceField('Organization', dbref=dbref)
 
         class Organization(DocClass2):
             name = StringField()
-            owner = ReferenceField('Person')
+            owner = ReferenceField('Person', dbref=dbref)
-            employees = ListField(ReferenceField('Person'))
+            employees = ListField(ReferenceField('Person', dbref=dbref))
 
         Person.drop_collection()
         Organization.drop_collection()
 
-        person = Person(name="owner")
+        person = Person(name="owner").save()
-        person.save()
+        employee = Person(name="employee").save()
-
+        organization = Organization(name="company").save()
-        employee = Person(name="employee")
-        employee.save()
-
-        organization = Organization(name="company")
-        organization.save()
 
         person.owns.append(organization)
         organization.owner = person
@@ -354,6 +350,8 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(o.owner, p)
         self.assertEqual(e.employer, o)
 
+        return person, organization, employee
+
     def test_delta_db_field(self):
         self.delta_db_field(Document)
         self.delta_db_field(DynamicDocument)
@@ -613,13 +611,13 @@ class DeltaTest(unittest.TestCase):
         Person.drop_collection()
 
         p = Person(name="James", age=34)
-        self.assertEqual(p._delta(), ({'age': 34, 'name': 'James',
+        self.assertEqual(p._delta(), (
-                                       '_cls': 'Person'}, {}))
+            SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {}))
 
         p.doc = 123
         del(p.doc)
-        self.assertEqual(p._delta(), ({'age': 34, 'name': 'James',
+        self.assertEqual(p._delta(), (
-                                       '_cls': 'Person'}, {'doc': 1}))
+            SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {}))
 
         p = Person()
         p.name = "Dean"
@@ -631,14 +629,14 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(p._get_changed_fields(), ['age'])
         self.assertEqual(p._delta(), ({'age': 24}, {}))
 
-        p = self.Person.objects(age=22).get()
+        p = Person.objects(age=22).get()
         p.age = 24
         self.assertEqual(p.age, 24)
         self.assertEqual(p._get_changed_fields(), ['age'])
         self.assertEqual(p._delta(), ({'age': 24}, {}))
 
         p.save()
-        self.assertEqual(1, self.Person.objects(age=24).count())
+        self.assertEqual(1, Person.objects(age=24).count())
 
     def test_dynamic_delta(self):
 
@@ -685,6 +683,36 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc._get_changed_fields(), ['list_field'])
         self.assertEqual(doc._delta(), ({}, {'list_field': 1}))
 
+    def test_delta_with_dbref_true(self):
+        person, organization, employee = self.circular_reference_deltas_2(Document, Document, True)
+        employee.name = 'test'
+
+        self.assertEqual(organization._get_changed_fields(), [])
+
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertEqual({}, updates)
+
+        organization.employees.append(person)
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertTrue('employees' in updates)
+
+    def test_delta_with_dbref_false(self):
+        person, organization, employee = self.circular_reference_deltas_2(Document, Document, False)
+        employee.name = 'test'
+
+        self.assertEqual(organization._get_changed_fields(), [])
+
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertEqual({}, updates)
+
+        organization.employees.append(person)
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertTrue('employees' in updates)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/document/indexes.py

@@ -156,6 +156,25 @@ class IndexesTest(unittest.TestCase):
         self.assertEqual([{'fields': [('_cls', 1), ('title', 1)]}],
                          A._meta['index_specs'])
 
+    def test_index_no_cls(self):
+        """Ensure index specs are inhertited correctly"""
+
+        class A(Document):
+            title = StringField()
+            meta = {
+                'indexes': [
+                        {'fields': ('title',), 'cls': False},
+                ],
+                'allow_inheritance': True,
+                'index_cls': False
+                }
+
+        self.assertEqual([('title', 1)], A._meta['index_specs'][0]['fields'])
+        A._get_collection().drop_indexes()
+        A.ensure_indexes()
+        info = A._get_collection().index_information()
+        self.assertEqual(len(info.keys()), 2)
+
     def test_build_index_spec_is_not_destructive(self):
 
         class MyDoc(Document):

tests/document/inheritance.py

@@ -189,6 +189,41 @@ class InheritanceTest(unittest.TestCase):
         self.assertEqual(Employee._get_collection_name(),
                          Person._get_collection_name())
 
+    def test_indexes_and_multiple_inheritance(self):
+        """ Ensure that all of the indexes are created for a document with
+        multiple inheritance.
+        """
+
+        class A(Document):
+            a = StringField()
+
+            meta = {
+                'allow_inheritance': True,
+                'indexes': ['a']
+            }
+
+        class B(Document):
+            b = StringField()
+
+            meta = {
+                'allow_inheritance': True,
+                'indexes': ['b']
+            }
+
+        class C(A, B):
+            pass
+
+        A.drop_collection()
+        B.drop_collection()
+        C.drop_collection()
+
+        C.ensure_indexes()
+
+        self.assertEqual(
+            sorted([idx['key'] for idx in C._get_collection().index_information().values()]),
+            sorted([[(u'_cls', 1), (u'b', 1)], [(u'_id', 1)], [(u'_cls', 1), (u'a', 1)]])
+        )
+
     def test_polymorphic_queries(self):
         """Ensure that the correct subclasses are returned from a query
         """

tests/document/instance.py

@@ -9,7 +9,9 @@ import unittest
 import uuid
 
 from datetime import datetime
-from tests.fixtures import PickleEmbedded, PickleTest, PickleSignalsTest
+from bson import DBRef
+from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest,
+                            PickleDyanmicEmbedded, PickleDynamicTest)
 
 from mongoengine import *
 from mongoengine.errors import (NotRegistered, InvalidDocumentError,
@@ -442,6 +444,13 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(),
                          ['_cls', 'name', 'age', 'salary'])
 
+    def test_embedded_document_to_mongo_id(self):
+        class SubDoc(EmbeddedDocument):
+            id = StringField(required=True)
+
+        sub_doc = SubDoc(id="abc")
+        self.assertEqual(sub_doc.to_mongo().keys(), ['id'])
+
     def test_embedded_document(self):
         """Ensure that embedded documents are set up correctly.
         """
@@ -664,7 +673,7 @@ class InstanceTest(unittest.TestCase):
 
         p = Person.objects(name="Wilson Jr").get()
         p.parent.name = "Daddy Wilson"
-        p.save()
+        p.save(cascade=True)
 
         p1.reload()
         self.assertEqual(p1.name, p.parent.name)
@@ -683,14 +692,12 @@ class InstanceTest(unittest.TestCase):
 
         p2 = Person(name="Wilson Jr")
         p2.parent = p1
+        p1.name = "Daddy Wilson"
         p2.save(force_insert=True, cascade_kwargs={"force_insert": False})
 
-        p = Person.objects(name="Wilson Jr").get()
-        p.parent.name = "Daddy Wilson"
-        p.save()
-
         p1.reload()
-        self.assertEqual(p1.name, p.parent.name)
+        p2.reload()
+        self.assertEqual(p1.name, p2.parent.name)
 
     def test_save_cascade_meta_false(self):
 
@@ -765,6 +772,10 @@ class InstanceTest(unittest.TestCase):
         p.parent.name = "Daddy Wilson"
         p.save()
 
+        p1.reload()
+        self.assertNotEqual(p1.name, p.parent.name)
+
+        p.save(cascade=True)
         p1.reload()
         self.assertEqual(p1.name, p.parent.name)
 
@@ -1018,6 +1029,99 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person.age, 21)
         self.assertEqual(person.active, False)
 
+    def test_query_count_when_saving(self):
+        """Ensure references don't cause extra fetches when saving"""
+        class Organization(Document):
+            name = StringField()
+
+        class User(Document):
+            name = StringField()
+            orgs = ListField(ReferenceField('Organization'))
+
+        class Feed(Document):
+            name = StringField()
+
+        class UserSubscription(Document):
+            name = StringField()
+            user = ReferenceField(User)
+            feed = ReferenceField(Feed)
+
+        Organization.drop_collection()
+        User.drop_collection()
+        Feed.drop_collection()
+        UserSubscription.drop_collection()
+
+        o1 = Organization(name="o1").save()
+        o2 = Organization(name="o2").save()
+
+        u1 = User(name="Ross", orgs=[o1, o2]).save()
+        f1 = Feed(name="MongoEngine").save()
+
+        sub = UserSubscription(user=u1, feed=f1).save()
+
+        user = User.objects.first()
+        # Even if stored as ObjectId's internally mongoengine uses DBRefs
+        # As ObjectId's aren't automatically derefenced
+        self.assertTrue(isinstance(user._data['orgs'][0], DBRef))
+        self.assertTrue(isinstance(user.orgs[0], Organization))
+        self.assertTrue(isinstance(user._data['orgs'][0], Organization))
+
+        # Changing a value
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            sub = UserSubscription.objects.first()
+            self.assertEqual(q, 1)
+            sub.name = "Test Sub"
+            sub.save()
+            self.assertEqual(q, 2)
+
+        # Changing a value that will cascade
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            sub = UserSubscription.objects.first()
+            self.assertEqual(q, 1)
+            sub.user.name = "Test"
+            self.assertEqual(q, 2)
+            sub.save(cascade=True)
+            self.assertEqual(q, 3)
+
+        # Changing a value and one that will cascade
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            sub = UserSubscription.objects.first()
+            sub.name = "Test Sub 2"
+            self.assertEqual(q, 1)
+            sub.user.name = "Test 2"
+            self.assertEqual(q, 2)
+            sub.save(cascade=True)
+            self.assertEqual(q, 4)  # One for the UserSub and one for the User
+
+        # Saving with just the refs
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            sub = UserSubscription(user=u1.pk, feed=f1.pk)
+            self.assertEqual(q, 0)
+            sub.save()
+            self.assertEqual(q, 1)
+
+        # Saving with just the refs on a ListField
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            User(name="Bob", orgs=[o1.pk, o2.pk]).save()
+            self.assertEqual(q, 1)
+
+        # Saving new objects
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            user = User.objects.first()
+            self.assertEqual(q, 1)
+            feed = Feed.objects.first()
+            self.assertEqual(q, 2)
+            sub = UserSubscription(user=user, feed=feed)
+            self.assertEqual(q, 2)  # Check no change
+            sub.save()
+            self.assertEqual(q, 3)
+
     def test_set_unset_one_operation(self):
         """Ensure that $set and $unset actions are performed in the same
         operation.
@@ -1709,6 +1813,7 @@ class InstanceTest(unittest.TestCase):
 
         pickle_doc = PickleTest(number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleEmbedded()
+        pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
         pickle_doc.save()
 
         pickled_doc = pickle.dumps(pickle_doc)
@@ -1730,6 +1835,29 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(pickle_doc.string, "Two")
         self.assertEqual(pickle_doc.lists, ["1", "2", "3"])
 
+    def test_dynamic_document_pickle(self):
+
+        pickle_doc = PickleDynamicTest(name="test", number=1, string="One", lists=['1', '2'])
+        pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar")
+        pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
+
+        pickle_doc.save()
+
+        pickled_doc = pickle.dumps(pickle_doc)
+        resurrected = pickle.loads(pickled_doc)
+
+        self.assertEqual(resurrected, pickle_doc)
+        self.assertEqual(resurrected._fields_ordered,
+                         pickle_doc._fields_ordered)
+        self.assertEqual(resurrected._dynamic_fields.keys(),
+                         pickle_doc._dynamic_fields.keys())
+
+        self.assertEqual(resurrected.embedded, pickle_doc.embedded)
+        self.assertEqual(resurrected.embedded._fields_ordered,
+                         pickle_doc.embedded._fields_ordered)
+        self.assertEqual(resurrected.embedded._dynamic_fields.keys(),
+                         pickle_doc.embedded._dynamic_fields.keys())
+
     def test_picklable_on_signals(self):
         pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleEmbedded()
@@ -2192,6 +2320,16 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 42)
 
+    def test_mixed_creation_dynamic(self):
+        """Ensure that document may be created using mixed arguments.
+        """
+        class Person(DynamicDocument):
+            name = StringField()
+
+        person = Person("Test User", age=42)
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 42)
+
     def test_bad_mixed_creation(self):
         """Ensure that document gives correct error when duplicating arguments
         """

tests/document/json_serialisation.py

@@ -31,6 +31,10 @@ class TestJson(unittest.TestCase):
 
         doc = Doc(string="Hi", embedded_field=Embedded(string="Hi"))
 
+        doc_json = doc.to_json(sort_keys=True, separators=(',', ':'))
+        expected_json = """{"embedded_field":{"string":"Hi"},"string":"Hi"}"""
+        self.assertEqual(doc_json, expected_json)
+
         self.assertEqual(doc, Doc.from_json(doc.to_json()))
 
     def test_json_complex(self):

tests/fields/fields.py

@@ -6,6 +6,11 @@ import datetime
 import unittest
 import uuid
 
+try:
+    import dateutil
+except ImportError:
+    dateutil = None
+
 from decimal import Decimal
 
 from bson import Binary, DBRef, ObjectId
@@ -29,20 +34,137 @@ class FieldTest(unittest.TestCase):
         self.db.drop_collection('fs.files')
         self.db.drop_collection('fs.chunks')
 
-    def test_default_values(self):
+    def test_default_values_nothing_set(self):
         """Ensure that default field values are used when creating a document.
         """
         class Person(Document):
             name = StringField()
-            age = IntField(default=30, help_text="Your real age")
+            age = IntField(default=30, required=False)
-            userid = StringField(default=lambda: 'test', verbose_name="User Identity")
+            userid = StringField(default=lambda: 'test', required=True)
+            created = DateTimeField(default=datetime.datetime.utcnow)
 
-        person = Person(name='Test Person')
+        person = Person(name="Ross")
-        self.assertEqual(person._data['age'], 30)
+
-        self.assertEqual(person._data['userid'], 'test')
+        # Confirm saving now would store values
-        self.assertEqual(person._fields['name'].help_text, None)
+        data_to_be_saved = sorted(person.to_mongo().keys())
-        self.assertEqual(person._fields['age'].help_text, "Your real age")
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'name', 'userid'])
-        self.assertEqual(person._fields['userid'].verbose_name, "User Identity")
+
+        self.assertTrue(person.validate() is None)
+
+        self.assertEqual(person.name, person.name)
+        self.assertEqual(person.age, person.age)
+        self.assertEqual(person.userid, person.userid)
+        self.assertEqual(person.created, person.created)
+
+        self.assertEqual(person._data['name'], person.name)
+        self.assertEqual(person._data['age'], person.age)
+        self.assertEqual(person._data['userid'], person.userid)
+        self.assertEqual(person._data['created'], person.created)
+
+        # Confirm introspection changes nothing
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'name', 'userid'])
+
+    def test_default_values_set_to_None(self):
+        """Ensure that default field values are used when creating a document.
+        """
+        class Person(Document):
+            name = StringField()
+            age = IntField(default=30, required=False)
+            userid = StringField(default=lambda: 'test', required=True)
+            created = DateTimeField(default=datetime.datetime.utcnow)
+
+        # Trying setting values to None
+        person = Person(name=None, age=None, userid=None, created=None)
+
+        # Confirm saving now would store values
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
+
+        self.assertTrue(person.validate() is None)
+
+        self.assertEqual(person.name, person.name)
+        self.assertEqual(person.age, person.age)
+        self.assertEqual(person.userid, person.userid)
+        self.assertEqual(person.created, person.created)
+
+        self.assertEqual(person._data['name'], person.name)
+        self.assertEqual(person._data['age'], person.age)
+        self.assertEqual(person._data['userid'], person.userid)
+        self.assertEqual(person._data['created'], person.created)
+
+        # Confirm introspection changes nothing
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
+
+    def test_default_values_when_setting_to_None(self):
+        """Ensure that default field values are used when creating a document.
+        """
+        class Person(Document):
+            name = StringField()
+            age = IntField(default=30, required=False)
+            userid = StringField(default=lambda: 'test', required=True)
+            created = DateTimeField(default=datetime.datetime.utcnow)
+
+        person = Person()
+        person.name = None
+        person.age = None
+        person.userid = None
+        person.created = None
+
+        # Confirm saving now would store values
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
+
+        self.assertTrue(person.validate() is None)
+
+        self.assertEqual(person.name, person.name)
+        self.assertEqual(person.age, person.age)
+        self.assertEqual(person.userid, person.userid)
+        self.assertEqual(person.created, person.created)
+
+        self.assertEqual(person._data['name'], person.name)
+        self.assertEqual(person._data['age'], person.age)
+        self.assertEqual(person._data['userid'], person.userid)
+        self.assertEqual(person._data['created'], person.created)
+
+        # Confirm introspection changes nothing
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
+
+    def test_default_values_when_deleting_value(self):
+        """Ensure that default field values are used when creating a document.
+        """
+        class Person(Document):
+            name = StringField()
+            age = IntField(default=30, required=False)
+            userid = StringField(default=lambda: 'test', required=True)
+            created = DateTimeField(default=datetime.datetime.utcnow)
+
+        person = Person(name="Ross")
+        del person.name
+        del person.age
+        del person.userid
+        del person.created
+
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
+
+        self.assertTrue(person.validate() is None)
+
+        self.assertEqual(person.name, person.name)
+        self.assertEqual(person.age, person.age)
+        self.assertEqual(person.userid, person.userid)
+        self.assertEqual(person.created, person.created)
+
+        self.assertEqual(person._data['name'], person.name)
+        self.assertEqual(person._data['age'], person.age)
+        self.assertEqual(person._data['userid'], person.userid)
+        self.assertEqual(person._data['created'], person.created)
+
+        # Confirm introspection changes nothing
+        data_to_be_saved = sorted(person.to_mongo().keys())
+        self.assertEqual(data_to_be_saved, ['age', 'created', 'userid'])
 
     def test_required_values(self):
         """Ensure that required field constraints are enforced.
@@ -403,9 +525,16 @@ class FieldTest(unittest.TestCase):
         log.time = datetime.date.today()
         log.validate()
 
+        log.time = datetime.datetime.now().isoformat(' ')
+        log.validate()
+
+        if dateutil:
+            log.time = datetime.datetime.now().isoformat('T')
+            log.validate()
+
         log.time = -1
         self.assertRaises(ValidationError, log.validate)
-        log.time = '1pm'
+        log.time = 'ABC'
         self.assertRaises(ValidationError, log.validate)
 
     def test_datetime_tz_aware_mark_as_changed(self):
@@ -482,6 +611,66 @@ class FieldTest(unittest.TestCase):
 
         LogEntry.drop_collection()
 
+    def test_datetime_usage(self):
+        """Tests for regular datetime fields"""
+        class LogEntry(Document):
+            date = DateTimeField()
+
+        LogEntry.drop_collection()
+
+        d1 = datetime.datetime(1970, 01, 01, 00, 00, 01)
+        log = LogEntry()
+        log.date = d1
+        log.validate()
+        log.save()
+
+        for query in (d1, d1.isoformat(' ')):
+            log1 = LogEntry.objects.get(date=query)
+            self.assertEqual(log, log1)
+
+        if dateutil:
+            log1 = LogEntry.objects.get(date=d1.isoformat('T'))
+            self.assertEqual(log, log1)
+
+        LogEntry.drop_collection()
+
+        # create 60 log entries
+        for i in xrange(1950, 2010):
+            d = datetime.datetime(i, 01, 01, 00, 00, 01)
+            LogEntry(date=d).save()
+
+        self.assertEqual(LogEntry.objects.count(), 60)
+
+        # Test ordering
+        logs = LogEntry.objects.order_by("date")
+        count = logs.count()
+        i = 0
+        while i == count - 1:
+            self.assertTrue(logs[i].date <= logs[i + 1].date)
+            i += 1
+
+        logs = LogEntry.objects.order_by("-date")
+        count = logs.count()
+        i = 0
+        while i == count - 1:
+            self.assertTrue(logs[i].date >= logs[i + 1].date)
+            i += 1
+
+        # Test searching
+        logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980, 1, 1))
+        self.assertEqual(logs.count(), 30)
+
+        logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980, 1, 1))
+        self.assertEqual(logs.count(), 30)
+
+        logs = LogEntry.objects.filter(
+            date__lte=datetime.datetime(2011, 1, 1),
+            date__gte=datetime.datetime(2000, 1, 1),
+        )
+        self.assertEqual(logs.count(), 10)
+
+        LogEntry.drop_collection()
+
     def test_complexdatetime_storage(self):
         """Tests for complex datetime fields - which can handle microseconds
         without rounding.
@@ -808,6 +997,53 @@ class FieldTest(unittest.TestCase):
 
         self.assertRaises(ValidationError, e.save)
 
+    def test_complex_field_same_value_not_changed(self):
+        """
+        If a complex field is set to the same value, it should not be marked as
+        changed.
+        """
+        class Simple(Document):
+            mapping = ListField()
+
+        Simple.drop_collection()
+        e = Simple().save()
+        e.mapping = []
+        self.assertEqual([], e._changed_fields)
+
+        class Simple(Document):
+            mapping = DictField()
+
+        Simple.drop_collection()
+        e = Simple().save()
+        e.mapping = {}
+        self.assertEqual([], e._changed_fields)
+
+    def test_slice_marks_field_as_changed(self):
+
+        class Simple(Document):
+            widgets = ListField()
+
+        simple = Simple(widgets=[1, 2, 3, 4]).save()
+        simple.widgets[:3] = []
+        self.assertEqual(['widgets'], simple._changed_fields)
+        simple.save()
+
+        simple = simple.reload()
+        self.assertEqual(simple.widgets, [4])
+
+    def test_del_slice_marks_field_as_changed(self):
+
+        class Simple(Document):
+            widgets = ListField()
+
+        simple = Simple(widgets=[1, 2, 3, 4]).save()
+        del simple.widgets[:3]
+        self.assertEqual(['widgets'], simple._changed_fields)
+        simple.save()
+
+        simple = simple.reload()
+        self.assertEqual(simple.widgets, [4])
+
     def test_list_field_complex(self):
         """Ensure that the list fields can handle the complex types."""
 
@@ -1929,7 +2165,7 @@ class FieldTest(unittest.TestCase):
         self.db['mongoengine.counters'].drop()
 
         self.assertEqual(Person.id.get_next_value(), '1')
-        
+
     def test_sequence_field_sequence_name(self):
         class Person(Document):
             id = SequenceField(primary_key=True, sequence_name='jelly')
@@ -2238,6 +2474,78 @@ class FieldTest(unittest.TestCase):
         user = User(email='me@example.com')
         self.assertTrue(user.validate() is None)
 
+    def test_tuples_as_tuples(self):
+        """
+        Ensure that tuples remain tuples when they are
+        inside a ComplexBaseField
+        """
+        from mongoengine.base import BaseField
+
+        class EnumField(BaseField):
+
+            def __init__(self, **kwargs):
+                super(EnumField, self).__init__(**kwargs)
+
+            def to_mongo(self, value):
+                return value
+
+            def to_python(self, value):
+                return tuple(value)
+
+        class TestDoc(Document):
+            items = ListField(EnumField())
+
+        TestDoc.drop_collection()
+        tuples = [(100, 'Testing')]
+        doc = TestDoc()
+        doc.items = tuples
+        doc.save()
+        x = TestDoc.objects().get()
+        self.assertTrue(x is not None)
+        self.assertTrue(len(x.items) == 1)
+        self.assertTrue(tuple(x.items[0]) in tuples)
+        self.assertTrue(x.items[0] in tuples)
+
+    def test_dynamic_fields_class(self):
+
+        class Doc2(Document):
+            field_1 = StringField(db_field='f')
+
+        class Doc(Document):
+            my_id = IntField(required=True, unique=True, primary_key=True)
+            embed_me = DynamicField(db_field='e')
+            field_x = StringField(db_field='x')
+
+        Doc.drop_collection()
+        Doc2.drop_collection()
+
+        doc2 = Doc2(field_1="hello")
+        doc = Doc(my_id=1, embed_me=doc2, field_x="x")
+        self.assertRaises(OperationError, doc.save)
+
+        doc2.save()
+        doc.save()
+
+        doc = Doc.objects.get()
+        self.assertEqual(doc.embed_me.field_1, "hello")
+
+    def test_dynamic_fields_embedded_class(self):
+
+        class Embed(EmbeddedDocument):
+            field_1 = StringField(db_field='f')
+
+        class Doc(Document):
+            my_id = IntField(required=True, unique=True, primary_key=True)
+            embed_me = DynamicField(db_field='e')
+            field_x = StringField(db_field='x')
+
+        Doc.drop_collection()
+
+        Doc(my_id=1, embed_me=Embed(field_1="hello"), field_x="x").save()
+
+        doc = Doc.objects.get()
+        self.assertEqual(doc.embed_me.field_1, "hello")
+
 
 if __name__ == '__main__':
     unittest.main()

tests/fields/file_tests.py

@@ -14,6 +14,12 @@ from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.python_support import PY3, b, StringIO
 
+try:
+    from PIL import Image
+    HAS_PIL = True
+except ImportError:
+    HAS_PIL = False
+
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png')
 TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), 'mongodb_leaf.png')
 
@@ -47,11 +53,12 @@ class FileTest(unittest.TestCase):
         content_type = 'text/plain'
 
         putfile = PutFile()
-        putfile.the_file.put(text, content_type=content_type)
+        putfile.the_file.put(text, content_type=content_type, filename="hello")
         putfile.save()
 
         result = PutFile.objects.first()
         self.assertTrue(putfile == result)
+        self.assertEqual("%s" % result.the_file, "<GridFSProxy: hello>")
         self.assertEqual(result.the_file.read(), text)
         self.assertEqual(result.the_file.content_type, content_type)
         result.the_file.delete()  # Remove file from GridFS
@@ -255,14 +262,25 @@ class FileTest(unittest.TestCase):
         self.assertFalse(test_file.the_file in [{"test": 1}])
 
     def test_image_field(self):
-        if PY3:
+        if not HAS_PIL:
-            raise SkipTest('PIL does not have Python 3 support')
+            raise SkipTest('PIL not installed')
 
         class TestImage(Document):
             image = ImageField()
 
         TestImage.drop_collection()
 
+        with tempfile.TemporaryFile() as f:
+            f.write(b("Hello World!"))
+            f.flush()
+
+            t = TestImage()
+            try:
+                t.image.put(f)
+                self.fail("Should have raised an invalidation error")
+            except ValidationError, e:
+                self.assertEquals("%s" % e, "Invalid image: cannot identify image file")
+
         t = TestImage()
         t.image.put(open(TEST_IMAGE_PATH, 'rb'))
         t.save()
@@ -278,8 +296,8 @@ class FileTest(unittest.TestCase):
         t.image.delete()
 
     def test_image_field_reassigning(self):
-        if PY3:
+        if not HAS_PIL:
-            raise SkipTest('PIL does not have Python 3 support')
+            raise SkipTest('PIL not installed')
 
         class TestFile(Document):
             the_file = ImageField()
@@ -294,8 +312,8 @@ class FileTest(unittest.TestCase):
         self.assertEqual(test_file.the_file.size, (45, 101))
 
     def test_image_field_resize(self):
-        if PY3:
+        if not HAS_PIL:
-            raise SkipTest('PIL does not have Python 3 support')
+            raise SkipTest('PIL not installed')
 
         class TestImage(Document):
             image = ImageField(size=(185, 37))
@@ -317,8 +335,8 @@ class FileTest(unittest.TestCase):
         t.image.delete()
 
     def test_image_field_resize_force(self):
-        if PY3:
+        if not HAS_PIL:
-            raise SkipTest('PIL does not have Python 3 support')
+            raise SkipTest('PIL not installed')
 
         class TestImage(Document):
             image = ImageField(size=(185, 37, True))
@@ -340,8 +358,8 @@ class FileTest(unittest.TestCase):
         t.image.delete()
 
     def test_image_field_thumbnail(self):
-        if PY3:
+        if not HAS_PIL:
-            raise SkipTest('PIL does not have Python 3 support')
+            raise SkipTest('PIL not installed')
 
         class TestImage(Document):
             image = ImageField(thumbnail_size=(92, 18))
@@ -388,6 +406,14 @@ class FileTest(unittest.TestCase):
         self.assertEqual(test_file.the_file.read(),
                           b('Hello, World!'))
 
+        test_file = TestFile.objects.first()
+        test_file.the_file = b('HELLO, WORLD!')
+        test_file.save()
+
+        test_file = TestFile.objects.first()
+        self.assertEqual(test_file.the_file.read(),
+                          b('HELLO, WORLD!'))
+
     def test_copyable(self):
         class PutFile(Document):
             the_file = FileField()
@@ -407,6 +433,54 @@ class FileTest(unittest.TestCase):
         self.assertEqual(putfile, copy.copy(putfile))
         self.assertEqual(putfile, copy.deepcopy(putfile))
 
+    def test_get_image_by_grid_id(self):
+
+        if not HAS_PIL:
+            raise SkipTest('PIL not installed')
+
+        class TestImage(Document):
+
+            image1 = ImageField()
+            image2 = ImageField()
+
+        TestImage.drop_collection()
+
+        t = TestImage()
+        t.image1.put(open(TEST_IMAGE_PATH, 'rb'))
+        t.image2.put(open(TEST_IMAGE2_PATH, 'rb'))
+        t.save()
+
+        test = TestImage.objects.first()
+        grid_id = test.image1.grid_id
+
+        self.assertEqual(1, TestImage.objects(Q(image1=grid_id)
+                                              or Q(image2=grid_id)).count())
+
+    def test_complex_field_filefield(self):
+        """Ensure you can add meta data to file"""
+
+        class Animal(Document):
+            genus = StringField()
+            family = StringField()
+            photos = ListField(FileField())
+
+        Animal.drop_collection()
+        marmot = Animal(genus='Marmota', family='Sciuridae')
+
+        marmot_photo = open(TEST_IMAGE_PATH, 'rb')  # Retrieve a photo from disk
+
+        photos_field = marmot._fields['photos'].field
+        new_proxy = photos_field.get_proxy_obj('photos', marmot)
+        new_proxy.put(marmot_photo, content_type='image/jpeg', foo='bar')
+        marmot_photo.close()
+
+        marmot.photos.append(new_proxy)
+        marmot.save()
+
+        marmot = Animal.objects.get()
+        self.assertEqual(marmot.photos[0].content_type, 'image/jpeg')
+        self.assertEqual(marmot.photos[0].foo, 'bar')
+        self.assertEqual(marmot.photos[0].get().length, 8313)
 
 if __name__ == '__main__':
     unittest.main()

tests/fixtures.py

@@ -17,6 +17,14 @@ class PickleTest(Document):
     photo = FileField()
 
 
+class PickleDyanmicEmbedded(DynamicEmbeddedDocument):
+    date = DateTimeField(default=datetime.now)
+
+
+class PickleDynamicTest(DynamicDocument):
+    number = IntField()
+
+
 class PickleSignalsTest(Document):
     number = IntField()
     string = StringField(choices=(('One', '1'), ('Two', '2')))

tests/queryset/field_list.py

@@ -162,6 +162,10 @@ class OnlyExcludeAllTest(unittest.TestCase):
         self.assertEqual(obj.name, person.name)
         self.assertEqual(obj.age, person.age)
 
+        obj = self.Person.objects.only(*('id', 'name',)).get()
+        self.assertEqual(obj.name, person.name)
+        self.assertEqual(obj.age, None)
+
         # Check polymorphism still works
         class Employee(self.Person):
             salary = IntField(db_field='wage')
@@ -395,5 +399,28 @@ class OnlyExcludeAllTest(unittest.TestCase):
         numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get()
         self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1])
 
+
+    def test_exclude_from_subclasses_docs(self):
+
+        class Base(Document):
+            username = StringField()
+
+            meta = {'allow_inheritance': True}
+
+        class Anon(Base):
+            anon = BooleanField()
+
+        class User(Base):
+            password = StringField()
+            wibble = StringField()
+
+        Base.drop_collection()
+        User(username="mongodb", password="secret").save()
+
+        user = Base.objects().exclude("password", "wibble").first()
+        self.assertEqual(user.password, None)
+
+        self.assertRaises(LookUpError, Base.objects.exclude, "made_up")
+
 if __name__ == '__main__':
     unittest.main()

tests/queryset/queryset.py

@@ -30,12 +30,17 @@ class QuerySetTest(unittest.TestCase):
     def setUp(self):
         connect(db='mongoenginetest')
 
+        class PersonMeta(EmbeddedDocument):
+            weight = IntField()
+
         class Person(Document):
             name = StringField()
             age = IntField()
+            person_meta = EmbeddedDocumentField(PersonMeta)
             meta = {'allow_inheritance': True}
 
         Person.drop_collection()
+        self.PersonMeta = PersonMeta
         self.Person = Person
 
     def test_initialisation(self):
@@ -536,6 +541,23 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(club.members['John']['gender'], "F")
         self.assertEqual(club.members['John']['age'], 14)
 
+    def test_update_results(self):
+        self.Person.drop_collection()
+
+        result = self.Person(name="Bob", age=25).update(upsert=True, full_result=True)
+        self.assertTrue(isinstance(result, dict))
+        self.assertTrue("upserted" in result)
+        self.assertFalse(result["updatedExisting"])
+
+        bob = self.Person.objects.first()
+        result = bob.update(set__age=30, full_result=True)
+        self.assertTrue(isinstance(result, dict))
+        self.assertTrue(result["updatedExisting"])
+
+        self.Person(name="Bob", age=20).save()
+        result = self.Person.objects(name="Bob").update(set__name="bobby", multi=True)
+        self.assertEqual(result, 2)
+
     def test_upsert(self):
         self.Person.drop_collection()
 
@@ -545,6 +567,15 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual("Bob", bob.name)
         self.assertEqual(30, bob.age)
 
+    def test_upsert_one(self):
+        self.Person.drop_collection()
+
+        self.Person.objects(name="Bob", age=30).update_one(upsert=True)
+
+        bob = self.Person.objects.first()
+        self.assertEqual("Bob", bob.name)
+        self.assertEqual(30, bob.age)
+
     def test_set_on_insert(self):
         self.Person.drop_collection()
 
@@ -622,14 +653,13 @@ class QuerySetTest(unittest.TestCase):
             self.assertEqual(q, 1)  # 1 for the insert
 
         Blog.drop_collection()
+        Blog.ensure_indexes()
+
         with query_counter() as q:
             self.assertEqual(q, 0)
 
-            Blog.ensure_indexes()
-            self.assertEqual(q, 1)
-
             Blog.objects.insert(blogs)
-            self.assertEqual(q, 3)  # 1 for insert, and 1 for in bulk fetch (3 in total)
+            self.assertEqual(q, 2)  # 1 for insert, and 1 for in bulk fetch
 
         Blog.drop_collection()
 
@@ -1467,9 +1497,6 @@ class QuerySetTest(unittest.TestCase):
 
     def test_pull_nested(self):
 
-        class User(Document):
-            name = StringField()
-
         class Collaborator(EmbeddedDocument):
             user = StringField()
 
@@ -1484,8 +1511,7 @@ class QuerySetTest(unittest.TestCase):
         Site.drop_collection()
 
         c = Collaborator(user='Esteban')
-        s = Site(name="test", collaborators=[c])
+        s = Site(name="test", collaborators=[c]).save()
-        s.save()
 
         Site.objects(id=s.id).update_one(pull__collaborators__user='Esteban')
         self.assertEqual(Site.objects.first().collaborators, [])
@@ -1495,6 +1521,71 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(InvalidQueryError, pull_all)
 
+    def test_pull_from_nested_embedded(self):
+
+        class User(EmbeddedDocument):
+            name = StringField()
+
+            def __unicode__(self):
+                return '%s' % self.name
+
+        class Collaborator(EmbeddedDocument):
+            helpful = ListField(EmbeddedDocumentField(User))
+            unhelpful = ListField(EmbeddedDocumentField(User))
+
+        class Site(Document):
+            name = StringField(max_length=75, unique=True, required=True)
+            collaborators = EmbeddedDocumentField(Collaborator)
+
+
+        Site.drop_collection()
+
+        c = User(name='Esteban')
+        f = User(name='Frank')
+        s = Site(name="test", collaborators=Collaborator(helpful=[c], unhelpful=[f])).save()
+
+        Site.objects(id=s.id).update_one(pull__collaborators__helpful=c)
+        self.assertEqual(Site.objects.first().collaborators['helpful'], [])
+
+        Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'name': 'Frank'})
+        self.assertEqual(Site.objects.first().collaborators['unhelpful'], [])
+
+        def pull_all():
+            Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__name=['Ross'])
+
+        self.assertRaises(InvalidQueryError, pull_all)
+
+    def test_pull_from_nested_mapfield(self):
+
+        class Collaborator(EmbeddedDocument):
+            user = StringField()
+
+            def __unicode__(self):
+                return '%s' % self.user
+
+        class Site(Document):
+            name = StringField(max_length=75, unique=True, required=True)
+            collaborators = MapField(ListField(EmbeddedDocumentField(Collaborator)))
+
+
+        Site.drop_collection()
+
+        c = Collaborator(user='Esteban')
+        f = Collaborator(user='Frank')
+        s = Site(name="test", collaborators={'helpful':[c],'unhelpful':[f]})
+        s.save()
+
+        Site.objects(id=s.id).update_one(pull__collaborators__helpful__user='Esteban')
+        self.assertEqual(Site.objects.first().collaborators['helpful'], [])
+
+        Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'user':'Frank'})
+        self.assertEqual(Site.objects.first().collaborators['unhelpful'], [])
+
+        def pull_all():
+            Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__user=['Ross'])
+
+        self.assertRaises(InvalidQueryError, pull_all)
+
     def test_update_one_pop_generic_reference(self):
 
         class BlogTag(Document):
@@ -1588,6 +1679,32 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(message.authors[1].name, "Ross")
         self.assertEqual(message.authors[2].name, "Adam")
 
+    def test_reload_embedded_docs_instance(self):
+
+        class SubDoc(EmbeddedDocument):
+            val = IntField()
+
+        class Doc(Document):
+            embedded = EmbeddedDocumentField(SubDoc)
+
+        doc = Doc(embedded=SubDoc(val=0)).save()
+        doc.reload()
+
+        self.assertEqual(doc.pk, doc.embedded._instance.pk)
+
+    def test_reload_list_embedded_docs_instance(self):
+
+        class SubDoc(EmbeddedDocument):
+            val = IntField()
+
+        class Doc(Document):
+            embedded = ListField(EmbeddedDocumentField(SubDoc))
+
+        doc = Doc(embedded=[SubDoc(val=0)]).save()
+        doc.reload()
+
+        self.assertEqual(doc.pk, doc.embedded[0]._instance.pk)
+
     def test_order_by(self):
         """Ensure that QuerySets may be ordered.
         """
@@ -2157,6 +2274,19 @@ class QuerySetTest(unittest.TestCase):
         self.Person(name='ageless person').save()
         self.assertEqual(int(self.Person.objects.average('age')), avg)
 
+        # dot notation
+        self.Person(name='person meta', person_meta=self.PersonMeta(weight=0)).save()
+        self.assertAlmostEqual(int(self.Person.objects.average('person_meta.weight')), 0)
+
+        for i, weight in enumerate(ages):
+            self.Person(name='test meta%i', person_meta=self.PersonMeta(weight=weight)).save()
+
+        self.assertAlmostEqual(int(self.Person.objects.average('person_meta.weight')), avg)
+
+        self.Person(name='test meta none').save()
+        self.assertEqual(int(self.Person.objects.average('person_meta.weight')), avg)
+
+
     def test_sum(self):
         """Ensure that field can be summed over correctly.
         """
@@ -2169,6 +2299,153 @@ class QuerySetTest(unittest.TestCase):
         self.Person(name='ageless person').save()
         self.assertEqual(int(self.Person.objects.sum('age')), sum(ages))
 
+        for i, age in enumerate(ages):
+            self.Person(name='test meta%s' % i, person_meta=self.PersonMeta(weight=age)).save()
+
+        self.assertEqual(int(self.Person.objects.sum('person_meta.weight')), sum(ages))
+
+        self.Person(name='weightless person').save()
+        self.assertEqual(int(self.Person.objects.sum('age')), sum(ages))
+
+    def test_embedded_average(self):
+        class Pay(EmbeddedDocument):
+            value = DecimalField()
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(value=150)).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(value=530)).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(value=165)).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(value=115)).save()
+
+        self.assertEqual(
+            Doc.objects.average('pay.value'),
+            240)
+
+    def test_embedded_array_average(self):
+        class Pay(EmbeddedDocument):
+            values = ListField(DecimalField())
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(values=[150, 100])).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(values=[530, 100])).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(values=[165, 100])).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(values=[115, 100])).save()
+
+        self.assertEqual(
+            Doc.objects.average('pay.values'),
+            170)
+
+    def test_array_average(self):
+        class Doc(Document):
+            values = ListField(DecimalField())
+
+        Doc.drop_collection()
+
+        Doc(values=[150, 100]).save()
+        Doc(values=[530, 100]).save()
+        Doc(values=[165, 100]).save()
+        Doc(values=[115, 100]).save()
+
+        self.assertEqual(
+            Doc.objects.average('values'),
+            170)
+
+    def test_embedded_sum(self):
+        class Pay(EmbeddedDocument):
+            value = DecimalField()
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(value=150)).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(value=530)).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(value=165)).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(value=115)).save()
+
+        self.assertEqual(
+            Doc.objects.sum('pay.value'),
+            960)
+
+
+    def test_embedded_array_sum(self):
+        class Pay(EmbeddedDocument):
+            values = ListField(DecimalField())
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(values=[150, 100])).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(values=[530, 100])).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(values=[165, 100])).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(values=[115, 100])).save()
+
+        self.assertEqual(
+            Doc.objects.sum('pay.values'),
+            1360)
+
+    def test_array_sum(self):
+        class Doc(Document):
+            values = ListField(DecimalField())
+
+        Doc.drop_collection()
+
+        Doc(values=[150, 100]).save()
+        Doc(values=[530, 100]).save()
+        Doc(values=[165, 100]).save()
+        Doc(values=[115, 100]).save()
+
+        self.assertEqual(
+            Doc.objects.sum('values'),
+            1360)
+
     def test_distinct(self):
         """Ensure that the QuerySet.distinct method works.
         """
@@ -3040,7 +3317,7 @@ class QuerySetTest(unittest.TestCase):
         class Foo(EmbeddedDocument):
             shape = StringField()
             color = StringField()
-            trick = BooleanField()
+            thick = BooleanField()
             meta = {'allow_inheritance': False}
 
         class Bar(Document):
@@ -3049,17 +3326,20 @@ class QuerySetTest(unittest.TestCase):
 
         Bar.drop_collection()
 
-        b1 = Bar(foo=[Foo(shape= "square", color ="purple", thick = False),
+        b1 = Bar(foo=[Foo(shape="square", color="purple", thick=False),
-                      Foo(shape= "circle", color ="red", thick = True)])
+                      Foo(shape="circle", color="red", thick=True)])
         b1.save()
 
-        b2 = Bar(foo=[Foo(shape= "square", color ="red", thick = True),
+        b2 = Bar(foo=[Foo(shape="square", color="red", thick=True),
-                      Foo(shape= "circle", color ="purple", thick = False)])
+                      Foo(shape="circle", color="purple", thick=False)])
         b2.save()
 
         ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"}))
         self.assertEqual([b1], ak)
 
+        ak = list(Bar.objects(foo__match=Foo(shape="square", color="purple")))
+        self.assertEqual([b1], ak)
+
     def test_upsert_includes_cls(self):
         """Upserts should include _cls information for inheritable classes
         """
@@ -3080,6 +3360,13 @@ class QuerySetTest(unittest.TestCase):
         Test.objects(test='foo').update_one(upsert=True, set__test='foo')
         self.assertTrue('_cls' in Test._collection.find_one())
 
+    def test_update_upsert_looks_like_a_digit(self):
+        class MyDoc(DynamicDocument):
+            pass
+        MyDoc.drop_collection()
+        self.assertEqual(1, MyDoc.objects.update_one(upsert=True, inc__47=1))
+        self.assertEqual(MyDoc.objects.get()['47'], 1)
+
     def test_read_preference(self):
         class Bar(Document):
             pass
@@ -3089,7 +3376,10 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual([], bars)
 
         self.assertRaises(ConfigurationError, Bar.objects,
-                            read_preference='Primary')
+                          read_preference='Primary')
+
+        bars = Bar.objects(read_preference=ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
 
     def test_json_simple(self):
 
@@ -3105,7 +3395,7 @@ class QuerySetTest(unittest.TestCase):
         Doc(string="Bye", embedded_field=Embedded(string="Bye")).save()
 
         Doc().save()
-        json_data = Doc.objects.to_json()
+        json_data = Doc.objects.to_json(sort_keys=True, separators=(',', ':'))
         doc_objects = list(Doc.objects)
 
         self.assertEqual(doc_objects, Doc.objects.from_json(json_data))
@@ -3170,6 +3460,9 @@ class QuerySetTest(unittest.TestCase):
         User(name="Bob Dole", age=89, price=Decimal('1.11')).save()
         User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
 
+        results = User.objects.only('id', 'name').as_pymongo()
+        self.assertEqual(sorted(results[0].keys()), sorted(['_id', 'name']))
+
         users = User.objects.only('name', 'price').as_pymongo()
         results = list(users)
         self.assertTrue(isinstance(results[0], dict))
@@ -3230,6 +3523,8 @@ class QuerySetTest(unittest.TestCase):
         self.assertTrue(isinstance(qs.first().organization, Organization))
         self.assertFalse(isinstance(qs.no_dereference().first().organization,
                                     Organization))
+        self.assertFalse(isinstance(qs.no_dereference().get().organization,
+                                    Organization))
         self.assertTrue(isinstance(qs.first().organization, Organization))
 
     def test_cached_queryset(self):
@@ -3256,6 +3551,27 @@ class QuerySetTest(unittest.TestCase):
             people.count()  # count is cached
             self.assertEqual(q, 1)
 
+    def test_no_cached_queryset(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        for i in xrange(100):
+            Person(name="No: %s" % i).save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            people = Person.objects.no_cache()
+
+            [x for x in people]
+            self.assertEqual(q, 1)
+
+            list(people)
+            self.assertEqual(q, 2)
+
+            people.count()
+            self.assertEqual(q, 3)
+
     def test_cache_not_cloned(self):
 
         class User(Document):
@@ -3277,6 +3593,34 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual("%s" % users, "[<User: Bob>]")
         self.assertEqual(1, len(users._result_cache))
 
+    def test_no_cache(self):
+        """Ensure you can add meta data to file"""
+
+        class Noddy(Document):
+            fields = DictField()
+
+        Noddy.drop_collection()
+        for i in xrange(100):
+            noddy = Noddy()
+            for j in range(20):
+                noddy.fields["key"+str(j)] = "value "+str(j)
+            noddy.save()
+
+        docs = Noddy.objects.no_cache()
+
+        counter = len([1 for i in docs])
+        self.assertEquals(counter, 100)
+
+        self.assertEquals(len(list(docs)), 100)
+        self.assertRaises(TypeError, lambda: len(docs))
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            list(docs)
+            self.assertEqual(q, 1)
+            list(docs)
+            self.assertEqual(q, 2)
+
     def test_nested_queryset_iterator(self):
         # Try iterating the same queryset twice, nested.
         names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George']
@@ -3380,6 +3724,51 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(B.objects.get(a=a).a, a)
         self.assertEqual(B.objects.get(a=a.id).a, a)
 
+    def test_cls_query_in_subclassed_docs(self):
+
+        class Animal(Document):
+            name = StringField()
+
+            meta = {
+                'allow_inheritance': True
+            }
+
+        class Dog(Animal):
+            pass
+
+        class Cat(Animal):
+            pass
+
+        self.assertEqual(Animal.objects(name='Charlie')._query, {
+            'name': 'Charlie',
+            '_cls': { '$in': ('Animal', 'Animal.Dog', 'Animal.Cat') }
+        })
+        self.assertEqual(Dog.objects(name='Charlie')._query, {
+            'name': 'Charlie',
+            '_cls': 'Animal.Dog'
+        })
+        self.assertEqual(Cat.objects(name='Charlie')._query, {
+            'name': 'Charlie',
+            '_cls': 'Animal.Cat'
+        })
+
+    def test_can_have_field_same_name_as_query_operator(self):
+
+        class Size(Document):
+            name = StringField()
+
+        class Example(Document):
+            size = ReferenceField(Size)
+
+        Size.drop_collection()
+        Example.drop_collection()
+
+        instance_size = Size(name="Large").save()
+        Example(size=instance_size).save()
+
+        self.assertEqual(Example.objects(size=instance_size).count(), 1)
+        self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/queryset/transform.py

@@ -31,6 +31,31 @@ class TransformTest(unittest.TestCase):
         self.assertEqual(transform.query(name__exists=True),
                          {'name': {'$exists': True}})
 
+    def test_transform_update(self):
+        class DicDoc(Document):
+            dictField = DictField()
+
+        class Doc(Document):
+            pass
+
+        DicDoc.drop_collection()
+        Doc.drop_collection()
+
+        doc = Doc().save()
+        dic_doc = DicDoc().save()
+
+        for k, v in (("set", "$set"), ("set_on_insert", "$setOnInsert"), ("push", "$push")):
+            update = transform.update(DicDoc, **{"%s__dictField__test" % k: doc})
+            self.assertTrue(isinstance(update[v]["dictField.test"], dict))
+
+        # Update special cases
+        update = transform.update(DicDoc, unset__dictField__test=doc)
+        self.assertEqual(update["$unset"]["dictField.test"], 1)
+
+        update = transform.update(DicDoc, pull__dictField__test=doc)
+        self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict))
+
+
     def test_query_field_name(self):
         """Ensure that the correct field name is used when querying.
         """

tests/queryset/visitor.py

@@ -68,11 +68,11 @@ class QTest(unittest.TestCase):
             x = IntField()
             y = StringField()
 
-        # Check than an error is raised when conflicting queries are anded
+        query = (Q(x__lt=7) & Q(x__lt=3)).to_query(TestDoc)
-        def invalid_combination():
+        self.assertEqual(query, {'$and': [{'x': {'$lt': 7}}, {'x': {'$lt': 3}}]})
-            query = Q(x__lt=7) & Q(x__lt=3)
+
-            query.to_query(TestDoc)
+        query = (Q(y="a") & Q(x__lt=7) & Q(x__lt=3)).to_query(TestDoc)
-        self.assertRaises(InvalidQueryError, invalid_combination)
+        self.assertEqual(query, {'$and': [{'y': "a"}, {'x': {'$lt': 7}}, {'x': {'$lt': 3}}]})
 
         # Check normal cases work without an error
         query = Q(x__lt=7) & Q(x__gt=3)
@@ -325,10 +325,26 @@ class QTest(unittest.TestCase):
         pk = ObjectId()
         User(email='example@example.com', pk=pk).save()
 
-        self.assertEqual(1, User.objects.filter(
+        self.assertEqual(1, User.objects.filter(Q(email='example@example.com') |
-                                Q(email='example@example.com') |
+                                                Q(name='John Doe')).limit(2).filter(pk=pk).count())
-                                Q(name='John Doe')
+
-                                ).limit(2).filter(pk=pk).count())
+    def test_chained_q_or_filtering(self):
+
+        class Post(EmbeddedDocument):
+            name = StringField(required=True)
+
+        class Item(Document):
+            postables = ListField(EmbeddedDocumentField(Post))
+
+        Item.drop_collection()
+
+        Item(postables=[Post(name="a"), Post(name="b")]).save()
+        Item(postables=[Post(name="a"), Post(name="c")]).save()
+        Item(postables=[Post(name="a"), Post(name="b"), Post(name="c")]).save()
+
+        self.assertEqual(Item.objects(Q(postables__name="a") & Q(postables__name="b")).count(), 2)
+        self.assertEqual(Item.objects.filter(postables__name="a").filter(postables__name="b").count(), 2)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_connection.py

@@ -56,6 +56,35 @@ class ConnectionTest(unittest.TestCase):
         self.assertTrue(isinstance(db, pymongo.database.Database))
         self.assertEqual(db.name, 'mongoenginetest')
 
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
+    def test_connect_uri_without_db(self):
+        """Ensure that the connect() method works properly with uri's
+        without database_name
+        """
+        c = connect(db='mongoenginetest', alias='admin')
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
+        c.admin.add_user("admin", "password")
+        c.admin.authenticate("admin", "password")
+        c.mongoenginetest.add_user("username", "password")
+
+        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
+
+        connect("mongoenginetest", host='mongodb://localhost/')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
     def test_register_connection(self):
         """Ensure that connections with different aliases may be registered.
         """

tests/test_dereference.py

@@ -1121,37 +1121,32 @@ class FieldTest(unittest.TestCase):
 
             self.assertEqual(q, 2)
 
-    def test_tuples_as_tuples(self):
+    def test_objectid_reference_across_databases(self):
-        """
+        # mongoenginetest - Is default connection alias from setUp()
-        Ensure that tuples remain tuples when they are
+        # Register Aliases
-        inside a ComplexBaseField
+        register_connection('testdb-1', 'mongoenginetest2')
-        """
-        from mongoengine.base import BaseField
 
-        class EnumField(BaseField):
+        class User(Document):
+            name = StringField()
+            meta = {"db_alias": "testdb-1"}
 
-            def __init__(self, **kwargs):
+        class Book(Document):
-                super(EnumField, self).__init__(**kwargs)
+            name = StringField()
+            author = ReferenceField(User)
 
-            def to_mongo(self, value):
+        # Drops
-                return value
+        User.drop_collection()
+        Book.drop_collection()
 
-            def to_python(self, value):
+        user = User(name="Ross").save()
-                return tuple(value)
+        Book(name="MongoEngine for pros", author=user).save()
 
-        class TestDoc(Document):
+        # Can't use query_counter across databases - so test the _data object
-            items = ListField(EnumField())
+        book = Book.objects.first()
+        self.assertFalse(isinstance(book._data['author'], User))
 
-        TestDoc.drop_collection()
+        book.select_related()
-        tuples = [(100, 'Testing')]
+        self.assertTrue(isinstance(book._data['author'], User))
-        doc = TestDoc()
-        doc.items = tuples
-        doc.save()
-        x = TestDoc.objects().get()
-        self.assertTrue(x is not None)
-        self.assertTrue(len(x.items) == 1)
-        self.assertTrue(tuple(x.items[0]) in tuples)
-        self.assertTrue(x.items[0] in tuples)
 
     def test_non_ascii_pk(self):
         """
@@ -1176,6 +1171,30 @@ class FieldTest(unittest.TestCase):
 
         self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands]))
 
+    def test_dereferencing_embedded_listfield_referencefield(self):
+        class Tag(Document):
+            meta = {'collection': 'tags'}
+            name = StringField()
+
+        class Post(EmbeddedDocument):
+            body = StringField()
+            tags = ListField(ReferenceField("Tag", dbref=True))
+
+        class Page(Document):
+            meta = {'collection': 'pages'}
+            tags = ListField(ReferenceField("Tag", dbref=True))
+            posts = ListField(EmbeddedDocumentField(Post))
+
+        Tag.drop_collection()
+        Page.drop_collection()
+
+        tag = Tag(name='test').save()
+        post = Post(body='test body', tags=[tag])
+        Page(tags=[tag], posts=[post]).save()
+
+        page = Page.objects.first()
+        self.assertEqual(page.tags[0], page.posts[0].tags[0])
+
 if __name__ == '__main__':
     unittest.main()
 

tests/test_django.py

@@ -2,48 +2,44 @@ import sys
 sys.path[0:0] = [""]
 import unittest
 from nose.plugins.skip import SkipTest
-from mongoengine.python_support import PY3
 from mongoengine import *
 
+
+from mongoengine.django.shortcuts import get_document_or_404
+
+from django.http import Http404
+from django.template import Context, Template
+from django.conf import settings
+from django.core.paginator import Paginator
+
+settings.configure(
+    USE_TZ=True,
+    INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'),
+    AUTH_USER_MODEL=('mongo_auth.MongoUser'),
+)
+
 try:
-    from mongoengine.django.shortcuts import get_document_or_404
+    from django.contrib.auth import authenticate, get_user_model
-
+    from mongoengine.django.auth import User
-    from django.http import Http404
+    from mongoengine.django.mongo_auth.models import (
-    from django.template import Context, Template
+        MongoUser,
-    from django.conf import settings
+        MongoUserManager,
-    from django.core.paginator import Paginator
+        get_user_document,
-
-    settings.configure(
-        USE_TZ=True,
-        INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'),
-        AUTH_USER_MODEL=('mongo_auth.MongoUser'),
     )
-
+    DJ15 = True
-    try:
+except Exception:
-        from django.contrib.auth import authenticate, get_user_model
+    DJ15 = False
-        from mongoengine.django.auth import User
+from django.contrib.sessions.tests import SessionTestsMixin
-        from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager
+from mongoengine.django.sessions import SessionStore, MongoSession
-        DJ15 = True
-    except Exception:
-        DJ15 = False
-    from django.contrib.sessions.tests import SessionTestsMixin
-    from mongoengine.django.sessions import SessionStore, MongoSession
-except Exception, err:
-    if PY3:
-        SessionTestsMixin = type  # dummy value so no error
-        SessionStore = None  # dummy value so no error
-    else:
-        raise err
-
-
 from datetime import tzinfo, timedelta
 ZERO = timedelta(0)
 
+
 class FixedOffset(tzinfo):
     """Fixed offset in minutes east from UTC."""
 
     def __init__(self, offset, name):
-        self.__offset = timedelta(minutes = offset)
+        self.__offset = timedelta(minutes=offset)
         self.__name = name
 
     def utcoffset(self, dt):
@@ -70,8 +66,6 @@ def activate_timezone(tz):
 class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
-        if PY3:
-            raise SkipTest('django does not have Python 3 support')
         connect(db='mongoenginetest')
 
         class Person(Document):
@@ -173,6 +167,8 @@ class QuerySetTest(unittest.TestCase):
         class Note(Document):
             text = StringField()
 
+        Note.drop_collection()
+
         for i in xrange(1, 101):
             Note(name="Note: %s" % i).save()
 
@@ -223,8 +219,6 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase):
     backend = SessionStore
 
     def setUp(self):
-        if PY3:
-            raise SkipTest('django does not have Python 3 support')
         connect(db='mongoenginetest')
         MongoSession.drop_collection()
         super(MongoDBSessionTest, self).setUp()
@@ -262,20 +256,21 @@ class MongoAuthTest(unittest.TestCase):
     }
 
     def setUp(self):
-        if PY3:
-            raise SkipTest('django does not have Python 3 support')
         if not DJ15:
             raise SkipTest('mongo_auth requires Django 1.5')
         connect(db='mongoenginetest')
         User.drop_collection()
         super(MongoAuthTest, self).setUp()
 
-    def test_user_model(self):
+    def test_get_user_model(self):
         self.assertEqual(get_user_model(), MongoUser)
 
+    def test_get_user_document(self):
+        self.assertEqual(get_user_document(), User)
+
     def test_user_manager(self):
         manager = get_user_model()._default_manager
-        self.assertIsInstance(manager, MongoUserManager)
+        self.assertTrue(isinstance(manager, MongoUserManager))
 
     def test_user_manager_exception(self):
         manager = get_user_model()._default_manager
@@ -285,14 +280,14 @@ class MongoAuthTest(unittest.TestCase):
     def test_create_user(self):
         manager = get_user_model()._default_manager
         user = manager.create_user(**self.user_data)
-        self.assertIsInstance(user, User)
+        self.assertTrue(isinstance(user, User))
         db_user = User.objects.get(username='user')
         self.assertEqual(user.id, db_user.id)
 
     def test_authenticate(self):
         get_user_model()._default_manager.create_user(**self.user_data)
         user = authenticate(username='user', password='fail')
-        self.assertIsNone(user)
+        self.assertEqual(None, user)
         user = authenticate(username='user', password='test')
         db_user = User.objects.get(username='user')
         self.assertEqual(user.id, db_user.id)

tests/test_signals.py

@@ -43,6 +43,15 @@ class SignalTests(unittest.TestCase):
             def pre_save(cls, sender, document, **kwargs):
                 signal_output.append('pre_save signal, %s' % document)
 
+            @classmethod
+            def pre_save_post_validation(cls, sender, document, **kwargs):
+                signal_output.append('pre_save_post_validation signal, %s' % document)
+                if 'created' in kwargs:
+                    if kwargs['created']:
+                        signal_output.append('Is created')
+                    else:
+                        signal_output.append('Is updated')
+
             @classmethod
             def post_save(cls, sender, document, **kwargs):
                 signal_output.append('post_save signal, %s' % document)
@@ -75,40 +84,19 @@ class SignalTests(unittest.TestCase):
         Author.drop_collection()
 
         class Another(Document):
+
             name = StringField()
 
             def __unicode__(self):
                 return self.name
 
-            @classmethod
-            def pre_init(cls, sender, document, **kwargs):
-                signal_output.append('pre_init Another signal, %s' % cls.__name__)
-                signal_output.append(str(kwargs['values']))
-
-            @classmethod
-            def post_init(cls, sender, document, **kwargs):
-                signal_output.append('post_init Another signal, %s' % document)
-
-            @classmethod
-            def pre_save(cls, sender, document, **kwargs):
-                signal_output.append('pre_save Another signal, %s' % document)
-
-            @classmethod
-            def post_save(cls, sender, document, **kwargs):
-                signal_output.append('post_save Another signal, %s' % document)
-                if 'created' in kwargs:
-                    if kwargs['created']:
-                        signal_output.append('Is created')
-                    else:
-                        signal_output.append('Is updated')
-
             @classmethod
             def pre_delete(cls, sender, document, **kwargs):
-                signal_output.append('pre_delete Another signal, %s' % document)
+                signal_output.append('pre_delete signal, %s' % document)
 
             @classmethod
             def post_delete(cls, sender, document, **kwargs):
-                signal_output.append('post_delete Another signal, %s' % document)
+                signal_output.append('post_delete signal, %s' % document)
 
         self.Another = Another
         Another.drop_collection()
@@ -133,6 +121,7 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_init.receivers),
             len(signals.post_init.receivers),
             len(signals.pre_save.receivers),
+            len(signals.pre_save_post_validation.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
             len(signals.post_delete.receivers),
@@ -143,16 +132,13 @@ class SignalTests(unittest.TestCase):
         signals.pre_init.connect(Author.pre_init, sender=Author)
         signals.post_init.connect(Author.post_init, sender=Author)
         signals.pre_save.connect(Author.pre_save, sender=Author)
+        signals.pre_save_post_validation.connect(Author.pre_save_post_validation, sender=Author)
         signals.post_save.connect(Author.post_save, sender=Author)
         signals.pre_delete.connect(Author.pre_delete, sender=Author)
         signals.post_delete.connect(Author.post_delete, sender=Author)
         signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author)
         signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author)
 
-        signals.pre_init.connect(Another.pre_init, sender=Another)
-        signals.post_init.connect(Another.post_init, sender=Another)
-        signals.pre_save.connect(Another.pre_save, sender=Another)
-        signals.post_save.connect(Another.post_save, sender=Another)
         signals.pre_delete.connect(Another.pre_delete, sender=Another)
         signals.post_delete.connect(Another.post_delete, sender=Another)
 
@@ -164,16 +150,13 @@ class SignalTests(unittest.TestCase):
         signals.post_delete.disconnect(self.Author.post_delete)
         signals.pre_delete.disconnect(self.Author.pre_delete)
         signals.post_save.disconnect(self.Author.post_save)
+        signals.pre_save_post_validation.disconnect(self.Author.pre_save_post_validation)
         signals.pre_save.disconnect(self.Author.pre_save)
         signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert)
         signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert)
 
-        signals.pre_init.disconnect(self.Another.pre_init)
-        signals.post_init.disconnect(self.Another.post_init)
         signals.post_delete.disconnect(self.Another.post_delete)
         signals.pre_delete.disconnect(self.Another.pre_delete)
-        signals.post_save.disconnect(self.Another.post_save)
-        signals.pre_save.disconnect(self.Another.pre_save)
 
         signals.post_save.disconnect(self.ExplicitId.post_save)
 
@@ -182,6 +165,7 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_init.receivers),
             len(signals.post_init.receivers),
             len(signals.pre_save.receivers),
+            len(signals.pre_save_post_validation.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
             len(signals.post_delete.receivers),
@@ -216,6 +200,8 @@ class SignalTests(unittest.TestCase):
         a1 = self.Author(name='Bill Shakespeare')
         self.assertEqual(self.get_signal_output(a1.save), [
             "pre_save signal, Bill Shakespeare",
+            "pre_save_post_validation signal, Bill Shakespeare",
+            "Is created",
             "post_save signal, Bill Shakespeare",
             "Is created"
         ])
@@ -224,6 +210,8 @@ class SignalTests(unittest.TestCase):
         a1.name = 'William Shakespeare'
         self.assertEqual(self.get_signal_output(a1.save), [
             "pre_save signal, William Shakespeare",
+            "pre_save_post_validation signal, William Shakespeare",
+            "Is updated",
             "post_save signal, William Shakespeare",
             "Is updated"
         ])
@@ -252,7 +240,14 @@ class SignalTests(unittest.TestCase):
             "Not loaded",
         ])
 
-        self.Author.objects.delete()
+    def test_queryset_delete_signals(self):
+        """ Queryset delete should throw some signals. """
+
+        self.Another(name='Bill Shakespeare').save()
+        self.assertEqual(self.get_signal_output(self.Another.objects.delete), [
+            'pre_delete signal, Bill Shakespeare',
+            'post_delete signal, Bill Shakespeare',
+        ])
 
     def test_signals_with_explicit_doc_ids(self):
         """ Model saves must have a created flag the first time."""