Version bump

fixes hmarr/mongoengine#519

.gitignore

@@ -1,7 +1,8 @@
 .*
 !.gitignore
-*.pyc
-.*.swp
+*~
+*.py[co]
+.*.sw[po]
 *.egg
 docs/.build
 docs/_build
@@ -12,4 +13,5 @@ env/
 .settings
 .project
 .pydevproject
-tests/bugfix.py
+tests/test_bugfix.py
+htmlcov/

.travis.yml

@@ -0,0 +1,12 @@
+# http://travis-ci.org/#!/MongoEngine/mongoengine
+language: python
+python:
+    - 2.6
+    - 2.7
+install:
+    - sudo apt-get install zlib1g zlib1g-dev
+    - sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/
+    - pip install PIL --use-mirrors ; true
+    - python setup.py install
+script:
+    - python setup.py test

AUTHORS

@@ -1,11 +1,11 @@
 The PRIMARY AUTHORS are (and/or have been):
 
+Ross Lawley <ross.lawley@gmail.com>
 Harry Marr <harry@hmarr.com>
 Matt Dennewitz <mattdennewitz@gmail.com>
 Deepak Thukral <iapain@yahoo.com>
 Florian Schlachter <flori@n-schlachter.de>
 Steve Challis <steve@stevechallis.com>
-Ross Lawley <ross.lawley@gmail.com>
 Wilson Júnior <wilsonpjunior@gmail.com>
 Dan Crosta https://github.com/dcrosta
 
@@ -67,5 +67,44 @@ that much better:
  * Gareth Lloyd
  * Albert Choi
  * John Arnfield
+ * grubberr
+ * Paul Aliagas
+ * Paul Cunnane
  * Julien Rebetez
-
+ * Marc Tamlyn
+ * Karim Allah
+ * Adam Parrish
+ * jpfarias
+ * jonrscott
+ * Alice Zoë Bevan-McGregor
+ * Stephen Young
+ * tkloc
+ * aid
+ * yamaneko1212
+ * dave mankoff
+ * Alexander G. Morano
+ * jwilder
+ * Joe Shaw
+ * Adam Flynn
+ * Ankhbayar
+ * Jan Schrewe
+ * David Koblas
+ * Crittercism
+ * Alvin Liang
+ * andrewmlevy
+ * Chris Faulkner
+ * Ashwin Purohit
+ * Shalabh Aggarwal
+ * Chris Williams
+ * Robert Kajic
+ * Jacob Peddicord
+ * Nils Hasenbanck
+ * mostlystatic
+ * Greg Banks
+ * swashbuckler
+ * Adam Reeve
+ * Anthony Nemitz
+ * deignacio
+ * shaunduncan
+ * Meir Kriheli
+ * Andrey Fedoseev

README.rst

@@ -3,25 +3,29 @@ MongoEngine
 ===========
 :Info: MongoEngine is an ORM-like layer on top of PyMongo.
 :Author: Harry Marr (http://github.com/hmarr)
+:Maintainer: Ross Lawley (http://github.com/rozza)
+
+.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master
+  :target: http://travis-ci.org/MongoEngine/mongoengine
 
 About
 =====
 MongoEngine is a Python Object-Document Mapper for working with MongoDB.
-Documentation available at http://hmarr.com/mongoengine/ - there is currently 
-a `tutorial <http://hmarr.com/mongoengine/tutorial.html>`_, a `user guide 
-<http://hmarr.com/mongoengine/userguide.html>`_ and an `API reference
-<http://hmarr.com/mongoengine/apireference.html>`_.
+Documentation available at http://mongoengine-odm.rtfd.org - there is currently
+a `tutorial <http://readthedocs.org/docs/mongoengine-odm/en/latest/tutorial.html>`_, a `user guide
+<http://readthedocs.org/docs/mongoengine-odm/en/latest/userguide.html>`_ and an `API reference
+<http://readthedocs.org/docs/mongoengine-odm/en/latest/apireference.html>`_.
 
 Installation
 ============
 If you have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
 you can use ``easy_install -U mongoengine``. Otherwise, you can download the
-source from `GitHub <http://github.com/hmarr/mongoengine>`_ and run ``python
+source from `GitHub <http://github.com/MongoEngine/mongoengine>`_ and run ``python
 setup.py install``.
 
 Dependencies
 ============
-- pymongo 1.1+
+- pymongo 2.1.1+
 - sphinx (optional - for documentation generation)
 
 Examples
@@ -92,6 +96,7 @@ Community
 
 Contributing
 ============
-The source is available on `GitHub <http://github.com/hmarr/mongoengine>`_ - to
+The source is available on `GitHub <http://github.com/MongoEngine/mongoengine>`_ - to
 contribute to the project, fork it on GitHub and send a pull request, all
 contributions and suggestions are welcome!
+

benchmark.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+
+import timeit
+
+
+def cprofile_main():
+    from pymongo import Connection
+    connection = Connection()
+    connection.drop_database('timeit_test')
+    connection.disconnect()
+
+    from mongoengine import Document, DictField, connect
+    connect("timeit_test")
+
+    class Noddy(Document):
+        fields = DictField()
+
+    for i in xrange(1):
+        noddy = Noddy()
+        for j in range(20):
+            noddy.fields["key" + str(j)] = "value " + str(j)
+        noddy.save()
+
+
+def main():
+    """
+    0.4 Performance Figures ...
+
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    1.1141769886
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    2.37724113464
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    1.92479610443
+
+    0.5.X
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    1.10552310944
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    16.5169169903
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    14.9446101189
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    14.912801981
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, force=True
+    14.9617750645
+
+    Performance
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    1.10072994232
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    5.27341103554
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    4.49365401268
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    4.43459296227
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, force=True
+    4.40114378929
+    """
+
+    setup = """
+from pymongo import Connection
+connection = Connection()
+connection.drop_database('timeit_test')
+"""
+
+    stmt = """
+from pymongo import Connection
+connection = Connection()
+
+db = connection.timeit_test
+noddy = db.noddy
+
+for i in xrange(10000):
+    example = {'fields': {}}
+    for j in range(20):
+        example['fields']["key"+str(j)] = "value "+str(j)
+
+    noddy.insert(example)
+
+myNoddys = noddy.find()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - Pymongo"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    setup = """
+from pymongo import Connection
+connection = Connection()
+connection.drop_database('timeit_test')
+connection.disconnect()
+
+from mongoengine import Document, DictField, connect
+connect("timeit_test")
+
+class Noddy(Document):
+    fields = DictField()
+"""
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save()
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(safe=False, validate=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(safe=False, validate=False, cascade=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(force_insert=True, safe=False, validate=False, cascade=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, force=True"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+if __name__ == "__main__":
+    main()

docs/apireference.rst

@@ -6,6 +6,7 @@ Connecting
 ==========
 
 .. autofunction:: mongoengine.connect
+.. autofunction:: mongoengine.register_connection
 
 Documents
 =========
@@ -21,9 +22,18 @@ Documents
 .. autoclass:: mongoengine.EmbeddedDocument
    :members:
 
+.. autoclass:: mongoengine.DynamicDocument
+   :members:
+
+.. autoclass:: mongoengine.DynamicEmbeddedDocument
+   :members:
+
 .. autoclass:: mongoengine.document.MapReduceDocument
   :members:
 
+.. autoclass:: mongoengine.ValidationError
+  :members:
+
 Querying
 ========
 

docs/changelog.rst

@@ -2,15 +2,162 @@
 Changelog
 =========
 
+Changes in 0.6.12
+=================
+- Fixes scalar lookups for primary_key
+- Fixes error with _delta handling DBRefs
+
+Changes in 0.6.11
+==================
+- Fixed inconsistency handling None values field attrs
+- Fixed map_field embedded db_field issue
+- Fixed .save() _delta issue with DbRefs
+- Fixed Django TestCase
+- Added cmp to Embedded Document
+- Added PULL reverse_delete_rule
+- Fixed CASCADE delete bug
+- Fixed db_field data load error
+- Fixed recursive save with FileField
+
+Changes in 0.6.10
+=================
+- Fixed basedict / baselist to return super(..)
+- Promoted BaseDynamicField to DynamicField
+
+Changes in 0.6.9
+================
+- Fixed sparse indexes on inherited docs
+- Removed FileField auto deletion, needs more work maybe 0.7
+
+Changes in 0.6.8
+================
+- Fixed FileField losing reference when no default set
+- Removed possible race condition from FileField (grid_file)
+- Added assignment to save, can now do: b = MyDoc(**kwargs).save()
+- Added support for pull operations on nested EmbeddedDocuments
+- Added support for choices with GenericReferenceFields
+- Added support for choices with GenericEmbeddedDocumentFields
+- Fixed Django 1.4 sessions first save data loss
+- FileField now automatically delete files on .delete()
+- Fix for GenericReference to_mongo method
+- Fixed connection regression
+- Updated Django User document, now allows inheritance
+
+Changes in 0.6.7
+================
+- Fixed indexing on '_id' or 'pk' or 'id'
+- Invalid data from the DB now raises a InvalidDocumentError
+- Cleaned up the Validation Error - docs and code
+- Added meta `auto_create_index` so you can disable index creation
+- Added write concern options to inserts
+- Fixed typo in meta for index options
+- Bug fix Read preference now passed correctly
+- Added support for File like objects for GridFS
+- Fix for #473 - Dereferencing abstracts
+
+Changes in 0.6.6
+================
+- Django 1.4 fixed (finally)
+- Added tests for Django
+
+Changes in 0.6.5
+================
+- More Django updates
+
+Changes in 0.6.4
+================
+
+- Refactored connection / fixed replicasetconnection
+- Bug fix for unknown connection alias error message
+- Sessions support Django 1.3 and Django 1.4
+- Minor fix for ReferenceField
+
+Changes in 0.6.3
+================
+- Updated sessions for Django 1.4
+- Bug fix for updates where listfields contain embedded documents
+- Bug fix for collection naming and mixins
+
+Changes in 0.6.2
+================
+- Updated documentation for ReplicaSet connections
+- Hack round _types issue with SERVER-5247 - querying other arrays may also cause problems.
+
+Changes in 0.6.1
+================
+- Fix for replicaSet connections
+
+Changes in 0.6
+================
+
+- Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7
+- Added support for covered indexes when inheritance is off
+- No longer always upsert on save for items with a '_id'
+- Error raised if update doesn't have an operation
+- DeReferencing is now thread safe
+- Errors raised if trying to perform a join in a query
+- Updates can now take __raw__ queries
+- Added custom 2D index declarations
+- Added replicaSet connection support
+- Updated deprecated imports from pymongo (safe for pymongo 2.2)
+- Added uri support for connections
+- Added scalar for efficiently returning partial data values (aliased to values_list)
+- Fixed limit skip bug
+- Improved Inheritance / Mixin
+- Added sharding support
+- Added pymongo 2.1 support
+- Fixed Abstract documents can now declare indexes
+- Added db_alias support to individual documents
+- Fixed GridFS documents can now be pickled
+- Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field
+- Added InvalidQueryError when calling with_id with a filter
+- Added support for DBRefs in distinct()
+- Fixed issue saving False booleans
+- Fixed issue with dynamic documents deltas
+- Added Reverse Delete Rule support to ListFields - MapFields aren't supported
+- Added customisable cascade kwarg options
+- Fixed Handle None values for non-required fields
+- Removed Document._get_subclasses() - no longer required
+- Fixed bug requiring subclasses when not actually needed
+- Fixed deletion of dynamic data
+- Added support for the $elementMatch operator
+- Added reverse option to SortedListFields
+- Fixed dereferencing - multi directional list dereferencing
+- Fixed issue creating indexes with recursive embedded documents
+- Fixed recursive lookup in _unique_with_indexes
+- Fixed passing ComplexField defaults to constructor for ReferenceFields
+- Fixed validation of DictField Int keys
+- Added optional cascade saving
+- Fixed dereferencing - max_depth now taken into account
+- Fixed document mutation saving issue
+- Fixed positional operator when replacing embedded documents
+- Added Non-Django Style choices back (you can have either)
+- Fixed __repr__ of a sliced queryset
+- Added recursive validation error of documents / complex fields
+- Fixed breaking during queryset iteration
+- Added pre and post bulk-insert signals
+- Added ImageField - requires PIL
+- Fixed Reference Fields can be None in get_or_create / queries
+- Fixed accessing pk on an embedded document
+- Fixed calling a queryset after drop_collection now recreates the collection
+- Add field name to validation exception messages
+- Added UUID field
+- Improved efficiency of .get()
+- Updated ComplexFields so if required they won't accept empty lists / dicts
+- Added spec file for rpm-based distributions
+- Fixed ListField so it doesnt accept strings
+- Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas
+
 Changes in v0.5.2
 =================
 
 - A Robust Circular reference bugfix
 
+
 Changes in v0.5.1
 =================
 
-- Circular reference bugfix
+- Fixed simple circular reference bug
 
 Changes in v0.5
 ===============

docs/conf.py

@@ -38,7 +38,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'MongoEngine'
-copyright = u'2009-2011, Harry Marr'
+copyright = u'2009-2012, MongoEngine Authors'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -121,7 +121,7 @@ html_theme_path = ['_themes']
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

docs/django.rst

@@ -2,6 +2,8 @@
 Using MongoEngine with Django
 =============================
 
+.. note :: Updated to support Django 1.4
+
 Connecting
 ==========
 In your **settings.py** file, ignore the standard database settings (unless you

docs/guide/connecting.rst

@@ -3,6 +3,7 @@
 =====================
 Connecting to MongoDB
 =====================
+
 To connect to a running instance of :program:`mongod`, use the
 :func:`~mongoengine.connect` function. The first argument is the name of the
 database to connect to. If the database does not exist, it will be created. If
@@ -18,3 +19,47 @@ provide :attr:`host` and :attr:`port` arguments to
 :func:`~mongoengine.connect`::
 
     connect('project1', host='192.168.1.35', port=12345)
+
+Uri style connections are also supported as long as you include the database
+name - just supply the uri as the :attr:`host` to
+:func:`~mongoengine.connect`::
+
+    connect('project1', host='mongodb://localhost/database_name')
+
+ReplicaSets
+===========
+
+MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnection`
+to use them please use a URI style connection and provide the `replicaSet` name in the
+connection kwargs.
+
+Multiple Databases
+==================
+
+Multiple database support was added in MongoEngine 0.6. To use multiple
+databases you can use :func:`~mongoengine.connect` and provide an `alias` name
+for the connection - if no `alias` is provided then "default" is used.
+
+In the background this uses :func:`~mongoengine.register_connection` to
+store the data and you can register all aliases up front if required.
+
+Individual documents can also support multiple databases by providing a
+`db_alias` in their meta data.  This allows :class:`~pymongo.dbref.DBRef` objects
+to point across databases and collections.  Below is an example schema, using
+3 different databases to store data::
+
+        class User(Document):
+            name = StringField()
+
+            meta = {"db_alias": "user-db"}
+
+        class Book(Document):
+            name = StringField()
+
+            meta = {"db_alias": "book-db"}
+
+        class AuthorBooks(Document):
+            author = ReferenceField(User)
+            book = ReferenceField(Book)
+
+            meta = {"db_alias": "users-books-db"}

docs/guide/defining-documents.rst

@@ -24,6 +24,34 @@ objects** as class attributes to the document class::
         title = StringField(max_length=200, required=True)
         date_modified = DateTimeField(default=datetime.datetime.now)
 
+Dynamic document schemas
+========================
+One of the benefits of MongoDb is dynamic schemas for a collection, whilst data
+should be planned and organised (after all explicit is better than implicit!)
+there are scenarios where having dynamic / expando style documents is desirable.
+
+:class:`~mongoengine.DynamicDocument` documents work in the same way as
+:class:`~mongoengine.Document` but any data / attributes set to them will also
+be saved ::
+
+    from mongoengine import *
+
+    class Page(DynamicDocument):
+        title = StringField(max_length=200, required=True)
+
+    # Create a new page and add tags
+    >>> page = Page(title='Using MongoEngine')
+    >>> page.tags = ['mongodb', 'mongoengine']
+    >>> page.save()
+
+    >>> Page.objects(tags='mongoengine').count()
+    >>> 1
+
+..note::
+
+   There is one caveat on Dynamic Documents: fields cannot start with `_`
+
+
 Fields
 ======
 By default, fields are not required. To make a field mandatory, set the
@@ -70,7 +98,7 @@ arguments can be set on all fields:
 
 :attr:`required` (Default: False)
     If set to True and the field is not set on the document instance, a
-    :class:`~mongoengine.base.ValidationError` will be raised when the document is
+    :class:`~mongoengine.ValidationError` will be raised when the document is
     validated.
 
 :attr:`default` (Default: None)
@@ -107,12 +135,33 @@ arguments can be set on all fields:
     When True, use this field as a primary key for the collection.
 
 :attr:`choices` (Default: None)
-    An iterable of choices to which the value of this field should be limited.
+    An iterable (e.g. a list or tuple) of choices to which the value of this
+    field should be limited.
+
+    Can be either be a nested tuples of value (stored in mongo) and a
+    human readable key ::
+
+        SIZE = (('S', 'Small'),
+                ('M', 'Medium'),
+                ('L', 'Large'),
+                ('XL', 'Extra Large'),
+                ('XXL', 'Extra Extra Large'))
+
+
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=SIZE)
+
+    Or a flat iterable just containing values ::
+
+        SIZE = ('S', 'M', 'L', 'XL', 'XXL')
+
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=SIZE)
 
 :attr:`help_text` (Default: None)
     Optional help text to output with the field - used by form libraries
 
-:attr:`verbose` (Default: None)
+:attr:`verbose_name` (Default: None)
     Optional human-readable name for the field - used by form libraries
 
 
@@ -240,6 +289,10 @@ Its value can take any of the following constants:
 :const:`mongoengine.CASCADE`
   Any object containing fields that are refererring to the object being deleted
   are deleted first.
+:const:`mongoengine.PULL`
+  Removes the reference to the object (using MongoDB's "pull" operation)
+  from any object's fields of
+  :class:`~mongoengine.ListField` (:class:`~mongoengine.ReferenceField`).
 
 
 .. warning::
@@ -382,11 +435,32 @@ If a dictionary is passed then the following options are available:
 :attr:`unique` (Default: False)
     Whether the index should be sparse.
 
-.. note::
+.. warning::
 
+
+   Inheritance adds extra indices.
+   If don't need inheritance for a document turn inheritance off - see :ref:`document-inheritance`.
+
+
+Geospatial indexes
+---------------------------
 Geospatial indexes will be automatically created for all
 :class:`~mongoengine.GeoPointField`\ s
 
+It is also possible to explicitly define geospatial indexes. This is
+useful if you need to define a geospatial index on a subfield of a
+:class:`~mongoengine.DictField` or a custom field that contains a
+point. To create a geospatial index you must prefix the field with the
+***** sign. ::
+
+    class Place(Document):
+        location = DictField()
+        meta = {
+            'indexes': [
+                '*location.point',
+            ],
+        }
+
 Ordering
 ========
 A default ordering can be specified for your
@@ -427,8 +501,31 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. ::
     first_post = BlogPost.objects.order_by("+published_date").first()
     assert first_post.title == "Blog Post #1"
 
+Shard keys
+==========
+
+If your collection is sharded, then you need to specify the shard key as a tuple,
+using the :attr:`shard_key` attribute of :attr:`-mongoengine.Document.meta`.
+This ensures that the shard key is sent with the query when calling the
+:meth:`~mongoengine.document.Document.save` or
+:meth:`~mongoengine.document.Document.update` method on an existing
+:class:`-mongoengine.Document` instance::
+
+    class LogEntry(Document):
+        machine = StringField()
+        app = StringField()
+        timestamp = DateTimeField()
+        data = StringField()
+
+        meta = {
+            'shard_key': ('machine', 'timestamp',)
+        }
+
+.. _document-inheritance:
+
 Document inheritance
 ====================
+
 To create a specialised type of a :class:`~mongoengine.Document` you have
 defined, you may subclass it and add any extra fields or methods you may need.
 As this is new class is not a direct subclass of
@@ -440,10 +537,15 @@ convenient and efficient retrieval of related documents::
     class Page(Document):
         title = StringField(max_length=200, required=True)
 
+        meta = {'allow_inheritance': True}
+
     # Also stored in the collection named 'page'
     class DatedPage(Page):
         date = DateTimeField()
 
+.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document meta.
+
+
 Working with existing data
 --------------------------
 To enable correct retrieval of documents involved in this kind of heirarchy,

docs/guide/document-instances.rst

@@ -35,13 +35,23 @@ already exist, then any changes will be updated atomically.  For example::
     * ``list_field.pop(0)`` - *sets* the resulting list
     * ``del(list_field)``   - *unsets* whole list
 
-To delete a document, call the :meth:`~mongoengine.Document.delete` method.
-Note that this will only work if the document exists in the database and has a
-valide :attr:`id`.
-
 .. seealso::
     :ref:`guide-atomic-updates`
 
+Cascading Saves
+---------------
+If your document contains :class:`~mongoengine.ReferenceField` or
+:class:`~mongoengine.GenericReferenceField` objects, then by default the
+:meth:`~mongoengine.Document.save` method will automatically save any changes to
+those objects as well.  If this is not desired passing :attr:`cascade` as False
+to the save method turns this feature off.
+
+Deleting documents
+------------------
+To delete a document, call the :meth:`~mongoengine.Document.delete` method.
+Note that this will only work if the document exists in the database and has a
+valid :attr:`id`.
+
 Document IDs
 ============
 Each document in the database has a unique id. This may be accessed through the
@@ -81,5 +91,5 @@ is an alias to :attr:`id`::
 .. note::
 
    If you define your own primary key field, the field implicitly becomes
-   required, so a :class:`ValidationError` will be thrown if you don't provide
-   it.
+   required, so a :class:`~mongoengine.ValidationError` will be thrown if
+   you don't provide it.

docs/guide/gridfs.rst

@@ -65,7 +65,7 @@ Deleting stored files is achieved with the :func:`delete` method::
 
     marmot.photo.delete()
 
-.. note::
+.. warning::
 
     The FileField in a Document actually only stores the ID of a file in a
     separate GridFS collection. This means that deleting a document

docs/guide/querying.rst

@@ -76,6 +76,7 @@ expressions:
 * ``istartswith`` -- string field starts with value (case insensitive)
 * ``endswith`` -- string field ends with value
 * ``iendswith`` -- string field ends with value (case insensitive)
+* ``match``  -- performs an $elemMatch so you can match an entire document within an array
 
 There are a few special operators for performing geographical queries, that
 may used with :class:`~mongoengine.GeoPointField`\ s:
@@ -194,22 +195,6 @@ to be created::
     >>> a.name == b.name and a.age == b.age
     True
 
-Dereferencing results
----------------------
-When iterating the results of :class:`~mongoengine.ListField` or
-:class:`~mongoengine.DictField` we automatically dereference any
-:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the
-number the queries to mongo.
-
-There are times when that efficiency is not enough, documents that have
-:class:`~mongoengine.ReferenceField` objects or
-:class:`~mongoengine.GenericReferenceField` objects at the top level are
-expensive as the number of queries to MongoDB can quickly rise.
-
-To limit the number of queries use
-:func:`~mongoengine.queryset.QuerySet.select_related` which converts the
-QuerySet to a list and dereferences as efficiently as possible.
-
 Default Document queries
 ========================
 By default, the objects :attr:`~mongoengine.Document.objects` attribute on a
@@ -312,8 +297,16 @@ would be generating "tag-clouds"::
     from operator import itemgetter
     top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10]
 
+
+Query efficiency and performance
+================================
+
+There are a couple of methods to improve efficiency when querying, reducing the
+information returned by the query or efficient dereferencing .
+
 Retrieving a subset of fields
-=============================
+-----------------------------
+
 Sometimes a subset of fields on a :class:`~mongoengine.Document` is required,
 and for efficiency only these should be retrieved from the database. This issue
 is especially important for MongoDB, as fields may often be extremely large
@@ -346,6 +339,27 @@ will be given::
 If you later need the missing fields, just call
 :meth:`~mongoengine.Document.reload` on your document.
 
+Getting related data
+--------------------
+
+When iterating the results of :class:`~mongoengine.ListField` or
+:class:`~mongoengine.DictField` we automatically dereference any
+:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the
+number the queries to mongo.
+
+There are times when that efficiency is not enough, documents that have
+:class:`~mongoengine.ReferenceField` objects or
+:class:`~mongoengine.GenericReferenceField` objects at the top level are
+expensive as the number of queries to MongoDB can quickly rise.
+
+To limit the number of queries use
+:func:`~mongoengine.queryset.QuerySet.select_related` which converts the
+QuerySet to a list and dereferences as efficiently as possible.  By default
+:func:`~mongoengine.queryset.QuerySet.select_related` only dereferences any
+references to the depth of 1 level.  If you have more complicated documents and
+want to dereference more of the object at once then increasing the :attr:`max_depth`
+will dereference more levels of the document.
+
 Advanced queries
 ================
 Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword

docs/guide/signals.rst

@@ -5,11 +5,13 @@ Signals
 
 .. versionadded:: 0.5
 
+.. note::
+
   Signal support is provided by the excellent `blinker`_ library and
   will gracefully fall back if it is not available.
 
 
-The following document signals exist in MongoEngine and are pretty self explaintary:
+The following document signals exist in MongoEngine and are pretty self-explanatory:
 
   * `mongoengine.signals.pre_init`
   * `mongoengine.signals.post_init`
@@ -17,6 +19,8 @@ The following document signals exist in MongoEngine and are pretty self explaint
   * `mongoengine.signals.post_save`
   * `mongoengine.signals.pre_delete`
   * `mongoengine.signals.post_delete`
+  * `mongoengine.signals.pre_bulk_insert`
+  * `mongoengine.signals.post_bulk_insert`
 
 Example usage::
 

docs/index.rst

@@ -18,6 +18,9 @@ MongoDB. To install it, simply run
 :doc:`apireference`
   The complete API documentation.
 
+:doc:`upgrade`
+  How to upgrade MongoEngine.
+
 :doc:`django`
   Using MongoEngine and Django
 
@@ -42,7 +45,8 @@ Also, you can join the developers' `mailing list
 
 Changes
 -------
-See the :doc:`changelog` for a full list of changes to MongoEngine.
+See the :doc:`changelog` for a full list of changes to MongoEngine and
+:doc:`upgrade` for upgrade information.
 
 .. toctree::
    :hidden:

docs/tutorial.rst

@@ -167,6 +167,11 @@ To delete all the posts if a user is deleted set the rule::
 
 See :class:`~mongoengine.ReferenceField` for more information.
 
+..note::
+    MapFields and DictFields currently don't support automatic handling of
+    deleted references
+
+
 Adding data to our Tumblelog
 ============================
 Now that we've defined how our documents will be structured, let's start adding

docs/upgrade.rst

@@ -2,6 +2,24 @@
 Upgrading
 =========
 
+0.5 to 0.6
+==========
+
+Embedded Documents - if you had a `pk` field you will have to rename it from `_id`
+to `pk` as pk is no longer a property of Embedded Documents.
+
+Reverse Delete Rules in Embedded Documents, MapFields and DictFields now throw
+an InvalidDocument error as they aren't currently supported.
+
+Document._get_subclasses - Is no longer used and the class method has been removed.
+
+Document.objects.with_id - now raises an InvalidQueryError if used with a filter.
+
+FutureWarning - A future warning has been added to all inherited classes that
+don't define `allow_inheritance` in their meta.
+
+You may need to update pyMongo to 2.0 for use with Sharding.
+
 0.4 to 0.5
 ===========
 
@@ -9,7 +27,7 @@ There have been the following backwards incompatibilities from 0.4 to 0.5.  The
 main areas of changed are: choices in fields, map_reduce and collection names.
 
 Choice options:
---------------
+---------------
 
 Are now expected to be an iterable of tuples, with  the first element in each
 tuple being the actual value to be stored. The second element is the
@@ -58,7 +76,7 @@ To upgrade use a Mixin class to set meta like so ::
     class MyAceDocument(Document, BaseMixin):
         pass
 
-    MyAceDocument._get_collection_name() == myacedocument
+    MyAceDocument._get_collection_name() == "myacedocument"
 
 Alternatively, you can rename your collections eg ::
 

mongoengine/__init__.py

@@ -12,9 +12,7 @@ from signals import *
 __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
            queryset.__all__ + signals.__all__)
 
-__author__ = 'Harry Marr'
-
-VERSION = (0, 5, 2)
+VERSION = (0, 6, 12)
 
 
 def get_version():

mongoengine/base.py

@@ -1,15 +1,18 @@
+import warnings
+
 from queryset import QuerySet, QuerySetManager
 from queryset import DoesNotExist, MultipleObjectsReturned
 from queryset import DO_NOTHING
 
 from mongoengine import signals
 
-import weakref
 import sys
 import pymongo
-import pymongo.objectid
+from bson import ObjectId
 import operator
+
 from functools import partial
+from bson.dbref import DBRef
 
 
 class NotRegistered(Exception):
@@ -19,8 +22,88 @@ class NotRegistered(Exception):
 class InvalidDocumentError(Exception):
     pass
 
-class ValidationError(Exception):
-    pass
+
+class ValidationError(AssertionError):
+    """Validation exception.
+
+    May represent an error validating a field or a
+    document containing fields with validation errors.
+
+    :ivar errors: A dictionary of errors for fields within this
+        document or list, or None if the error is for an
+        individual field.
+    """
+
+    errors = {}
+    field_name = None
+    _message = None
+
+    def __init__(self, message="", **kwargs):
+        self.errors = kwargs.get('errors', {})
+        self.field_name = kwargs.get('field_name')
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return '%s(%s,)' % (self.__class__.__name__, self.message)
+
+    def __getattribute__(self, name):
+        message = super(ValidationError, self).__getattribute__(name)
+        if name == 'message':
+            if self.field_name:
+                message = '%s ("%s")' % (message, self.field_name)
+            if self.errors:
+                message = '%s:\n%s' % (message, self._format_errors())
+        return message
+
+    def _get_message(self):
+        return self._message
+
+    def _set_message(self, message):
+        self._message = message
+
+    message = property(_get_message, _set_message)
+
+    def to_dict(self):
+        """Returns a dictionary of all errors within a document
+
+        Keys are field names or list indices and values are the
+        validation error messages, or a nested dictionary of
+        errors for an embedded document or list.
+        """
+
+        def build_dict(source):
+            errors_dict = {}
+            if not source:
+                return errors_dict
+            if isinstance(source, dict):
+                for field_name, error in source.iteritems():
+                    errors_dict[field_name] = build_dict(error)
+            elif isinstance(source, ValidationError) and source.errors:
+                return build_dict(source.errors)
+            else:
+                return unicode(source)
+            return errors_dict
+        if not self.errors:
+            return {}
+        return build_dict(self.errors)
+
+    def _format_errors(self):
+        """Returns a string listing all errors within a document"""
+
+        def format_error(field, value, prefix=''):
+            prefix = "%s.%s" % (prefix, field) if prefix else "%s" % field
+            if isinstance(value, dict):
+
+                return '\n'.join(
+                        [format_error(k, value[k], prefix) for k in value])
+            else:
+                return "%s: %s" % (prefix, value)
+
+        return '\n'.join(
+                [format_error(k, v) for k, v in self.to_dict().items()])
 
 
 _document_registry = {}
@@ -50,6 +133,8 @@ class BaseField(object):
     .. versionchanged:: 0.5 - added verbose and help text
     """
 
+    name = None
+
     # Fields may have _types inserted into indexes by default
     _index_with_types = True
     _geo_index = False
@@ -65,7 +150,6 @@ class BaseField(object):
                  validation=None, choices=None, verbose_name=None, help_text=None):
         self.db_field = (db_field or name) if not primary_key else '_id'
         if name:
-            import warnings
             msg = "Fields' 'name' attribute deprecated in favour of 'db_field'"
             warnings.warn(msg, DeprecationWarning)
         self.name = None
@@ -97,17 +181,13 @@ class BaseField(object):
 
         # Get value from document instance if available, if not use default
         value = instance._data.get(self.name)
+
         if value is None:
             value = self.default
             # Allow callable default values
             if callable(value):
                 value = value()
 
-        # Convert lists / values so we can watch for any changes on them
-        if isinstance(value, (list, tuple)) and not isinstance(value, BaseList):
-            value = BaseList(value, instance=instance, name=self.name)
-        elif isinstance(value, dict) and not isinstance(value, BaseDict):
-            value = BaseDict(value, instance=instance, name=self.name)
         return value
 
     def __set__(self, instance, value):
@@ -116,6 +196,12 @@ class BaseField(object):
         instance._data[self.name] = value
         instance._mark_as_changed(self.name)
 
+    def error(self, message="", errors=None, field_name=None):
+        """Raises a ValidationError.
+        """
+        field_name = field_name if field_name else self.name
+        raise ValidationError(message, errors=errors, field_name=field_name)
+
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
         """
@@ -137,20 +223,27 @@ class BaseField(object):
         pass
 
     def _validate(self, value):
+        from mongoengine import Document, EmbeddedDocument
         # check choices
-        if self.choices is not None:
+        if self.choices:
+            is_cls = isinstance(value, (Document, EmbeddedDocument))
+            value_to_check = value.__class__ if is_cls else value
+            err_msg = 'an instance' if is_cls else 'one'
+            if isinstance(self.choices[0], (list, tuple)):
                 option_keys = [option_key for option_key, option_value in self.choices]
-            if value not in option_keys:
-                raise ValidationError("Value must be one of %s." % unicode(option_keys))
+                if value_to_check not in option_keys:
+                    self.error('Value must be %s of %s' % (err_msg, unicode(option_keys)))
+            elif value_to_check not in self.choices:
+                self.error('Value must be %s of %s' % (err_msg, unicode(self.choices)))
 
         # check validation argument
         if self.validation is not None:
             if callable(self.validation):
                 if not self.validation(value):
-                    raise ValidationError('Value does not match custom' \
-                                          'validation method.')
+                    self.error('Value does not match custom validation method')
             else:
-                raise ValueError('validation argument must be a callable.')
+                raise ValueError('validation argument for "%s" must be a '
+                                 'callable.' % self.name)
 
         self.validate(value)
 
@@ -166,6 +259,7 @@ class ComplexBaseField(BaseField):
     """
 
     field = None
+    _dereference = False
 
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references.
@@ -174,11 +268,39 @@ class ComplexBaseField(BaseField):
             # Document class being used rather than a document object
             return self
 
-        from dereference import dereference
-        instance._data[self.name] = dereference(
-            instance._data.get(self.name), max_depth=1, instance=instance, name=self.name, get=True
+        if not self._dereference and instance._initialised:
+            from dereference import DeReference
+            self._dereference = DeReference()  # Cached
+            instance._data[self.name] = self._dereference(
+                instance._data.get(self.name), max_depth=1, instance=instance,
+                name=self.name
             )
-        return super(ComplexBaseField, self).__get__(instance, owner)
+
+        value = super(ComplexBaseField, self).__get__(instance, owner)
+
+        # Convert lists / values so we can watch for any changes on them
+        if isinstance(value, (list, tuple)) and not isinstance(value, BaseList):
+            value = BaseList(value, instance, self.name)
+            instance._data[self.name] = value
+        elif isinstance(value, dict) and not isinstance(value, BaseDict):
+            value = BaseDict(value, instance, self.name)
+            instance._data[self.name] = value
+
+        if self._dereference and instance._initialised and \
+            isinstance(value, (BaseList, BaseDict)) and not value._dereferenced:
+            value = self._dereference(
+                value, max_depth=1, instance=instance, name=self.name
+            )
+            value._dereferenced = True
+            instance._data[self.name] = value
+
+        return value
+
+    def __set__(self, instance, value):
+        """Descriptor for assigning a value to a field in a document.
+        """
+        instance._data[self.name] = value
+        instance._mark_as_changed(self.name)
 
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
@@ -207,10 +329,10 @@ class ComplexBaseField(BaseField):
                 if isinstance(v, Document):
                     # We need the id from the saved object to create the DBRef
                     if v.pk is None:
-                        raise ValidationError('You can only reference documents once '
-                                      'they have been saved to the database')
+                        self.error('You can only reference documents once they'
+                                   ' have been saved to the database')
                     collection = v._get_collection_name()
-                    value_dict[k] = pymongo.dbref.DBRef(collection, v.pk)
+                    value_dict[k] = DBRef(collection, v.pk)
                 elif hasattr(v, 'to_python'):
                     value_dict[k] = v.to_python()
                 else:
@@ -247,19 +369,19 @@ class ComplexBaseField(BaseField):
                 if isinstance(v, Document):
                     # We need the id from the saved object to create the DBRef
                     if v.pk is None:
-                        raise ValidationError('You can only reference documents once '
-                                      'they have been saved to the database')
+                        self.error('You can only reference documents once they'
+                                   ' have been saved to the database')
 
                     # If its a document that is not inheritable it won't have
                     # _types / _cls data so make it a generic reference allows
                     # us to dereference
                     meta = getattr(v, 'meta', getattr(v, '_meta', {}))
-                    if meta and not meta['allow_inheritance'] and not self.field:
+                    if meta and not meta.get('allow_inheritance', True) and not self.field:
                         from fields import GenericReferenceField
                         value_dict[k] = GenericReferenceField().to_mongo(v)
                     else:
                         collection = v._get_collection_name()
-                        value_dict[k] = pymongo.dbref.DBRef(collection, v.pk)
+                        value_dict[k] = DBRef(collection, v.pk)
                 elif hasattr(v, 'to_mongo'):
                     value_dict[k] = v.to_mongo()
                 else:
@@ -270,17 +392,29 @@ class ComplexBaseField(BaseField):
         return value_dict
 
     def validate(self, value):
-        """If field provided ensure the value is valid.
+        """If field is provided ensure the value is valid.
         """
+        errors = {}
         if self.field:
-            try:
             if hasattr(value, 'iteritems'):
-                    [self.field.validate(v) for k,v in value.iteritems()]
+                sequence = value.iteritems()
             else:
-                    [self.field.validate(v) for v in value]
-            except Exception, err:
-                raise ValidationError('Invalid %s item (%s)' % (
-                        self.field.__class__.__name__, str(v)))
+                sequence = enumerate(value)
+            for k, v in sequence:
+                try:
+                    self.field._validate(v)
+                except (ValidationError, AssertionError), error:
+                    if hasattr(error, 'errors'):
+                        errors[k] = error.errors
+                    else:
+                        errors[k] = error
+            if errors:
+                field_class = self.field.__class__.__name__
+                self.error('Invalid %s item (%s)' % (field_class, value),
+                           errors=errors)
+        # Don't allow empty values if required
+        if self.required and not value:
+            self.error('Field is required and cannot be empty')
 
     def prepare_query_value(self, op, value):
         return self.to_mongo(value)
@@ -309,12 +443,12 @@ class ObjectIdField(BaseField):
         return value
 
     def to_mongo(self, value):
-        if not isinstance(value, pymongo.objectid.ObjectId):
+        if not isinstance(value, ObjectId):
             try:
-                return pymongo.objectid.ObjectId(unicode(value))
+                return ObjectId(unicode(value))
             except Exception, e:
                 # e.message attribute has been deprecated since Python 2.6
-                raise ValidationError(unicode(e))
+                self.error(unicode(e))
         return value
 
     def prepare_query_value(self, op, value):
@@ -322,9 +456,9 @@ class ObjectIdField(BaseField):
 
     def validate(self, value):
         try:
-            pymongo.objectid.ObjectId(unicode(value))
+            ObjectId(unicode(value))
         except:
-            raise ValidationError('Invalid Object ID')
+            self.error('Invalid Object ID')
 
 
 class DocumentMetaclass(type):
@@ -332,6 +466,25 @@ class DocumentMetaclass(type):
     """
 
     def __new__(cls, name, bases, attrs):
+        def _get_mixin_fields(base):
+            attrs = {}
+            attrs.update(dict([(k, v) for k, v in base.__dict__.items()
+                               if issubclass(v.__class__, BaseField)]))
+
+            # Handle simple mixin's with meta
+            if hasattr(base, 'meta') and not isinstance(base, DocumentMetaclass):
+                meta = attrs.get('meta', {})
+                meta.update(base.meta)
+                attrs['meta'] = meta
+
+            for p_base in base.__bases__:
+                #optimize :-)
+                if p_base in (object, BaseDocument):
+                    continue
+
+                attrs.update(_get_mixin_fields(p_base))
+            return attrs
+
         metaclass = attrs.get('__metaclass__')
         super_new = super(DocumentMetaclass, cls).__new__
         if metaclass and issubclass(metaclass, DocumentMetaclass):
@@ -343,6 +496,7 @@ class DocumentMetaclass(type):
         simple_class = True
 
         for base in bases:
+
             # Include all fields present in superclasses
             if hasattr(base, '_fields'):
                 doc_fields.update(base._fields)
@@ -350,14 +504,20 @@ class DocumentMetaclass(type):
                 superclasses[base._class_name] = base
                 superclasses.update(base._superclasses)
             else:  # Add any mixin fields
-                attrs.update(dict([(k,v) for k,v in base.__dict__.items()
-                                    if issubclass(v.__class__, BaseField)]))
+                attrs.update(_get_mixin_fields(base))
 
             if hasattr(base, '_meta') and not base._meta.get('abstract'):
                 # Ensure that the Document class may be subclassed -
                 # inheritance may be disabled to remove dependency on
                 # additional fields _cls and _types
                 class_name.append(base._class_name)
+                if not base._meta.get('allow_inheritance_defined', True):
+                    warnings.warn(
+                        "%s uses inheritance, the default for allow_inheritance "
+                        "is changing to off by default.  Please add it to the "
+                        "document meta." % name,
+                        FutureWarning
+                    )
                 if base._meta.get('allow_inheritance', True) == False:
                     raise ValueError('Document %s may not be subclassed' %
                                      base.__name__)
@@ -365,7 +525,8 @@ class DocumentMetaclass(type):
                     simple_class = False
 
         doc_class_name = '.'.join(reversed(class_name))
-        meta = attrs.get('_meta', attrs.get('meta', {}))
+        meta = attrs.get('_meta', {})
+        meta.update(attrs.get('meta', {}))
 
         if 'allow_inheritance' not in meta:
             meta['allow_inheritance'] = True
@@ -380,6 +541,7 @@ class DocumentMetaclass(type):
         attrs['_superclasses'] = superclasses
 
         # Add the document's fields to the _fields attribute
+        field_names = {}
         for attr_name, attr_value in attrs.items():
             if hasattr(attr_value, "__class__") and \
                issubclass(attr_value.__class__, BaseField):
@@ -387,21 +549,35 @@ class DocumentMetaclass(type):
                 if not attr_value.db_field:
                     attr_value.db_field = attr_name
                 doc_fields[attr_name] = attr_value
+                field_names[attr_value.db_field] = field_names.get(attr_value.db_field, 0) + 1
+
+        duplicate_db_fields = [k for k, v in field_names.items() if v > 1]
+        if duplicate_db_fields:
+            raise InvalidDocumentError("Multiple db_fields defined for: %s " % ", ".join(duplicate_db_fields))
         attrs['_fields'] = doc_fields
         attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k != v.db_field])
         attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()])
 
-        from mongoengine import Document
+        from mongoengine import Document, EmbeddedDocument, DictField
 
         new_class = super_new(cls, name, bases, attrs)
         for field in new_class._fields.values():
             field.owner_document = new_class
-            delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING)
-            if delete_rule != DO_NOTHING:
-                field.document_type.register_delete_rule(new_class, field.name,
-                        delete_rule)
 
-            if field.name and hasattr(Document, field.name):
+            delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING)
+            f = field
+            if isinstance(f, ComplexBaseField) and hasattr(f, 'field'):
+                delete_rule = getattr(f.field, 'reverse_delete_rule', DO_NOTHING)
+                if isinstance(f, DictField) and delete_rule != DO_NOTHING:
+                    raise InvalidDocumentError("Reverse delete rules are not supported for %s (field: %s)" % (field.__class__.__name__, field.name))
+                f = field.field
+
+            if delete_rule != DO_NOTHING:
+                if issubclass(new_class, EmbeddedDocument):
+                    raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name)
+                f.document_type.register_delete_rule(new_class, field.name, delete_rule)
+
+            if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro():
                 raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name)
 
         module = attrs.get('__module__')
@@ -453,6 +629,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         collection = ''.join('_%s' % c if c.isupper() else c for c in name).strip('_').lower()
 
         id_field = None
+        abstract_base_indexes = []
         base_indexes = []
         base_meta = {}
 
@@ -472,6 +649,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                         base_meta[key] = base._meta[key]
 
                 id_field = id_field or base._meta.get('id_field')
+                if base._meta.get('abstract', False):
+                    abstract_base_indexes += base._meta.get('indexes', [])
+                else:
                     base_indexes += base._meta.get('indexes', [])
                 # Propagate 'allow_inheritance'
                 if 'allow_inheritance' in base._meta:
@@ -480,6 +660,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                     base_meta['queryset_class'] = base._meta['queryset_class']
             try:
                 base_meta['objects'] = base.__getattribute__(base, 'objects')
+            except TypeError:
+                pass
             except AttributeError:
                 pass
 
@@ -498,6 +680,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             'delete_rules': {},
             'allow_inheritance': True
         }
+
+        allow_inheritance_defined = ('allow_inheritance' in base_meta or
+                                     'allow_inheritance'in attrs.get('meta', {}))
+        meta['allow_inheritance_defined'] = allow_inheritance_defined
         meta.update(base_meta)
 
         # Apply document-defined meta options
@@ -518,8 +704,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             meta['queryset_class'] = manager.queryset_class
         new_class.objects = manager
 
+        indicies = meta['indexes'] + abstract_base_indexes
         user_indexes = [QuerySet._build_index_spec(new_class, spec)
-                        for spec in meta['indexes']] + base_indexes
+                        for spec in indicies] + base_indexes
         new_class._meta['indexes'] = user_indexes
 
         unique_indexes = cls._unique_with_indexes(new_class)
@@ -575,7 +762,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                 unique_indexes.append(index)
 
             # Grab any embedded document field unique indexes
-            if field.__class__.__name__ == "EmbeddedDocumentField":
+            if field.__class__.__name__ == "EmbeddedDocumentField" and field.document_type != new_class:
                 field_namespace = "%s." % field_name
                 unique_indexes += cls._unique_with_indexes(field.document_type,
                                     field_namespace)
@@ -585,30 +772,109 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
 class BaseDocument(object):
 
+    _dynamic = False
+    _created = True
+    _dynamic_lock = True
+    _initialised = False
+
     def __init__(self, **values):
         signals.pre_init.send(self.__class__, document=self, values=values)
 
         self._data = {}
-        self._initialised = False
+
         # Assign default values to instance
         for attr_name, field in self._fields.items():
             value = getattr(self, attr_name, None)
             setattr(self, attr_name, value)
 
-        # Assign initial values to instance
-        for attr_name in values.keys():
-            try:
-                value = values.pop(attr_name)
-                setattr(self, attr_name, value)
-            except AttributeError:
-                pass
+        # Set passed values after initialisation
+        if self._dynamic:
+            self._dynamic_fields = {}
+            dynamic_data = {}
+            for key, value in values.items():
+                if key in self._fields or key == '_id':
+                    setattr(self, key, value)
+                elif self._dynamic:
+                    dynamic_data[key] = value
+        else:
+            for key, value in values.items():
+                key = self._reverse_db_field_map.get(key, key)
+                setattr(self, key, value)
 
         # Set any get_fieldname_display methods
         self.__set_field_display()
+
+        if self._dynamic:
+            self._dynamic_lock = False
+            for key, value in dynamic_data.items():
+                setattr(self, key, value)
+
         # Flag initialised
         self._initialised = True
         signals.post_init.send(self.__class__, document=self)
 
+    def __setattr__(self, name, value):
+        # Handle dynamic data only if an initialised dynamic document
+        if self._dynamic and not self._dynamic_lock:
+
+            field = None
+            if not hasattr(self, name) and not name.startswith('_'):
+                from fields import DynamicField
+                field = DynamicField(db_field=name)
+                field.name = name
+                self._dynamic_fields[name] = field
+
+            if not name.startswith('_'):
+                value = self.__expand_dynamic_values(name, value)
+
+            # Handle marking data as changed
+            if name in self._dynamic_fields:
+                self._data[name] = value
+                if hasattr(self, '_changed_fields'):
+                    self._mark_as_changed(name)
+
+        if not self._created and name in self._meta.get('shard_key', tuple()):
+            from queryset import OperationError
+            raise OperationError("Shard Keys are immutable. Tried to update %s" % name)
+
+        super(BaseDocument, self).__setattr__(name, value)
+
+    def __expand_dynamic_values(self, name, value):
+        """expand any dynamic values to their correct types / values"""
+        if not isinstance(value, (dict, list, tuple)):
+            return value
+
+        is_list = False
+        if not hasattr(value, 'items'):
+            is_list = True
+            value = dict([(k, v) for k, v in enumerate(value)])
+
+        if not is_list and '_cls' in value:
+            cls = get_document(value['_cls'])
+            value = cls(**value)
+            value._dynamic = True
+            value._changed_fields = []
+            return value
+
+        data = {}
+        for k, v in value.items():
+            key = name if is_list else k
+            data[k] = self.__expand_dynamic_values(key, v)
+
+        if is_list:  # Convert back to a list
+            data_items = sorted(data.items(), key=operator.itemgetter(0))
+            value = [v for k, v in data_items]
+        else:
+            value = data
+
+        # Convert lists / values so we can watch for any changes on them
+        if isinstance(value, (list, tuple)) and not isinstance(value, BaseList):
+            value = BaseList(value, self, name)
+        elif isinstance(value, dict) and not isinstance(value, BaseDict):
+            value = BaseDict(value, self, name)
+
+        return value
+
     def validate(self):
         """Ensure that all fields' values are valid and that required fields
         are present.
@@ -618,25 +884,21 @@ class BaseDocument(object):
                   for name, field in self._fields.items()]
 
         # Ensure that each field is matched to a valid value
+        errors = {}
         for field, value in fields:
             if value is not None:
                 try:
                     field._validate(value)
-                except (ValueError, AttributeError, AssertionError), e:
-                    raise ValidationError('Invalid value for field named "%s" of type "%s": %s'
-                                          % (field.name, field.__class__.__name__, value))
+                except ValidationError, error:
+                    errors[field.name] = error.errors or error
+                except (ValueError, AttributeError, AssertionError), error:
+                    errors[field.name] = error
             elif field.required:
-                raise ValidationError('Field "%s" is required' % field.name)
-
-    @apply
-    def pk():
-        """Primary key alias
-        """
-        def fget(self):
-            return getattr(self, self._meta['id_field'])
-        def fset(self, value):
-            return setattr(self, self._meta['id_field'], value)
-        return property(fget, fset)
+                errors[field.name] = ValidationError('Field is required',
+                                                     field_name=field.name)
+        if errors:
+            raise ValidationError('Errors encountered validating document',
+                                  errors=errors)
 
     def to_mongo(self):
         """Return data dictionary ready for use with MongoDB.
@@ -653,6 +915,12 @@ class BaseDocument(object):
             data['_types'] = self._superclasses.keys() + [self._class_name]
         if '_id' in data and data['_id'] is None:
             del data['_id']
+
+        if not self._dynamic:
+            return data
+
+        for name, field in self._dynamic_fields.items():
+            data[name] = field.to_mongo(self._data.get(name, None))
         return data
 
     @classmethod
@@ -661,29 +929,14 @@ class BaseDocument(object):
         """
         return cls._meta.get('collection', None)
 
-    @classmethod
-    def _get_subclasses(cls):
-        """Return a dictionary of all subclasses (found recursively).
-        """
-        try:
-            subclasses = cls.__subclasses__()
-        except:
-            subclasses = cls.__subclasses__(cls)
-
-        all_subclasses = {}
-        for subclass in subclasses:
-            all_subclasses[subclass._class_name] = subclass
-            all_subclasses.update(subclass._get_subclasses())
-        return all_subclasses
-
     @classmethod
     def _from_son(cls, son):
         """Create an instance of a Document (subclass) from a PyMongo SON.
         """
         # get the class name from the document, falling back to the given
         # class if unavailable
-        class_name = son.get(u'_cls', cls._class_name)
-        data = dict((str(key), value) for key, value in son.items())
+        class_name = son.get('_cls', cls._class_name)
+        data = dict(("%s" % key, value) for key, value in son.items())
 
         if '_types' in data:
             del data['_types']
@@ -693,26 +946,35 @@ class BaseDocument(object):
 
         # Return correct subclass for document type
         if class_name != cls._class_name:
-            subclasses = cls._get_subclasses()
-            if class_name not in subclasses:
-                # Type of document is probably more generic than the class
-                # that has been queried to return this SON
-                raise NotRegistered("""
-                        `%s` has not been registered in the document registry.
-                        Importing the document class automatically registers it,
-                        has it been imported?
-                    """.strip() % class_name)
-            cls = subclasses[class_name]
+            cls = get_document(class_name)
+
+        changed_fields = []
+        errors_dict = {}
 
-        present_fields = data.keys()
         for field_name, field in cls._fields.items():
             if field.db_field in data:
                 value = data[field.db_field]
+                try:
                     data[field_name] = (value if value is None
                                     else field.to_python(value))
+                except (AttributeError, ValueError), e:
+                    errors_dict[field_name] = e
+            elif field.default:
+                default = field.default
+                if callable(default):
+                    default = default()
+                if isinstance(default, BaseDocument):
+                    changed_fields.append(field_name)
+
+        if errors_dict:
+            errors = "\n".join(["%s - %s" % (k, v) for k, v in errors_dict.items()])
+            raise InvalidDocumentError("""
+Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors))
 
         obj = cls(**data)
-        obj._changed_fields = []
+
+        obj._changed_fields = changed_fields
+        obj._created = False
         return obj
 
     def _mark_as_changed(self, key):
@@ -727,7 +989,7 @@ class BaseDocument(object):
     def _get_changed_fields(self, key='', inspected=None):
         """Returns a list of all fields that have explicitly been changed.
         """
-        from mongoengine import EmbeddedDocument
+        from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument
         _changed_fields = []
         _changed_fields += getattr(self, '_changed_fields', [])
 
@@ -738,6 +1000,8 @@ class BaseDocument(object):
             inspected.add(self.id)
 
         field_list = self._fields.copy()
+        if self._dynamic:
+            field_list.update(self._dynamic_fields)
 
         for field_name in field_list:
             db_field_name = self._db_field_map.get(field_name, field_name)
@@ -748,7 +1012,7 @@ class BaseDocument(object):
                     continue
                 inspected.add(field.id)
 
-            if isinstance(field, (EmbeddedDocument,)) and db_field_name not in _changed_fields:  # Grab all embedded fields that have been changed
+            if isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields:  # Grab all embedded fields that have been changed
                 _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key, inspected) if k]
             elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields:  # Loop list / dict fields as they contain documents
                 # Determine the iterator to use
@@ -772,19 +1036,23 @@ class BaseDocument(object):
         set_fields = self._get_changed_fields()
         set_data = {}
         unset_data = {}
+        parts = []
         if hasattr(self, '_changed_fields'):
             set_data = {}
             # Fetch each set item from its path
             for path in set_fields:
                 parts = path.split('.')
                 d = doc
+                new_path = []
                 for p in parts:
-                    if hasattr(d, '__getattr__'):
-                        d = getattr(p, d)
+                    if isinstance(d, DBRef):
+                        break
                     elif p.isdigit():
                         d = d[int(p)]
-                    else:
+                    elif hasattr(d, 'get'):
                         d = d.get(p)
+                    new_path.append(p)
+                path = '.'.join(new_path)
                 set_data[path] = d
         else:
             set_data = doc
@@ -793,13 +1061,16 @@ class BaseDocument(object):
 
         # Determine if any changed items were actually unset.
         for path, value in set_data.items():
-            if value:
+            if value or isinstance(value, bool):
                 continue
 
             # If we've set a value that ain't the default value dont unset it.
             default = None
-
-            if path in self._fields:
+            if self._dynamic and len(parts) and parts[0] in self._dynamic_fields:
+                del(set_data[path])
+                unset_data[path] = 1
+                continue
+            elif path in self._fields:
                 default = self._fields[path].default
             else:  # Perform a full lookup for lists / embedded lookups
                 d = self
@@ -818,7 +1089,10 @@ class BaseDocument(object):
                     field_name = d._reverse_db_field_map.get(db_field_name,
                                                              db_field_name)
 
-                    default = d._fields[field_name].default
+                    if field_name in d._fields:
+                        default = d._fields.get(field_name).default
+                    else:
+                        default = None
 
             if default is not None:
                 if callable(default):
@@ -831,23 +1105,22 @@ class BaseDocument(object):
         return set_data, unset_data
 
     @classmethod
-    def _geo_indices(cls, inspected_classes=None):
-        inspected_classes = inspected_classes or []
+    def _geo_indices(cls, inspected=None):
+        inspected = inspected or []
         geo_indices = []
-        inspected_classes.append(cls)
+        inspected.append(cls)
         for field in cls._fields.values():
             if hasattr(field, 'document_type'):
                 field_cls = field.document_type
-                if field_cls in inspected_classes:
+                if field_cls in inspected:
                     continue
                 if hasattr(field_cls, '_geo_indices'):
-                    geo_indices += field_cls._geo_indices(inspected_classes)
+                    geo_indices += field_cls._geo_indices(inspected)
             elif field._geo_index:
                 geo_indices.append(field)
         return geo_indices
 
     def __getstate__(self):
-        self_dict = self.__dict__
         removals = ["get_%s_display" % k for k, v in self._fields.items() if v.choices]
         for k in removals:
             if hasattr(self, k):
@@ -866,7 +1139,9 @@ class BaseDocument(object):
     def __get_field_display(self, field):
         """Returns the display value for a choice field"""
         value = getattr(self, field.name)
+        if field.choices and isinstance(field.choices[0], (list, tuple)):
             return dict(field.choices).get(value, value)
+        return value
 
     def __iter__(self):
         return iter(self._fields)
@@ -901,10 +1176,10 @@ class BaseDocument(object):
 
     def __repr__(self):
         try:
-            u = unicode(self)
+            u = unicode(self).encode('utf-8')
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = '[Bad Unicode data]'
-        return u'<%s: %s>' % (self.__class__.__name__, u)
+        return '<%s: %s>' % (self.__class__.__name__, u)
 
     def __str__(self):
         if hasattr(self, '__unicode__'):
@@ -932,18 +1207,30 @@ class BaseList(list):
     """A special list so we can watch any changes
     """
 
+    _dereferenced = False
+    _instance = None
+    _name = None
+
     def __init__(self, list_items, instance, name):
-        self.instance = instance
-        self.name = name
-        super(BaseList, self).__init__(list_items)
+        self._instance = instance
+        self._name = name
+        return super(BaseList, self).__init__(list_items)
 
     def __setitem__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseList, self).__setitem__(*args, **kwargs)
+        return super(BaseList, self).__setitem__(*args, **kwargs)
 
     def __delitem__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseList, self).__delitem__(*args, **kwargs)
+        return super(BaseList, self).__delitem__(*args, **kwargs)
+
+    def __getstate__(self):
+        self.observer = None
+        return self
+
+    def __setstate__(self, state):
+        self = state
+        return self
 
     def append(self, *args, **kwargs):
         self._mark_as_changed()
@@ -974,56 +1261,67 @@ class BaseList(list):
         return super(BaseList, self).sort(*args, **kwargs)
 
     def _mark_as_changed(self):
-        """Marks a list as changed if has an instance and a name"""
-        if hasattr(self, 'instance') and hasattr(self, 'name'):
-            self.instance._mark_as_changed(self.name)
+        if hasattr(self._instance, '_mark_as_changed'):
+            self._instance._mark_as_changed(self._name)
 
 
 class BaseDict(dict):
     """A special dict so we can watch any changes
     """
 
+    _dereferenced = False
+    _instance = None
+    _name = None
+
     def __init__(self, dict_items, instance, name):
-        self.instance = instance
-        self.name = name
-        super(BaseDict, self).__init__(dict_items)
+        self._instance = instance
+        self._name = name
+        return super(BaseDict, self).__init__(dict_items)
 
     def __setitem__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).__setitem__(*args, **kwargs)
-
-    def __setattr__(self, *args, **kwargs):
-        self._mark_as_changed()
-        super(BaseDict, self).__setattr__(*args, **kwargs)
+        return super(BaseDict, self).__setitem__(*args, **kwargs)
 
     def __delete__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).__delete__(*args, **kwargs)
+        return super(BaseDict, self).__delete__(*args, **kwargs)
 
     def __delitem__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).__delitem__(*args, **kwargs)
+        return super(BaseDict, self).__delitem__(*args, **kwargs)
 
     def __delattr__(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).__delattr__(*args, **kwargs)
+        return super(BaseDict, self).__delattr__(*args, **kwargs)
+
+    def __getstate__(self):
+        self.instance = None
+        self._dereferenced = False
+        return self
+
+    def __setstate__(self, state):
+        self = state
+        return self
 
     def clear(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).clear(*args, **kwargs)
+        return super(BaseDict, self).clear(*args, **kwargs)
 
     def pop(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).clear(*args, **kwargs)
+        return super(BaseDict, self).pop(*args, **kwargs)
 
     def popitem(self, *args, **kwargs):
         self._mark_as_changed()
-        super(BaseDict, self).clear(*args, **kwargs)
+        return super(BaseDict, self).popitem(*args, **kwargs)
+
+    def update(self, *args, **kwargs):
+        self._mark_as_changed()
+        return super(BaseDict, self).update(*args, **kwargs)
 
     def _mark_as_changed(self):
-        """Marks a dict as changed if has an instance and a name"""
-        if hasattr(self, 'instance') and hasattr(self, 'name'):
-            self.instance._mark_as_changed(self.name)
+        if hasattr(self._instance, '_mark_as_changed'):
+            self._instance._mark_as_changed(self._name)
 
 if sys.version_info < (2, 5):
     # Prior to Python 2.5, Exception was an old-style class

mongoengine/connection.py

@@ -1,82 +1,166 @@
-from pymongo import Connection
-import multiprocessing
-import threading
-
-__all__ = ['ConnectionError', 'connect']
+import pymongo
+from pymongo import Connection, ReplicaSetConnection, uri_parser
 
 
-_connection_defaults = {
-    'host': 'localhost',
-    'port': 27017,
-}
-_connection = {}
-_connection_settings = _connection_defaults.copy()
+__all__ = ['ConnectionError', 'connect', 'register_connection',
+           'DEFAULT_CONNECTION_NAME']
 
-_db_name = None
-_db_username = None
-_db_password = None
-_db = {}
+
+DEFAULT_CONNECTION_NAME = 'default'
 
 
 class ConnectionError(Exception):
     pass
 
 
-def _get_connection(reconnect=False):
-    """Handles the connection to the database
+_connection_settings = {}
+_connections = {}
+_dbs = {}
+
+
+def register_connection(alias, name, host='localhost', port=27017,
+                        is_slave=False, read_preference=False, slaves=None,
+                        username=None, password=None, **kwargs):
+    """Add a connection.
+
+    :param alias: the name that will be used to refer to this connection
+        throughout MongoEngine
+    :param name: the name of the specific database to use
+    :param host: the host name of the :program:`mongod` instance to connect to
+    :param port: the port that the :program:`mongod` instance is running on
+    :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+
+    :param read_preference: The read preference for the collection ** Added pymongo 2.1
+    :param slaves: a list of aliases of slave connections; each of these must
+        be a registered connection that has :attr:`is_slave` set to ``True``
+    :param username: username to authenticate with
+    :param password: password to authenticate with
+    :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver
+
     """
-    global _connection
-    identity = get_identity()
+    global _connection_settings
+
+    conn_settings = {
+        'name': name,
+        'host': host,
+        'port': port,
+        'is_slave': is_slave,
+        'slaves': slaves or [],
+        'username': username,
+        'password': password,
+        'read_preference': read_preference
+    }
+
+    # Handle uri style connections
+    if "://" in host:
+        uri_dict = uri_parser.parse_uri(host)
+        if uri_dict.get('database') is None:
+            raise ConnectionError("If using URI style connection include "\
+                                  "database name in string")
+        conn_settings.update({
+            'host': host,
+            'name': uri_dict.get('database'),
+            'username': uri_dict.get('username'),
+            'password': uri_dict.get('password'),
+            'read_preference': read_preference,
+        })
+        if "replicaSet" in host:
+            conn_settings['replicaSet'] = True
+
+    conn_settings.update(kwargs)
+    _connection_settings[alias] = conn_settings
+
+
+def disconnect(alias=DEFAULT_CONNECTION_NAME):
+    global _connections
+    global _dbs
+
+    if alias in _connections:
+        get_connection(alias=alias).disconnect()
+        del _connections[alias]
+    if alias in _dbs:
+        del _dbs[alias]
+
+
+def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
+    global _connections
     # Connect to the database if not already connected
-    if _connection.get(identity) is None or reconnect:
+    if reconnect:
+        disconnect(alias)
+
+    if alias not in _connections:
+        if alias not in _connection_settings:
+            msg = 'Connection with alias "%s" has not been defined' % alias
+            if alias == DEFAULT_CONNECTION_NAME:
+                msg = 'You have not defined a default connection'
+            raise ConnectionError(msg)
+        conn_settings = _connection_settings[alias].copy()
+
+        if hasattr(pymongo, 'version_tuple'):  # Support for 2.1+
+            conn_settings.pop('name', None)
+            conn_settings.pop('slaves', None)
+            conn_settings.pop('is_slave', None)
+            conn_settings.pop('username', None)
+            conn_settings.pop('password', None)
+        else:
+            # Get all the slave connections
+            if 'slaves' in conn_settings:
+                slaves = []
+                for slave_alias in conn_settings['slaves']:
+                    slaves.append(get_connection(slave_alias))
+                conn_settings['slaves'] = slaves
+                conn_settings.pop('read_preference', None)
+
+        connection_class = Connection
+        if 'replicaSet' in conn_settings:
+            conn_settings['hosts_or_uri'] = conn_settings.pop('host', None)
+            # Discard port since it can't be used on ReplicaSetConnection
+            conn_settings.pop('port', None)
+            # Discard replicaSet if not base string
+            if not isinstance(conn_settings['replicaSet'], basestring):
+                conn_settings.pop('replicaSet', None)
+            connection_class = ReplicaSetConnection
+
         try:
-            _connection[identity] = Connection(**_connection_settings)
+            _connections[alias] = connection_class(**conn_settings)
         except Exception, e:
-            raise ConnectionError("Cannot connect to the database:\n%s" % e)
-    return _connection[identity]
+            raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e))
+    return _connections[alias]
 
-def _get_db(reconnect=False):
-    """Handles database connections and authentication based on the current
-    identity
+
+def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
+    global _dbs
+    if reconnect:
+        disconnect(alias)
+
+    if alias not in _dbs:
+        conn = get_connection(alias)
+        conn_settings = _connection_settings[alias]
+        _dbs[alias] = conn[conn_settings['name']]
+        # Authenticate if necessary
+        if conn_settings['username'] and conn_settings['password']:
+            _dbs[alias].authenticate(conn_settings['username'],
+                                     conn_settings['password'])
+    return _dbs[alias]
+
+
+def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs):
+    """Connect to the database specified by the 'db' argument.
+
+    Connection settings may be provided here as well if the database is not
+    running on the default port on localhost. If authentication is needed,
+    provide username and password arguments as well.
+
+    Multiple databases are supported by using aliases.  Provide a separate
+    `alias` to connect to a different instance of :program:`mongod`.
+
+    .. versionchanged:: 0.6 - added multiple database support.
     """
-    global _db, _connection
-    identity = get_identity()
-    # Connect if not already connected
-    if _connection.get(identity) is None or reconnect:
-        _connection[identity] = _get_connection(reconnect=reconnect)
+    global _connections
+    if alias not in _connections:
+        register_connection(alias, db, **kwargs)
 
-    if _db.get(identity) is None or reconnect:
-        # _db_name will be None if the user hasn't called connect()
-        if _db_name is None:
-            raise ConnectionError('Not connected to the database')
-
-        # Get DB from current connection and authenticate if necessary
-        _db[identity] = _connection[identity][_db_name]
-        if _db_username and _db_password:
-            _db[identity].authenticate(_db_username, _db_password)
-
-    return _db[identity]
-
-def get_identity():
-    """Creates an identity key based on the current process and thread
-    identity.
-    """
-    identity = multiprocessing.current_process()._identity
-    identity = 0 if not identity else identity[0]
-
-    identity = (identity, threading.current_thread().ident)
-    return identity
-
-def connect(db, username=None, password=None, **kwargs):
-    """Connect to the database specified by the 'db' argument. Connection
-    settings may be provided here as well if the database is not running on
-    the default port on localhost. If authentication is needed, provide
-    username and password arguments as well.
-    """
-    global _connection_settings, _db_name, _db_username, _db_password, _db
-    _connection_settings = dict(_connection_defaults, **kwargs)
-    _db_name = db
-    _db_username = username
-    _db_password = password
-    return _get_db(reconnect=True)
+    return get_connection(alias)
 
+# Support old naming convention
+_get_connection = get_connection
+_get_db = get_db

mongoengine/dereference.py

@@ -1,17 +1,15 @@
-import operator
+from bson import DBRef, SON
 
-import pymongo
-
-from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass
-from fields import ReferenceField
-from connection import _get_db
+from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document)
+from fields import (ReferenceField, ListField, DictField, MapField)
+from connection import get_db
 from queryset import QuerySet
 from document import Document
 
 
 class DeReference(object):
 
-    def __call__(self, items, max_depth=1, instance=None, name=None, get=False):
+    def __call__(self, items, max_depth=1, instance=None, name=None):
         """
         Cheaply dereferences the items to a set depth.
         Also handles the convertion of complex data types.
@@ -45,7 +43,7 @@ class DeReference(object):
 
         self.reference_map = self._find_references(items)
         self.object_map = self._fetch_objects(doc_type=doc_type)
-        return self._attach_objects(items, 0, instance, name, get)
+        return self._attach_objects(items, 0, instance, name)
 
     def _find_references(self, items, depth=0):
         """
@@ -55,7 +53,7 @@ class DeReference(object):
         :param depth: The current depth of recursion
         """
         reference_map = {}
-        if not items:
+        if not items or depth >= self.max_depth:
             return reference_map
 
         # Determine the iterator to use
@@ -65,13 +63,14 @@ class DeReference(object):
             iterator = items.iteritems()
 
         # Recursively find dbreferences
+        depth += 1
         for k, item in iterator:
             if hasattr(item, '_fields'):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, (pymongo.dbref.DBRef)):
+                    if isinstance(v, (DBRef)):
                         reference_map.setdefault(field.document_type, []).append(v.id)
-                    elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v:
+                    elif isinstance(v, (dict, SON)) and '_ref' in v:
                         reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id)
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                         field_cls = getattr(getattr(field, 'field', None), 'document_type', None)
@@ -80,15 +79,15 @@ class DeReference(object):
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
                             reference_map.setdefault(key, []).extend(refs)
-            elif isinstance(item, (pymongo.dbref.DBRef)):
+            elif isinstance(item, (DBRef)):
                 reference_map.setdefault(item.collection, []).append(item.id)
-            elif isinstance(item, (dict, pymongo.son.SON)) and '_ref' in item:
+            elif isinstance(item, (dict, SON)) and '_ref' in item:
                 reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id)
-            elif isinstance(item, (dict, list, tuple)) and depth <= self.max_depth:
-                references = self._find_references(item, depth)
+            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:
+                references = self._find_references(item, depth - 1)
                 for key, refs in references.iteritems():
                     reference_map.setdefault(key, []).extend(refs)
-        depth += 1
+
         return reference_map
 
     def _fetch_objects(self, doc_type=None):
@@ -103,16 +102,26 @@ class DeReference(object):
                 for key, doc in references.iteritems():
                     object_map[key] = doc
             else:  # Generic reference: use the refs data to convert to document
-                references = _get_db()[col].find({'_id': {'$in': refs}})
+                if doc_type and not isinstance(doc_type, (ListField, DictField, MapField,) ):
+                    references = doc_type._get_db()[col].find({'_id': {'$in': refs}})
+                    for ref in references:
+                        doc = doc_type._from_son(ref)
+                        object_map[doc.id] = doc
+                else:
+                    references = get_db()[col].find({'_id': {'$in': refs}})
                     for ref in references:
                         if '_cls' in ref:
-                        doc = get_document(ref['_cls'])._from_son(ref)
+                            doc = get_document(ref["_cls"])._from_son(ref)
+                        elif doc_type is None:
+                            doc = get_document(
+                                ''.join(x.capitalize()
+                                        for x in col.split('_')))._from_son(ref)
                         else:
                             doc = doc_type._from_son(ref)
                         object_map[doc.id] = doc
         return object_map
 
-    def _attach_objects(self, items, depth=0, instance=None, name=None, get=False):
+    def _attach_objects(self, items, depth=0, instance=None, name=None):
         """
         Recursively finds all db references to be dereferenced
 
@@ -122,7 +131,6 @@ class DeReference(object):
             :class:`~mongoengine.base.ComplexBaseField`
         :param name: The name of the field, used for tracking changes by
             :class:`~mongoengine.base.ComplexBaseField`
-        :param get: A boolean determining if being called by __get__
         """
         if not items:
             if isinstance(items, (BaseDict, BaseList)):
@@ -130,17 +138,16 @@ class DeReference(object):
 
             if instance:
                 if isinstance(items, dict):
-                    return BaseDict(items, instance=instance, name=name)
+                    return BaseDict(items, instance, name)
                 else:
-                    return BaseList(items, instance=instance, name=name)
+                    return BaseList(items, instance, name)
 
-        if isinstance(items, (dict, pymongo.son.SON)):
+        if isinstance(items, (dict, SON)):
             if '_ref' in items:
                 return self.object_map.get(items['_ref'].id, items)
             elif '_types' in items and '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
-                if not get:
-                    doc._data = self._attach_objects(doc._data, depth, doc, name, get)
+                doc._data = self._attach_objects(doc._data, depth, doc, name)
                 return doc
 
         if not hasattr(items, 'items'):
@@ -152,6 +159,7 @@ class DeReference(object):
             iterator = items.iteritems()
             data = {}
 
+        depth += 1
         for k, v in iterator:
             if is_list:
                 data.append(v)
@@ -163,24 +171,22 @@ class DeReference(object):
             elif hasattr(v, '_fields'):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
-                    if isinstance(v, (pymongo.dbref.DBRef)):
+                    if isinstance(v, (DBRef)):
                         data[k]._data[field_name] = self.object_map.get(v.id, v)
-                    elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v:
+                    elif isinstance(v, (dict, SON)) and '_ref' in v:
                         data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
-                    elif isinstance(v, dict) and depth < self.max_depth:
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
-                    elif isinstance(v, (list, tuple)):
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
-            elif isinstance(v, (dict, list, tuple)) and depth < self.max_depth:
-                data[k] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
+                    elif isinstance(v, dict) and depth <= self.max_depth:
+                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+                    elif isinstance(v, (list, tuple)) and depth <= self.max_depth:
+                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
+                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name)
             elif hasattr(v, 'id'):
                 data[k] = self.object_map.get(v.id, v)
 
         if instance and name:
             if is_list:
-                return BaseList(data, instance=instance, name=name)
-            return BaseDict(data, instance=instance, name=name)
+                return BaseList(data, instance, name)
+            return BaseDict(data, instance, name)
         depth += 1
         return data
-
-dereference = DeReference()

mongoengine/django/auth.py

@@ -1,13 +1,16 @@
+import datetime
+
 from mongoengine import *
 
-from django.utils.hashcompat import md5_constructor, sha_constructor
 from django.utils.encoding import smart_str
 from django.contrib.auth.models import AnonymousUser
 from django.utils.translation import ugettext_lazy as _
 
-import datetime
-
-REDIRECT_FIELD_NAME = 'next'
+try:
+    from django.contrib.auth.hashers import check_password, make_password
+except ImportError:
+    """Handle older versions of Django"""
+    from django.utils.hashcompat import md5_constructor, sha_constructor
 
     def get_hexdigest(algorithm, salt, raw_password):
         raw_password, salt = smart_str(raw_password), smart_str(salt)
@@ -17,6 +20,19 @@ def get_hexdigest(algorithm, salt, raw_password):
             return sha_constructor(salt + raw_password).hexdigest()
         raise ValueError('Got unknown password algorithm type in password')
 
+    def check_password(raw_password, password):
+        algo, salt, hash = password.split('$')
+        return hash == get_hexdigest(algo, salt, raw_password)
+
+    def make_password(raw_password):
+        from random import random
+        algo = 'sha1'
+        salt = get_hexdigest(algo, str(random()), str(random()))[:5]
+        hash = get_hexdigest(algo, salt, raw_password)
+        return '%s$%s$%s' % (algo, salt, hash)
+
+
+REDIRECT_FIELD_NAME = 'next'
 
 class User(Document):
     """A User document that aims to mirror most of the API specified by Django
@@ -34,7 +50,7 @@ class User(Document):
     email = EmailField(verbose_name=_('e-mail address'))
     password = StringField(max_length=128,
                            verbose_name=_('password'),
-                           help_text=_("Use '[algo]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."))
+                           help_text=_("Use '[algo]$[iterations]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."))
     is_staff = BooleanField(default=False,
                             verbose_name=_('staff status'),
                             help_text=_("Designates whether the user can log into this admin site."))
@@ -50,6 +66,7 @@ class User(Document):
                                 verbose_name=_('date joined'))
 
     meta = {
+        'allow_inheritance': True,
         'indexes': [
             {'fields': ['username'], 'unique': True}
         ]
@@ -75,11 +92,7 @@ class User(Document):
         assigning to :attr:`~mongoengine.django.auth.User.password` as the
         password is hashed before storage.
         """
-        from random import random
-        algo = 'sha1'
-        salt = get_hexdigest(algo, str(random()), str(random()))[:5]
-        hash = get_hexdigest(algo, salt, raw_password)
-        self.password = '%s$%s$%s' % (algo, salt, hash)
+        self.password = make_password(raw_password)
         self.save()
         return self
 
@@ -89,8 +102,7 @@ class User(Document):
         :attr:`~mongoengine.django.auth.User.password` as the password is
         hashed before storage.
         """
-        algo, salt, hash = self.password.split('$')
-        return hash == get_hexdigest(algo, salt, raw_password)
+        return check_password(raw_password, self.password)
 
     @classmethod
     def create_user(cls, username, password, email=None):

mongoengine/django/sessions.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+from django.conf import settings
 from django.contrib.sessions.backends.base import SessionBase, CreateError
 from django.core.exceptions import SuspiciousOperation
 from django.utils.encoding import force_unicode
@@ -5,8 +8,12 @@ from django.utils.encoding import force_unicode
 from mongoengine.document import Document
 from mongoengine import fields
 from mongoengine.queryset import OperationError
+from mongoengine.connection import DEFAULT_CONNECTION_NAME
 
-from datetime import datetime
+
+MONGOENGINE_SESSION_DB_ALIAS = getattr(
+    settings, 'MONGOENGINE_SESSION_DB_ALIAS',
+    DEFAULT_CONNECTION_NAME)
 
 
 class MongoSession(Document):
@@ -14,7 +21,9 @@ class MongoSession(Document):
     session_data = fields.StringField()
     expire_date = fields.DateTimeField()
 
-    meta = {'collection': 'django_session', 'allow_inheritance': False}
+    meta = {'collection': 'django_session',
+            'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
+            'allow_inheritance': False}
 
 
 class SessionStore(SessionBase):
@@ -35,7 +44,7 @@ class SessionStore(SessionBase):
 
     def create(self):
         while True:
-            self.session_key = self._get_new_session_key()
+            self._session_key = self._get_new_session_key()
             try:
                 self.save(must_create=True)
             except CreateError:
@@ -45,6 +54,8 @@ class SessionStore(SessionBase):
             return
 
     def save(self, must_create=False):
+        if self.session_key is None:
+            self._session_key = self._get_new_session_key()
         s = MongoSession(session_key=self.session_key)
         s.session_data = self.encode(self._get_session(no_load=must_create))
         s.expire_date = self.get_expiry_date()

mongoengine/django/tests.py

@@ -10,7 +10,7 @@ class MongoTestCase(TestCase):
     """
     db_name = 'test_%s' % settings.MONGO_DATABASE_NAME
     def __init__(self, methodName='runtest'):
-        self.db = connect(self.db_name)
+        self.db = connect(self.db_name).get_db()
         super(MongoTestCase, self).__init__(methodName)
 
     def _post_teardown(self):

mongoengine/document.py

@@ -1,13 +1,15 @@
-from mongoengine import signals
-from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument,
-                  ValidationError, BaseDict, BaseList)
-from queryset import OperationError
-from connection import _get_db
-
 import pymongo
 
-__all__ = ['Document', 'EmbeddedDocument', 'ValidationError',
-           'OperationError', 'InvalidCollectionError']
+from bson.dbref import DBRef
+
+from mongoengine import signals
+from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument,
+                  BaseDict, BaseList)
+from queryset import OperationError
+from connection import get_db, DEFAULT_CONNECTION_NAME
+
+__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument',
+           'DynamicEmbeddedDocument', 'OperationError', 'InvalidCollectionError']
 
 
 class InvalidCollectionError(Exception):
@@ -23,6 +25,10 @@ class EmbeddedDocument(BaseDocument):
 
     __metaclass__ = DocumentMetaclass
 
+    def __init__(self, *args, **kwargs):
+        super(EmbeddedDocument, self).__init__(*args, **kwargs)
+        self._changed_fields = []
+
     def __delattr__(self, *args, **kwargs):
         """Handle deletions of fields"""
         field_name = args[0]
@@ -34,6 +40,10 @@ class EmbeddedDocument(BaseDocument):
         else:
             super(EmbeddedDocument, self).__delattr__(*args, **kwargs)
 
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return self._data == other._data
+        return False
 
 
 class Document(BaseDocument):
@@ -70,6 +80,12 @@ class Document(BaseDocument):
     names. Index direction may be specified by prefixing the field names with
     a **+** or **-** sign.
 
+    Automatic index creation can be disabled by specifying
+    attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to
+    False then indexes will not be created by MongoEngine.  This is useful in
+    production systems where index creation is performed as part of a deployment
+    system.
+
     By default, _types will be added to the start of every index (that
     doesn't contain a list) if allow_inheritence is True. This can be
     disabled by either setting types to False on the specific index or
@@ -77,42 +93,57 @@ class Document(BaseDocument):
     """
     __metaclass__ = TopLevelDocumentMetaclass
 
-    @classmethod
-    def _get_collection(self):
-        """Returns the collection for the document."""
-        db = _get_db()
-        collection_name = self._get_collection_name()
+    @apply
+    def pk():
+        """Primary key alias
+        """
+        def fget(self):
+            return getattr(self, self._meta['id_field'])
+        def fset(self, value):
+            return setattr(self, self._meta['id_field'], value)
+        return property(fget, fset)
 
-        if not hasattr(self, '_collection') or self._collection is None:
+    @classmethod
+    def _get_db(cls):
+        """Some Model using other db_alias"""
+        return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME ))
+
+    @classmethod
+    def _get_collection(cls):
+        """Returns the collection for the document."""
+        if not hasattr(cls, '_collection') or cls._collection is None:
+            db = cls._get_db()
+            collection_name = cls._get_collection_name()
             # Create collection as a capped collection if specified
-            if self._meta['max_size'] or self._meta['max_documents']:
+            if cls._meta['max_size'] or cls._meta['max_documents']:
                 # Get max document limit and max byte size from meta
-                max_size = self._meta['max_size'] or 10000000  # 10MB default
-                max_documents = self._meta['max_documents']
+                max_size = cls._meta['max_size'] or 10000000  # 10MB default
+                max_documents = cls._meta['max_documents']
 
                 if collection_name in db.collection_names():
-                    self._collection = db[collection_name]
+                    cls._collection = db[collection_name]
                     # The collection already exists, check if its capped
                     # options match the specified capped options
-                    options = self._collection.options()
+                    options = cls._collection.options()
                     if options.get('max') != max_documents or \
                        options.get('size') != max_size:
                         msg = ('Cannot create collection "%s" as a capped '
-                               'collection as it already exists') % self._collection
+                               'collection as it already exists') % cls._collection
                         raise InvalidCollectionError(msg)
                 else:
                     # Create the collection as a capped collection
                     opts = {'capped': True, 'size': max_size}
                     if max_documents:
                         opts['max'] = max_documents
-                    self._collection = db.create_collection(
+                    cls._collection = db.create_collection(
                         collection_name, **opts
                     )
             else:
-                self._collection = db[collection_name]
-        return self._collection
+                cls._collection = db[collection_name]
+        return cls._collection
 
-    def save(self, safe=True, force_insert=False, validate=True, write_options=None, _refs=None):
+    def save(self, safe=True, force_insert=False, validate=True, write_options=None,
+            cascade=None, cascade_kwargs=None, _refs=None):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -128,16 +159,25 @@ class Document(BaseDocument):
                 :meth:`~pymongo.collection.Collection.save` OR
                 :meth:`~pymongo.collection.Collection.insert`
                 which will be used as options for the resultant ``getLastError`` command.
-                For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers
-                have recorded the write and will force an fsync on each server being written to.
+                For example, ``save(..., write_options={w: 2, fsync: True}, ...)`` will
+                wait until at least two servers have recorded the write and will force an
+                fsync on each server being written to.
+        :param cascade: Sets the flag for cascading saves.  You can set a default by setting
+            "cascade" in the document __meta__
+        :param cascade_kwargs: optional kwargs dictionary to be passed throw to cascading saves
+        :param _refs: A list of processed references used in cascading saves
 
         .. versionchanged:: 0.5
             In existing documents it only saves changed fields using set / unset
-            Saves are cascaded and any :class:`~pymongo.dbref.DBRef` objects
+            Saves are cascaded and any :class:`~bson.dbref.DBRef` objects
             that have changes are saved as well.
-        """
-        from fields import ReferenceField, GenericReferenceField
+        .. versionchanged:: 0.6
+            Cascade saves are optional = defaults to True, if you want fine grain
+            control then you can turn off using document meta['cascade'] = False
+            Also you can pass different kwargs to the cascade save using cascade_kwargs
+            which overwrites the existing kwargs with custom values
 
+        """
         signals.pre_save.send(self.__class__, document=self)
 
         if validate:
@@ -148,11 +188,11 @@ class Document(BaseDocument):
 
         doc = self.to_mongo()
 
-        created = '_id' in doc
-        creation_mode = force_insert or not created
+        created = force_insert or '_id' not in doc
+
         try:
             collection = self.__class__.objects._collection
-            if creation_mode:
+            if created:
                 if force_insert:
                     object_id = collection.insert(doc, safe=safe, **write_options)
                 else:
@@ -160,21 +200,34 @@ class Document(BaseDocument):
             else:
                 object_id = doc['_id']
                 updates, removals = self._delta()
-                if updates:
-                    collection.update({'_id': object_id}, {"$set": updates}, upsert=True, safe=safe, **write_options)
-                if removals:
-                    collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options)
 
-            # Save any references / generic references
-            _refs = _refs or []
-            for name, cls in self._fields.items():
-                if isinstance(cls, (ReferenceField, GenericReferenceField)):
-                    ref = getattr(self, name)
-                    if ref and str(ref) not in _refs:
-                        _refs.append(str(ref))
-                        ref.save(safe=safe, force_insert=force_insert,
-                                 validate=validate, write_options=write_options,
-                                 _refs=_refs)
+                # Need to add shard key to query, or you get an error
+                select_dict = {'_id': object_id}
+                shard_key = self.__class__._meta.get('shard_key', tuple())
+                for k in shard_key:
+                    actual_key = self._db_field_map.get(k, k)
+                    select_dict[actual_key] = doc[actual_key]
+
+                upsert = self._created
+                if updates:
+                    collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options)
+                if removals:
+                    collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options)
+
+            cascade = self._meta.get('cascade', True) if cascade is None else cascade
+            if cascade:
+                kwargs = {
+                    "safe": safe,
+                    "force_insert": force_insert,
+                    "validate": validate,
+                    "write_options": write_options,
+                    "cascade": cascade
+                }
+                if cascade_kwargs:  # Allow granular control over cascades
+                    kwargs.update(cascade_kwargs)
+                kwargs['_refs'] = _refs
+                #self._changed_fields = []
+                self.cascade_save(**kwargs)
 
         except pymongo.errors.OperationFailure, err:
             message = 'Could not save document (%s)'
@@ -184,21 +237,28 @@ class Document(BaseDocument):
         id_field = self._meta['id_field']
         self[id_field] = self._fields[id_field].to_python(object_id)
 
-        def reset_changed_fields(doc, inspected_docs=None):
-            """Loop through and reset changed fields lists"""
+        self._changed_fields = []
+        self._created = False
+        signals.post_save.send(self.__class__, document=self, created=created)
+        return self
 
-            inspected_docs = inspected_docs or []
-            inspected_docs.append(doc)
-            if hasattr(doc, '_changed_fields'):
-                doc._changed_fields = []
+    def cascade_save(self, *args, **kwargs):
+        """Recursively saves any references / generic references on an object"""
+        from fields import ReferenceField, GenericReferenceField
+        _refs = kwargs.get('_refs', []) or []
 
-            for field_name in doc._fields:
-                field = getattr(doc, field_name)
-                if field not in inspected_docs and hasattr(field, '_changed_fields'):
-                    reset_changed_fields(field, inspected_docs)
-
-        reset_changed_fields(self)
-        signals.post_save.send(self.__class__, document=self, created=creation_mode)
+        for name, cls in self._fields.items():
+            if not isinstance(cls, (ReferenceField, GenericReferenceField)):
+                continue
+            ref = getattr(self, name)
+            if not ref:
+                continue
+            ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data))
+            if ref and ref_id not in _refs:
+                _refs.append(ref_id)
+                kwargs["_refs"] = _refs
+                ref.save(**kwargs)
+                ref._changed_fields = []
 
     def update(self, **kwargs):
         """Performs an update on the :class:`~mongoengine.Document`
@@ -210,7 +270,12 @@ class Document(BaseDocument):
         if not self.pk:
             raise OperationError('attempt to update a document not yet saved')
 
-        return self.__class__.objects(pk=self.pk).update_one(**kwargs)
+        # Need to add shard key to query, or you get an error
+        select_dict = {'pk': self.pk}
+        shard_key = self.__class__._meta.get('shard_key', tuple())
+        for k in shard_key:
+            select_dict[k] = getattr(self, k)
+        return self.__class__.objects(**select_dict).update_one(**kwargs)
 
     def delete(self, safe=False):
         """Delete the :class:`~mongoengine.Document` from the database. This
@@ -220,10 +285,8 @@ class Document(BaseDocument):
         """
         signals.pre_delete.send(self.__class__, document=self)
 
-        id_field = self._meta['id_field']
-        object_id = self._fields[id_field].to_mongo(self[id_field])
         try:
-            self.__class__.objects(**{id_field: object_id}).delete(safe=safe)
+            self.__class__.objects(pk=self.pk).delete(safe=safe)
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
@@ -231,25 +294,32 @@ class Document(BaseDocument):
         signals.post_delete.send(self.__class__, document=self)
 
     def select_related(self, max_depth=1):
-        """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to
+        """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to
         a maximum depth in order to cut down the number queries to mongodb.
 
         .. versionadded:: 0.5
         """
-        from dereference import dereference
-        self._data = dereference(self._data, max_depth)
+        from dereference import DeReference
+        self._data = DeReference()(self._data, max_depth)
         return self
 
-    def reload(self):
+    def reload(self, max_depth=1):
         """Reloads all attributes from the database.
 
         .. versionadded:: 0.1.2
+        .. versionchanged:: 0.6  Now chainable
         """
         id_field = self._meta['id_field']
-        obj = self.__class__.objects(**{id_field: self[id_field]}).first()
+        obj = self.__class__.objects(
+                **{id_field: self[id_field]}
+              ).first().select_related(max_depth=max_depth)
         for field in self._fields:
             setattr(self, field, self._reload(field, obj[field]))
-        self._changed_fields = []
+        if self._dynamic:
+            for name in self._dynamic_fields.keys():
+                setattr(self, name, self._reload(name, obj._data[name]))
+        self._changed_fields = obj._changed_fields
+        return obj
 
     def _reload(self, key, value):
         """Used by :meth:`~mongoengine.Document.reload` to ensure the
@@ -257,21 +327,21 @@ class Document(BaseDocument):
         """
         if isinstance(value, BaseDict):
             value = [(k, self._reload(k, v)) for k, v in value.items()]
-            value = BaseDict(value, instance=self, name=key)
+            value = BaseDict(value, self, key)
         elif isinstance(value, BaseList):
             value = [self._reload(key, v) for v in value]
-            value = BaseList(value, instance=self, name=key)
-        elif isinstance(value, EmbeddedDocument):
+            value = BaseList(value, self, key)
+        elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)):
             value._changed_fields = []
         return value
 
     def to_dbref(self):
-        """Returns an instance of :class:`~pymongo.dbref.DBRef` useful in
+        """Returns an instance of :class:`~bson.dbref.DBRef` useful in
         `__raw__` queries."""
         if not self.pk:
             msg = "Only saved documents can have a valid dbref"
             raise OperationError(msg)
-        return pymongo.dbref.DBRef(self.__class__._get_collection_name(), self.pk)
+        return DBRef(self.__class__._get_collection_name(), self.pk)
 
     @classmethod
     def register_delete_rule(cls, document_cls, field_name, rule):
@@ -285,8 +355,52 @@ class Document(BaseDocument):
         """Drops the entire collection associated with this
         :class:`~mongoengine.Document` type from the database.
         """
-        db = _get_db()
+        from mongoengine.queryset import QuerySet
+        db = cls._get_db()
         db.drop_collection(cls._get_collection_name())
+        QuerySet._reset_already_indexed(cls)
+
+
+class DynamicDocument(Document):
+    """A Dynamic Document class allowing flexible, expandable and uncontrolled
+    schemas.  As a :class:`~mongoengine.Document` subclass, acts in the same
+    way as an ordinary document but has expando style properties.  Any data
+    passed or set against the :class:`~mongoengine.DynamicDocument` that is
+    not a field is automatically converted into a
+    :class:`~mongoengine.DynamicField` and data can be attributed to that
+    field.
+
+    ..note::
+
+        There is one caveat on Dynamic Documents: fields cannot start with `_`
+    """
+    __metaclass__ = TopLevelDocumentMetaclass
+    _dynamic = True
+
+    def __delattr__(self, *args, **kwargs):
+        """Deletes the attribute by setting to None and allowing _delta to unset
+        it"""
+        field_name = args[0]
+        if field_name in self._dynamic_fields:
+            setattr(self, field_name, None)
+        else:
+            super(DynamicDocument, self).__delattr__(*args, **kwargs)
+
+
+class DynamicEmbeddedDocument(EmbeddedDocument):
+    """A Dynamic Embedded Document class allowing flexible, expandable and
+    uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more
+    information about dynamic documents.
+    """
+
+    __metaclass__ = DocumentMetaclass
+    _dynamic = True
+
+    def __delattr__(self, *args, **kwargs):
+        """Deletes the attribute by setting to None and allowing _delta to unset
+        it"""
+        field_name = args[0]
+        setattr(self, field_name, None)
 
 
 class MapReduceDocument(object):
@@ -294,7 +408,7 @@ class MapReduceDocument(object):
 
     :param collection: An instance of :class:`~pymongo.Collection`
     :param key: Document/result key, often an instance of
-                :class:`~pymongo.objectid.ObjectId`. If supplied as
+                :class:`~bson.objectid.ObjectId`. If supplied as
                 an ``ObjectId`` found in the given ``collection``,
                 the object can be accessed via the ``object`` property.
     :param value: The result(s) for this key.

mongoengine/fields.py

@@ -1,27 +1,39 @@
-from base import (BaseField, ComplexBaseField, ObjectIdField,
-                  ValidationError, get_document)
-from queryset import DO_NOTHING
-from document import Document, EmbeddedDocument
-from connection import _get_db
-from operator import itemgetter
-
-import re
-import pymongo
-import pymongo.dbref
-import pymongo.son
-import pymongo.binary
-import datetime, time
+import datetime
+import time
 import decimal
 import gridfs
+import re
+import uuid
+
+from bson import Binary, DBRef, SON, ObjectId
+
+from base import (BaseField, ComplexBaseField, ObjectIdField,
+                  ValidationError, get_document, BaseDocument)
+from queryset import DO_NOTHING, QuerySet
+from document import Document, EmbeddedDocument
+from connection import get_db, DEFAULT_CONNECTION_NAME
+from operator import itemgetter
+
+
+try:
+    from PIL import Image, ImageOps
+except ImportError:
+    Image = None
+    ImageOps = None
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 
 __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
            'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField',
            'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField',
-           'DecimalField', 'ComplexDateTimeField', 'URLField',
+           'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField',
            'GenericReferenceField', 'FileField', 'BinaryField',
-           'SortedListField', 'EmailField', 'GeoPointField',
-           'SequenceField', 'GenericEmbeddedDocumentField']
+           'SortedListField', 'EmailField', 'GeoPointField', 'ImageField',
+           'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField']
 
 RECURSIVE_REFERENCE_CONSTANT = 'self'
 
@@ -40,17 +52,17 @@ class StringField(BaseField):
         return unicode(value)
 
     def validate(self, value):
-        assert isinstance(value, (str, unicode))
+        if not isinstance(value, (str, unicode)):
+            self.error('StringField only accepts string values')
 
         if self.max_length is not None and len(value) > self.max_length:
-            raise ValidationError('String value is too long')
+            self.error('String value is too long')
 
         if self.min_length is not None and len(value) < self.min_length:
-            raise ValidationError('String value is too short')
+            self.error('String value is too short')
 
         if self.regex is not None and self.regex.match(value) is None:
-            message = 'String value did not match validation regex'
-            raise ValidationError(message)
+            self.error('String value did not match validation regex')
 
     def lookup_member(self, member_name):
         return None
@@ -100,16 +112,15 @@ class URLField(StringField):
 
     def validate(self, value):
         if not URLField.URL_REGEX.match(value):
-            raise ValidationError('Invalid URL: %s' % value)
+            self.error('Invalid URL: %s' % value)
 
         if self.verify_exists:
             import urllib2
             try:
                 request = urllib2.Request(value)
-                response = urllib2.urlopen(request)
+                urllib2.urlopen(request)
             except Exception, e:
-                message = 'This URL appears to be a broken link: %s' % e
-                raise ValidationError(message)
+                self.error('This URL appears to be a broken link: %s' % e)
 
 
 class EmailField(StringField):
@@ -126,7 +137,7 @@ class EmailField(StringField):
 
     def validate(self, value):
         if not EmailField.EMAIL_REGEX.match(value):
-            raise ValidationError('Invalid Mail-address: %s' % value)
+            self.error('Invalid Mail-address: %s' % value)
 
 
 class IntField(BaseField):
@@ -144,13 +155,13 @@ class IntField(BaseField):
         try:
             value = int(value)
         except:
-            raise ValidationError('%s could not be converted to int' % value)
+            self.error('%s could not be converted to int' % value)
 
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError('Integer value is too small')
+            self.error('Integer value is too small')
 
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError('Integer value is too large')
+            self.error('Integer value is too large')
 
     def prepare_query_value(self, op, value):
         return int(value)
@@ -170,13 +181,14 @@ class FloatField(BaseField):
     def validate(self, value):
         if isinstance(value, int):
             value = float(value)
-        assert isinstance(value, float)
+        if not isinstance(value, float):
+            self.error('FloatField only accepts float values')
 
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError('Float value is too small')
+            self.error('Float value is too small')
 
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError('Float value is too large')
+            self.error('Float value is too large')
 
     def prepare_query_value(self, op, value):
         return float(value)
@@ -207,13 +219,13 @@ class DecimalField(BaseField):
             try:
                 value = decimal.Decimal(value)
             except Exception, exc:
-                raise ValidationError('Could not convert to decimal: %s' % exc)
+                self.error('Could not convert value to decimal: %s' % exc)
 
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError('Decimal value is too small')
+            self.error('Decimal value is too small')
 
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError('Decimal value is too large')
+            self.error('Decimal value is too large')
 
 
 class BooleanField(BaseField):
@@ -226,7 +238,8 @@ class BooleanField(BaseField):
         return bool(value)
 
     def validate(self, value):
-        assert isinstance(value, bool)
+        if not isinstance(value, bool):
+            self.error('BooleanField only accepts boolean values')
 
 
 class DateTimeField(BaseField):
@@ -239,7 +252,8 @@ class DateTimeField(BaseField):
     """
 
     def validate(self, value):
-        assert isinstance(value, (datetime.datetime, datetime.date))
+        if not isinstance(value, (datetime.datetime, datetime.date)):
+            self.error(u'cannot parse date "%s"' % value)
 
     def to_mongo(self, value):
         return self.prepare_query_value(None, value)
@@ -355,13 +369,13 @@ class ComplexDateTimeField(StringField):
         return self._convert_from_string(data)
 
     def __set__(self, instance, value):
-        value = self._convert_from_datetime(value)
+        value = self._convert_from_datetime(value) if value else value
         return super(ComplexDateTimeField, self).__set__(instance, value)
 
     def validate(self, value):
         if not isinstance(value, datetime.datetime):
-            raise ValidationError('Only datetime objects may used in a \
-                                   ComplexDateTimeField')
+            self.error('Only datetime objects may used in a '
+                       'ComplexDateTimeField')
 
     def to_python(self, value):
         return self._convert_from_string(value)
@@ -381,8 +395,8 @@ class EmbeddedDocumentField(BaseField):
     def __init__(self, document_type, **kwargs):
         if not isinstance(document_type, basestring):
             if not issubclass(document_type, EmbeddedDocument):
-                raise ValidationError('Invalid embedded document class '
-                                      'provided to an EmbeddedDocumentField')
+                self.error('Invalid embedded document class provided to an '
+                           'EmbeddedDocumentField')
         self.document_type_obj = document_type
         super(EmbeddedDocumentField, self).__init__(**kwargs)
 
@@ -411,8 +425,8 @@ class EmbeddedDocumentField(BaseField):
         """
         # Using isinstance also works for subclasses of self.document
         if not isinstance(value, self.document_type):
-            raise ValidationError('Invalid embedded document instance '
-                                  'provided to an EmbeddedDocumentField')
+            self.error('Invalid embedded document instance provided to an '
+                       'EmbeddedDocumentField')
         self.document_type.validate(value)
 
     def lookup_member(self, member_name):
@@ -427,6 +441,9 @@ class GenericEmbeddedDocumentField(BaseField):
     :class:`~mongoengine.EmbeddedDocument` to be stored.
 
     Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`.
+
+    ..note :: You can use the choices param to limit the acceptable
+    EmbeddedDocument types
     """
 
     def prepare_query_value(self, op, value):
@@ -441,8 +458,8 @@ class GenericEmbeddedDocumentField(BaseField):
 
     def validate(self, value):
         if not isinstance(value, EmbeddedDocument):
-            raise ValidationError('Invalid embedded document instance '
-                                  'provided to an GenericEmbeddedDocumentField')
+            self.error('Invalid embedded document instance provided to an '
+                       'GenericEmbeddedDocumentField')
 
         value.validate()
 
@@ -456,9 +473,53 @@ class GenericEmbeddedDocumentField(BaseField):
         return data
 
 
+class DynamicField(BaseField):
+    """Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
+
+    def to_mongo(self, value):
+        """Convert a Python type to a MongoDBcompatible type.
+        """
+
+        if isinstance(value, basestring):
+            return value
+
+        if hasattr(value, 'to_mongo'):
+            return value.to_mongo()
+
+        if not isinstance(value, (dict, list, tuple)):
+            return value
+
+        is_list = False
+        if not hasattr(value, 'items'):
+            is_list = True
+            value = dict([(k, v) for k, v in enumerate(value)])
+
+        data = {}
+        for k, v in value.items():
+            data[k] = self.to_mongo(v)
+
+        if is_list:  # Convert back to a list
+            value = [v for k, v in sorted(data.items(), key=itemgetter(0))]
+        else:
+            value = data
+        return value
+
+    def lookup_member(self, member_name):
+        return member_name
+
+    def prepare_query_value(self, op, value):
+        if isinstance(value, basestring):
+            from mongoengine.fields import StringField
+            return StringField().prepare_query_value(op, value)
+        return self.to_mongo(value)
+
+
 class ListField(ComplexBaseField):
     """A list field that wraps a standard field, allowing multiple instances
     of the field to be used as a list in the database.
+
+    .. note::
+        Required means it cannot be empty - as the default for ListFields is []
     """
 
     # ListFields cannot be indexed with _types - MongoDB doesn't support this
@@ -472,14 +533,15 @@ class ListField(ComplexBaseField):
     def validate(self, value):
         """Make sure that a list of valid fields is being used.
         """
-        if not isinstance(value, (list, tuple)):
-            raise ValidationError('Only lists and tuples may be used in a '
-                                  'list field')
+        if (not isinstance(value, (list, tuple, QuerySet)) or
+            isinstance(value, basestring)):
+            self.error('Only lists and tuples may be used in a list field')
         super(ListField, self).validate(value)
 
     def prepare_query_value(self, op, value):
         if self.field:
             if op in ('set', 'unset') and (not isinstance(value, basestring)
+                and not isinstance(value, BaseDocument)
                 and hasattr(value, '__iter__')):
                 return [self.field.prepare_query_value(op, v) for v in value]
             return self.field.prepare_query_value(op, value)
@@ -491,27 +553,40 @@ class SortedListField(ListField):
     the database in order to ensure that a sorted list is always
     retrieved.
 
+    .. warning::
+        There is a potential race condition when handling lists.  If you set /
+        save the whole list then other processes trying to save the whole list
+        as well could overwrite changes.  The safest way to append to a list is
+        to perform a push operation.
+
     .. versionadded:: 0.4
+    .. versionchanged:: 0.6 - added reverse keyword
     """
 
     _ordering = None
+    _order_reverse = False
 
     def __init__(self, field, **kwargs):
         if 'ordering' in kwargs.keys():
             self._ordering = kwargs.pop('ordering')
+        if 'reverse' in kwargs.keys():
+            self._order_reverse = kwargs.pop('reverse')
         super(SortedListField, self).__init__(field, **kwargs)
 
     def to_mongo(self, value):
         value = super(SortedListField, self).to_mongo(value)
         if self._ordering is not None:
-            return sorted(value, key=itemgetter(self._ordering))
-        return sorted(value)
+            return sorted(value, key=itemgetter(self._ordering), reverse=self._order_reverse)
+        return sorted(value, reverse=self._order_reverse)
 
 
 class DictField(ComplexBaseField):
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
 
+    .. note::
+        Required means it cannot be empty - as the default for ListFields is []
+
     .. versionadded:: 0.3
     .. versionchanged:: 0.5 - Can now handle complex / varying types of data
     """
@@ -519,7 +594,8 @@ class DictField(ComplexBaseField):
     def __init__(self, basecls=None, field=None, *args, **kwargs):
         self.field = field
         self.basecls = basecls or BaseField
-        assert issubclass(self.basecls, BaseField)
+        if not issubclass(self.basecls, BaseField):
+            self.error('DictField only accepts dict values')
         kwargs.setdefault('default', lambda: {})
         super(DictField, self).__init__(*args, **kwargs)
 
@@ -527,12 +603,13 @@ class DictField(ComplexBaseField):
         """Make sure that a list of valid fields is being used.
         """
         if not isinstance(value, dict):
-            raise ValidationError('Only dictionaries may be used in a '
-                                  'DictField')
+            self.error('Only dictionaries may be used in a DictField')
 
-        if any(('.' in k or '$' in k) for k in value):
-            raise ValidationError('Invalid dictionary key name - keys may not '
-                                  'contain "." or "$" characters')
+        if any(k for k in value.keys() if not isinstance(k, basestring)):
+            self.error('Invalid dictionary key - documents must have only string keys')
+        if any(('.' in k or '$' in k) for k in value.keys()):
+            self.error('Invalid dictionary key name - keys may not contain "."'
+                       ' or "$" characters')
         super(DictField, self).validate(value)
 
     def lookup_member(self, member_name):
@@ -559,18 +636,19 @@ class MapField(DictField):
 
     def __init__(self, field=None, *args, **kwargs):
         if not isinstance(field, BaseField):
-            raise ValidationError('Argument to MapField constructor must be '
-                                  'a valid field')
+            self.error('Argument to MapField constructor must be a valid '
+                       'field')
         super(MapField, self).__init__(field=field, *args, **kwargs)
 
 
-
 class ReferenceField(BaseField):
     """A reference to a document that will be automatically dereferenced on
     access (lazily).
 
     Use the `reverse_delete_rule` to handle what should happen if the document
-    the field is referencing is deleted.
+    the field is referencing is deleted.  EmbeddedDocuments, DictFields and
+    MapFields do not support reverse_delete_rules and an `InvalidDocumentError`
+    will be raised if trying to set on one of these Document / Field types.
 
     The options are:
 
@@ -578,6 +656,18 @@ class ReferenceField(BaseField):
       * NULLIFY     - Updates the reference to null.
       * CASCADE     - Deletes the documents associated with the reference.
       * DENY        - Prevent the deletion of the reference object.
+      * PULL        - Pull the reference from a :class:`~mongoengine.ListField` of references
+
+    Alternative syntax for registering delete rules (useful when implementing
+    bi-directional delete rules)
+
+    .. code-block:: python
+
+        class Bar(Document):
+            content = StringField()
+            foo = ReferenceField('Foo')
+
+        Bar.register_delete_rule(Foo, 'bar', NULLIFY)
 
     .. versionchanged:: 0.5 added `reverse_delete_rule`
     """
@@ -590,8 +680,8 @@ class ReferenceField(BaseField):
         """
         if not isinstance(document_type, basestring):
             if not issubclass(document_type, (Document, basestring)):
-                raise ValidationError('Argument to ReferenceField constructor '
-                                      'must be a document class or a string')
+                self.error('Argument to ReferenceField constructor must be a '
+                           'document class or a string')
         self.document_type_obj = document_type
         self.reverse_delete_rule = reverse_delete_rule
         super(ReferenceField, self).__init__(**kwargs)
@@ -615,14 +705,17 @@ class ReferenceField(BaseField):
         # Get value from document instance if available
         value = instance._data.get(self.name)
         # Dereference DBRefs
-        if isinstance(value, (pymongo.dbref.DBRef)):
-            value = _get_db().dereference(value)
+        if isinstance(value, (DBRef)):
+            value = self.document_type._get_db().dereference(value)
             if value is not None:
                 instance._data[self.name] = self.document_type._from_son(value)
 
         return super(ReferenceField, self).__get__(instance, owner)
 
     def to_mongo(self, document):
+        if isinstance(document, DBRef):
+            return document
+
         id_field_name = self.document_type._meta['id_field']
         id_field = self.document_type._fields[id_field_name]
 
@@ -630,25 +723,28 @@ class ReferenceField(BaseField):
             # We need the id from the saved object to create the DBRef
             id_ = document.id
             if id_ is None:
-                raise ValidationError('You can only reference documents once '
-                                      'they have been saved to the database')
+                self.error('You can only reference documents once they have'
+                           ' been saved to the database')
         else:
             id_ = document
 
         id_ = id_field.to_mongo(id_)
         collection = self.document_type._get_collection_name()
-        return pymongo.dbref.DBRef(collection, id_)
+        return DBRef(collection, id_)
 
     def prepare_query_value(self, op, value):
+        if value is None:
+            return None
+
         return self.to_mongo(value)
 
     def validate(self, value):
-        assert isinstance(value, (self.document_type, pymongo.dbref.DBRef))
+        if not isinstance(value, (self.document_type, DBRef)):
+            self.error('A ReferenceField only accepts DBRef')
 
         if isinstance(value, Document) and value.id is None:
-            raise ValidationError('You can only reference documents once '
-                                  'they have been saved to the database')
-
+            self.error('You can only reference documents once they have been '
+                       'saved to the database')
 
     def lookup_member(self, member_name):
         return self.document_type._fields.get(member_name)
@@ -661,6 +757,8 @@ class GenericReferenceField(BaseField):
     ..note ::  Any documents used as a generic reference must be registered in the
     document registry.  Importing the model will automatically register it.
 
+    ..note :: You can use the choices param to limit the acceptable Document types
+
     .. versionadded:: 0.3
     """
 
@@ -669,24 +767,24 @@ class GenericReferenceField(BaseField):
             return self
 
         value = instance._data.get(self.name)
-        if isinstance(value, (dict, pymongo.son.SON)):
+        if isinstance(value, (dict, SON)):
             instance._data[self.name] = self.dereference(value)
 
         return super(GenericReferenceField, self).__get__(instance, owner)
 
     def validate(self, value):
-        if not isinstance(value, (Document, pymongo.dbref.DBRef)):
-            raise ValidationError('GenericReferences can only contain documents')
+        if not isinstance(value, (Document, DBRef)):
+            self.error('GenericReferences can only contain documents')
 
         # We need the id from the saved object to create the DBRef
         if isinstance(value, Document) and value.id is None:
-            raise ValidationError('You can only reference documents once '
-                                  'they have been saved to the database')
+            self.error('You can only reference documents once they have been'
+                       ' saved to the database')
 
     def dereference(self, value):
         doc_cls = get_document(value['_cls'])
         reference = value['_ref']
-        doc = _get_db().dereference(reference)
+        doc = doc_cls._get_db().dereference(reference)
         if doc is not None:
             doc = doc_cls._from_son(doc)
         return doc
@@ -695,6 +793,9 @@ class GenericReferenceField(BaseField):
         if document is None:
             return None
 
+        if isinstance(document, (dict, SON)):
+            return document
+
         id_field_name = document.__class__._meta['id_field']
         id_field = document.__class__._fields[id_field_name]
 
@@ -702,17 +803,20 @@ class GenericReferenceField(BaseField):
             # We need the id from the saved object to create the DBRef
             id_ = document.id
             if id_ is None:
-                raise ValidationError('You can only reference documents once '
-                                      'they have been saved to the database')
+                self.error('You can only reference documents once they have'
+                           ' been saved to the database')
         else:
             id_ = document
 
         id_ = id_field.to_mongo(id_)
         collection = document._get_collection_name()
-        ref = pymongo.dbref.DBRef(collection, id_)
+        ref = DBRef(collection, id_)
         return {'_cls': document._class_name, '_ref': ref}
 
     def prepare_query_value(self, op, value):
+        if value is None:
+            return None
+
         return self.to_mongo(value)
 
 
@@ -725,17 +829,18 @@ class BinaryField(BaseField):
         super(BinaryField, self).__init__(**kwargs)
 
     def to_mongo(self, value):
-        return pymongo.binary.Binary(value)
+        return Binary(value)
 
     def to_python(self, value):
         # Returns str not unicode as this is binary data
         return str(value)
 
     def validate(self, value):
-        assert isinstance(value, str)
+        if not isinstance(value, str):
+            self.error('BinaryField only accepts string values')
 
         if self.max_bytes is not None and len(value) > self.max_bytes:
-            raise ValidationError('Binary value is too long')
+            self.error('Binary value is too long')
 
 
 class GridFSError(Exception):
@@ -747,17 +852,28 @@ class GridFSProxy(object):
 
     .. versionadded:: 0.4
     .. versionchanged:: 0.5 - added optional size param to read
+    .. versionchanged:: 0.6 - added collection name param
     """
 
-    def __init__(self, grid_id=None, key=None, instance=None):
-        self.fs = gridfs.GridFS(_get_db())  # Filesystem instance
-        self.newfile = None                 # Used for partial writes
+    _fs = None
+
+    def __init__(self, grid_id=None, key=None,
+                 instance=None,
+                 db_alias=DEFAULT_CONNECTION_NAME,
+                 collection_name='fs'):
         self.grid_id = grid_id                  # Store GridFS id for file
-        self.gridout = None
         self.key = key
         self.instance = instance
+        self.db_alias = db_alias
+        self.collection_name = collection_name
+        self.newfile = None                     # Used for partial writes
+        self.gridout = None
 
     def __getattr__(self, name):
+        attrs = ('_fs', 'grid_id', 'key', 'instance', 'db_alias',
+                 'collection_name', 'newfile', 'gridout')
+        if name in attrs:
+            return self.__getattribute__(name)
         obj = self.get()
         if name in dir(obj):
             return getattr(obj, name)
@@ -769,6 +885,24 @@ class GridFSProxy(object):
     def __nonzero__(self):
         return bool(self.grid_id)
 
+    def __getstate__(self):
+        self_dict = self.__dict__
+        self_dict['_fs'] = None
+        return self_dict
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.grid_id)
+
+    def __cmp__(self, other):
+        return cmp((self.grid_id, self.collection_name, self.db_alias),
+                   (other.grid_id, other.collection_name, other.db_alias))
+
+    @property
+    def fs(self):
+        if not self._fs:
+            self._fs = gridfs.GridFS(get_db(self.db_alias), self.collection_name)
+        return self._fs
+
     def get(self, id=None):
         if id:
             self.grid_id = id
@@ -809,10 +943,14 @@ class GridFSProxy(object):
         self.newfile.writelines(lines)
 
     def read(self, size=-1):
-        try:
-            return self.get().read(size)
-        except:
+        gridout = self.get()
+        if gridout is None:
             return None
+        else:
+            try:
+                return gridout.read(size)
+            except:
+                return ""
 
     def delete(self):
         # Delete file from GridFS, FileField still remains
@@ -840,10 +978,16 @@ class FileField(BaseField):
 
     .. versionadded:: 0.4
     .. versionchanged:: 0.5 added optional size param for read
+    .. versionchanged:: 0.6 added db_alias for multidb support
     """
+    proxy_class = GridFSProxy
 
-    def __init__(self, **kwargs):
+    def __init__(self,
+                 db_alias=DEFAULT_CONNECTION_NAME,
+                 collection_name="fs", **kwargs):
         super(FileField, self).__init__(**kwargs)
+        self.collection_name = collection_name
+        self.db_alias = db_alias
 
     def __get__(self, instance, owner):
         if instance is None:
@@ -851,17 +995,20 @@ class FileField(BaseField):
 
         # Check if a file already exists for this model
         grid_file = instance._data.get(self.name)
-        self.grid_file = grid_file
-        if isinstance(self.grid_file, GridFSProxy):
-            if not self.grid_file.key:
-                self.grid_file.key = self.name
-                self.grid_file.instance = instance
-            return self.grid_file
-        return GridFSProxy(key=self.name, instance=instance)
+        if not isinstance(grid_file, self.proxy_class):
+            grid_file = self.proxy_class(key=self.name, instance=instance,
+                                         db_alias=self.db_alias,
+                                         collection_name=self.collection_name)
+            instance._data[self.name] = grid_file
+
+        if not grid_file.key:
+            grid_file.key = self.name
+            grid_file.instance = instance
+        return grid_file
 
     def __set__(self, instance, value):
         key = self.name
-        if isinstance(value, file) or isinstance(value, str):
+        if (hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, str):
             # using "FileField() = file/string" notation
             grid_file = instance._data.get(self.name)
             # If a file already exists, delete it
@@ -874,7 +1021,8 @@ class FileField(BaseField):
                 grid_file.put(value)
             else:
                 # Create a new proxy object as we don't already have one
-                instance._data[key] = GridFSProxy(key=key, instance=instance)
+                instance._data[key] = self.proxy_class(key=key, instance=instance,
+                                                       collection_name=self.collection_name)
                 instance._data[key].put(value)
         else:
             instance._data[key] = value
@@ -883,18 +1031,181 @@ class FileField(BaseField):
 
     def to_mongo(self, value):
         # Store the GridFS file id in MongoDB
-        if isinstance(value, GridFSProxy) and value.grid_id is not None:
+        if isinstance(value, self.proxy_class) and value.grid_id is not None:
             return value.grid_id
         return None
 
     def to_python(self, value):
         if value is not None:
-            return GridFSProxy(value)
+            return self.proxy_class(value,
+                                    collection_name=self.collection_name,
+                                    db_alias=self.db_alias)
 
     def validate(self, value):
         if value.grid_id is not None:
-            assert isinstance(value, GridFSProxy)
-            assert isinstance(value.grid_id, pymongo.objectid.ObjectId)
+            if not isinstance(value, self.proxy_class):
+                self.error('FileField only accepts GridFSProxy values')
+            if not isinstance(value.grid_id, ObjectId):
+                self.error('Invalid GridFSProxy value')
+
+
+class ImageGridFsProxy(GridFSProxy):
+    """
+    Proxy for ImageField
+
+    versionadded: 0.6
+    """
+    def put(self, file_obj, **kwargs):
+        """
+        Insert a image in database
+        applying field properties (size, thumbnail_size)
+        """
+        field = self.instance._fields[self.key]
+
+        try:
+            img = Image.open(file_obj)
+        except:
+            raise ValidationError('Invalid image')
+
+        if (field.size and (img.size[0] > field.size['width'] or
+                            img.size[1] > field.size['height'])):
+            size = field.size
+
+            if size['force']:
+                img = ImageOps.fit(img,
+                                   (size['width'],
+                                    size['height']),
+                                   Image.ANTIALIAS)
+            else:
+                img.thumbnail((size['width'],
+                               size['height']),
+                              Image.ANTIALIAS)
+
+        thumbnail = None
+        if field.thumbnail_size:
+            size = field.thumbnail_size
+
+            if size['force']:
+                thumbnail = ImageOps.fit(img,
+                                   (size['width'],
+                                    size['height']),
+                                   Image.ANTIALIAS)
+            else:
+                thumbnail = img.copy()
+                thumbnail.thumbnail((size['width'],
+                                     size['height']),
+                                    Image.ANTIALIAS)
+
+        if thumbnail:
+            thumb_id = self._put_thumbnail(thumbnail,
+                                          img.format)
+        else:
+            thumb_id = None
+
+        w, h = img.size
+
+        io = StringIO()
+        img.save(io, img.format)
+        io.seek(0)
+
+        return super(ImageGridFsProxy, self).put(io,
+                                                 width=w,
+                                                 height=h,
+                                                 format=img.format,
+                                                 thumbnail_id=thumb_id,
+                                                 **kwargs)
+
+    def delete(self, *args, **kwargs):
+        #deletes thumbnail
+        out = self.get()
+        if out and out.thumbnail_id:
+            self.fs.delete(out.thumbnail_id)
+
+        return super(ImageGridFsProxy, self).delete(*args, **kwargs)
+
+    def _put_thumbnail(self, thumbnail, format, **kwargs):
+        w, h = thumbnail.size
+
+        io = StringIO()
+        thumbnail.save(io, format)
+        io.seek(0)
+
+        return self.fs.put(io, width=w,
+                           height=h,
+                           format=format,
+                           **kwargs)
+    @property
+    def size(self):
+        """
+        return a width, height of image
+        """
+        out = self.get()
+        if out:
+            return out.width, out.height
+
+    @property
+    def format(self):
+        """
+        return format of image
+        ex: PNG, JPEG, GIF, etc
+        """
+        out = self.get()
+        if out:
+            return out.format
+
+    @property
+    def thumbnail(self):
+        """
+        return a gridfs.grid_file.GridOut
+        representing a thumbnail of Image
+        """
+        out = self.get()
+        if out and out.thumbnail_id:
+            return self.fs.get(out.thumbnail_id)
+
+    def write(self, *args, **kwargs):
+        raise RuntimeError("Please use \"put\" method instead")
+
+    def writelines(self, *args, **kwargs):
+        raise RuntimeError("Please use \"put\" method instead")
+
+
+class ImproperlyConfigured(Exception):
+    pass
+
+
+class ImageField(FileField):
+    """
+    A Image File storage field.
+
+    @size (width, height, force):
+        max size to store images, if larger will be automatically resized
+        ex: size=(800, 600, True)
+
+    @thumbnail (width, height, force):
+        size to generate a thumbnail
+
+    .. versionadded:: 0.6
+    """
+    proxy_class = ImageGridFsProxy
+
+    def __init__(self, size=None, thumbnail_size=None,
+                 collection_name='images', **kwargs):
+        if not Image:
+            raise ImproperlyConfigured("PIL library was not found")
+
+        params_size = ('width', 'height', 'force')
+        extra_args = dict(size=size, thumbnail_size=thumbnail_size)
+        for att_name, att in extra_args.items():
+            if att and (isinstance(att, tuple) or isinstance(att, list)):
+                setattr(self, att_name, dict(
+                        map(None, params_size, att)))
+            else:
+                setattr(self, att_name, None)
+
+        super(ImageField, self).__init__(
+            collection_name=collection_name,
+            **kwargs)
 
 
 class GeoPointField(BaseField):
@@ -909,14 +1220,14 @@ class GeoPointField(BaseField):
         """Make sure that a geo-value is of type (x, y)
         """
         if not isinstance(value, (list, tuple)):
-            raise ValidationError('GeoPointField can only accept tuples or '
-                                  'lists of (x, y)')
+            self.error('GeoPointField can only accept tuples or lists '
+                       'of (x, y)')
 
         if not len(value) == 2:
-            raise ValidationError('Value must be a two-dimensional point.')
+            self.error('Value must be a two-dimensional point')
         if (not isinstance(value[0], (float, int)) and
             not isinstance(value[1], (float, int))):
-            raise ValidationError('Both values in point must be float or int.')
+            self.error('Both values in point must be float or int')
 
 
 class SequenceField(IntField):
@@ -932,8 +1243,9 @@ class SequenceField(IntField):
 
     .. versionadded:: 0.5
     """
-    def __init__(self, collection_name=None, *args, **kwargs):
+    def __init__(self, collection_name=None, db_alias = None, *args, **kwargs):
         self.collection_name = collection_name or 'mongoengine.counters'
+        self.db_alias = db_alias or DEFAULT_CONNECTION_NAME
         return super(SequenceField, self).__init__(*args, **kwargs)
 
     def generate_new_value(self):
@@ -942,7 +1254,7 @@ class SequenceField(IntField):
         """
         sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(),
                                        self.name)
-        collection = _get_db()[self.collection_name]
+        collection = get_db(alias = self.db_alias )[self.collection_name]
         counter = collection.find_and_modify(query={"_id": sequence_id},
                                              update={"$inc": {"next": 1}},
                                              new=True,
@@ -977,3 +1289,30 @@ class SequenceField(IntField):
         if value is None:
             value = self.generate_new_value()
         return value
+
+
+class UUIDField(BaseField):
+    """A UUID field.
+
+    .. versionadded:: 0.6
+    """
+
+    def __init__(self, **kwargs):
+        super(UUIDField, self).__init__(**kwargs)
+
+    def to_python(self, value):
+        if not isinstance(value, basestring):
+            value = unicode(value)
+        return uuid.UUID(value)
+
+    def to_mongo(self, value):
+        return unicode(value)
+
+    def validate(self, value):
+        if not isinstance(value, uuid.UUID):
+            if not isinstance(value, basestring):
+                value = str(value)
+            try:
+                value = uuid.UUID(value)
+            except Exception, exc:
+                self.error('Could not convert to UUID: %s' % exc)

mongoengine/queryset.py

@@ -1,17 +1,16 @@
-from connection import _get_db
-
 import pprint
-import pymongo
-import pymongo.code
-import pymongo.dbref
-import pymongo.objectid
 import re
 import copy
 import itertools
 import operator
 
+import pymongo
+from bson.code import Code
+
+from mongoengine import signals
+
 __all__ = ['queryset_manager', 'Q', 'InvalidQueryError',
-           'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY']
+           'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL']
 
 
 # The maximum number of items to display in a QuerySet.__repr__
@@ -22,6 +21,7 @@ DO_NOTHING = 0
 NULLIFY = 1
 CASCADE = 2
 DENY = 3
+PULL = 4
 
 
 class DoesNotExist(Exception):
@@ -274,16 +274,20 @@ class Q(QNode):
 
 class QueryFieldList(object):
     """Object that handles combinations of .only() and .exclude() calls"""
-    ONLY = True
-    EXCLUDE = False
+    ONLY = 1
+    EXCLUDE = 0
 
     def __init__(self, fields=[], value=ONLY, always_include=[]):
         self.value = value
         self.fields = set(fields)
         self.always_include = set(always_include)
+        self._id = None
 
     def as_dict(self):
-        return dict((field, self.value) for field in self.fields)
+        field_list = dict((field, self.value) for field in self.fields)
+        if self._id is not None:
+            field_list['_id'] = self._id
+        return field_list
 
     def __add__(self, f):
         if not self.fields:
@@ -299,6 +303,9 @@ class QueryFieldList(object):
             self.value = self.ONLY
             self.fields = f.fields - self.fields
 
+        if '_id' in f.fields:
+            self._id = f.value
+
         if self.always_include:
             if self.value is self.ONLY and self.fields:
                 self.fields = self.fields.union(self.always_include)
@@ -334,6 +341,7 @@ class QuerySet(object):
         self._timeout = True
         self._class_check = True
         self._slave_okay = False
+        self._scalar = []
 
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -387,57 +395,6 @@ class QuerySet(object):
             unique=index_spec.get('unique', False))
         return self
 
-    @classmethod
-    def _build_index_spec(cls, doc_cls, spec):
-        """Build a PyMongo index spec from a MongoEngine index spec.
-        """
-        if isinstance(spec, basestring):
-            spec = {'fields': [spec]}
-        if isinstance(spec, (list, tuple)):
-            spec = {'fields': spec}
-
-        index_list = []
-        use_types = doc_cls._meta.get('allow_inheritance', True)
-        for key in spec['fields']:
-            # Get direction from + or -
-            direction = pymongo.ASCENDING
-            if key.startswith("-"):
-                direction = pymongo.DESCENDING
-            if key.startswith(("+", "-")):
-                    key = key[1:]
-
-            # Use real field name, do it manually because we need field
-            # objects for the next part (list field checking)
-            parts = key.split('.')
-            fields = QuerySet._lookup_field(doc_cls, parts)
-            parts = [field.db_field for field in fields]
-            key = '.'.join(parts)
-            index_list.append((key, direction))
-
-            # Check if a list field is being used, don't use _types if it is
-            if use_types and not all(f._index_with_types for f in fields):
-                use_types = False
-
-        # If _types is being used, prepend it to every specified index
-        index_types = doc_cls._meta.get('index_types', True)
-        allow_inheritance = doc_cls._meta.get('allow_inheritance')
-        if spec.get('types', index_types) and allow_inheritance and use_types:
-            index_list.insert(0, ('_types', 1))
-
-        spec['fields'] = index_list
-
-        if spec.get('sparse', False) and len(spec['fields']) > 1:
-            raise ValueError(
-                'Sparse indexes can only have one field in them. '
-                'See https://jira.mongodb.org/browse/SERVER-2193')
-
-        return spec
-
-    @classmethod
-    def _reset_already_indexed(cls):
-        """Helper to reset already indexed, can be useful for testing purposes"""
-        cls.__already_indexed = set()
-
     def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query):
         """Filter the selected documents by calling the
         :class:`~mongoengine.queryset.QuerySet` with a query.
@@ -470,17 +427,15 @@ class QuerySet(object):
         """Returns all documents."""
         return self.__call__()
 
-    @property
-    def _collection(self):
-        """Property that returns the collection object. This allows us to
-        perform operations only if the collection is accessed.
-        """
-        if self._document not in QuerySet.__already_indexed:
-            QuerySet.__already_indexed.add(self._document)
+    def _ensure_indexes(self):
+        """Checks the document meta data and ensures all the indexes exist.
 
+        .. note:: You can disable automatic index creation by setting
+                  `auto_create_index` to False in the documents meta data
+        """
         background = self._document._meta.get('index_background', False)
         drop_dups = self._document._meta.get('index_drop_dups', False)
-            index_opts = self._document._meta.get('index_options', {})
+        index_opts = self._document._meta.get('index_opts', {})
         index_types = self._document._meta.get('index_types', True)
 
         # determine if an index which we are creating includes
@@ -525,6 +480,86 @@ class QuerySet(object):
             self._collection.ensure_index(index_spec,
                 background=background, **index_opts)
 
+
+    @classmethod
+    def _build_index_spec(cls, doc_cls, spec):
+        """Build a PyMongo index spec from a MongoEngine index spec.
+        """
+        if isinstance(spec, basestring):
+            spec = {'fields': [spec]}
+        if isinstance(spec, (list, tuple)):
+            spec = {'fields': spec}
+
+        index_list = []
+        use_types = doc_cls._meta.get('allow_inheritance', True)
+        for key in spec['fields']:
+            # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from *
+            direction = pymongo.ASCENDING
+            if key.startswith("-"):
+                direction = pymongo.DESCENDING
+            elif key.startswith("*"):
+                direction = pymongo.GEO2D
+            if key.startswith(("+", "-", "*")):
+                key = key[1:]
+
+            # Use real field name, do it manually because we need field
+            # objects for the next part (list field checking)
+            parts = key.split('.')
+            if parts in (['pk'], ['id'], ['_id']):
+                key = '_id'
+            else:
+                fields = QuerySet._lookup_field(doc_cls, parts)
+                parts = [field if field == '_id' else field.db_field for field in fields]
+                key = '.'.join(parts)
+            index_list.append((key, direction))
+
+            # If sparse - dont include types
+            if spec.get('sparse', False):
+                use_types = False
+
+            # Check if a list field is being used, don't use _types if it is
+            if use_types and not all(f._index_with_types for f in fields):
+                use_types = False
+
+        # If _types is being used, prepend it to every specified index
+        index_types = doc_cls._meta.get('index_types', True)
+        allow_inheritance = doc_cls._meta.get('allow_inheritance')
+        if spec.get('types', index_types) and allow_inheritance and use_types and direction is not pymongo.GEO2D:
+            index_list.insert(0, ('_types', 1))
+
+        spec['fields'] = index_list
+        if spec.get('sparse', False) and len(spec['fields']) > 1:
+            raise ValueError(
+                'Sparse indexes can only have one field in them. '
+                'See https://jira.mongodb.org/browse/SERVER-2193')
+
+        return spec
+
+    @classmethod
+    def _reset_already_indexed(cls, document=None):
+        """Helper to reset already indexed, can be useful for testing purposes"""
+        if document:
+            cls.__already_indexed.discard(document)
+        cls.__already_indexed.clear()
+
+
+    @property
+    def _collection(self):
+        """Property that returns the collection object. This allows us to
+        perform operations only if the collection is accessed.
+        """
+        if self._document not in QuerySet.__already_indexed:
+            # Ensure collection exists
+            db = self._document._get_db()
+            if self._collection_obj.name not in db.collection_names():
+                self._document._collection = None
+                self._collection_obj = self._document._get_collection()
+
+            QuerySet.__already_indexed.add(self._document)
+
+            if self._document._meta.get('auto_create_index', True):
+                self._ensure_indexes()
+
         return self._collection_obj
 
     @property
@@ -555,7 +590,7 @@ class QuerySet(object):
                 self.order_by(*self._document._meta['ordering'])
 
             if self._limit is not None:
-                self._cursor_obj.limit(self._limit)
+                self._cursor_obj.limit(self._limit - (self._skip or 0))
 
             if self._skip is not None:
                 self._cursor_obj.skip(self._skip)
@@ -585,12 +620,26 @@ class QuerySet(object):
                         "Can't use index on unsubscriptable field (%s)" % err)
                 fields.append(field_name)
                 continue
+
             if field is None:
                 # Look up first field from the document
                 if field_name == 'pk':
                     # Deal with "primary key" alias
                     field_name = document._meta['id_field']
+                if field_name in document._fields:
                     field = document._fields[field_name]
+                elif document._dynamic:
+                    from fields import DynamicField
+                    field = DynamicField(db_field=field_name)
+                else:
+                    raise InvalidQueryError('Cannot resolve field "%s"'
+                                                % field_name)
+            else:
+                from mongoengine.fields import ReferenceField, GenericReferenceField
+                if isinstance(field, (ReferenceField, GenericReferenceField)):
+                    raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts))
+                if getattr(field, 'field', None):
+                    new_field = field.field.lookup_member(field_name)
                 else:
                    # Look up subfield on the previous field
                     new_field = field.lookup_member(field_name)
@@ -603,7 +652,6 @@ class QuerySet(object):
                                                 % field_name)
                 field = new_field  # update field to the new field type
             fields.append(field)
-
         return fields
 
     @classmethod
@@ -624,6 +672,7 @@ class QuerySet(object):
         match_operators = ['contains', 'icontains', 'startswith',
                            'istartswith', 'endswith', 'iendswith',
                            'exact', 'iexact']
+        custom_operators = ['match']
 
         mongo_query = {}
         for key, value in query.items():
@@ -636,7 +685,7 @@ class QuerySet(object):
             parts = [part for part in parts if not part.isdigit()]
             # Check for an operator and transform to mongo-style if there is
             op = None
-            if parts[-1] in operators + match_operators + geo_operators:
+            if parts[-1] in operators + match_operators + geo_operators + custom_operators:
                 op = parts.pop()
 
             negate = False
@@ -650,8 +699,8 @@ class QuerySet(object):
                 parts = []
 
                 cleaned_fields = []
-                append_field = True
                 for field in fields:
+                    append_field = True
                     if isinstance(field, str):
                         parts.append(field)
                         append_field = False
@@ -669,7 +718,7 @@ class QuerySet(object):
                     if isinstance(field, basestring):
                         if op in match_operators and isinstance(value, basestring):
                             from mongoengine import StringField
-                            value = StringField().prepare_query_value(op, value)
+                            value = StringField.prepare_query_value(op, value)
                         else:
                             value = field
                     else:
@@ -696,6 +745,12 @@ class QuerySet(object):
                     else:
                         raise NotImplementedError("Geo method '%s' has not "
                                                   "been implemented" % op)
+                elif op in custom_operators:
+                    if op == 'match':
+                        value = {"$elemMatch": value}
+                    else:
+                        NotImplementedError("Custom method '%s' has not "
+                                            "been implemented" % op)
                 elif op not in match_operators:
                     value = {'$' + op: value}
 
@@ -721,18 +776,23 @@ class QuerySet(object):
 
         .. versionadded:: 0.3
         """
+        self.limit(2)
         self.__call__(*q_objs, **query)
-        count = self.count()
-        if count == 1:
-            return self[0]
-        elif count > 1:
-            message = u'%d items returned, instead of 1' % count
-            raise self._document.MultipleObjectsReturned(message)
-        else:
+        try:
+            result1 = self.next()
+        except StopIteration:
             raise self._document.DoesNotExist("%s matching query does not exist."
                                               % self._document._class_name)
+        try:
+            result2 = self.next()
+        except StopIteration:
+            return result1
 
-    def get_or_create(self, write_options=None, *q_objs, **query):
+        self.rewind()
+        message = u'%d items returned, instead of 1' % self.count()
+        raise self._document.MultipleObjectsReturned(message)
+
+    def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query):
         """Retrieve unique object or create, if it doesn't exist. Returns a tuple of
         ``(object, created)``, where ``object`` is the retrieved or created object
         and ``created`` is a boolean specifying whether a new object was created. Raises
@@ -742,28 +802,35 @@ class QuerySet(object):
         dictionary of default values for the new document may be provided as a
         keyword argument called :attr:`defaults`.
 
+        .. note:: This requires two separate operations and therefore a
+        race condition exists.  Because there are no transactions in mongoDB
+        other approaches should be investigated, to ensure you don't
+        accidently duplicate data when using this method.
+
         :param write_options: optional extra keyword arguments used if we
             have to create a new document.
             Passes any write_options onto :meth:`~mongoengine.Document.save`
 
         .. versionadded:: 0.3
+
+        :param auto_save: if the object is to be saved automatically if not found.
+
+        .. versionadded:: 0.6
         """
         defaults = query.get('defaults', {})
         if 'defaults' in query:
             del query['defaults']
 
-        self.__call__(*q_objs, **query)
-        count = self.count()
-        if count == 0:
+        try:
+            doc = self.get(*q_objs, **query)
+            return doc, False
+        except self._document.DoesNotExist:
             query.update(defaults)
             doc = self._document(**query)
+
+            if auto_save:
                 doc.save(write_options=write_options)
             return doc, True
-        elif count == 1:
-            return self.first(), False
-        else:
-            message = u'%d items returned, instead of 1' % count
-            raise self._document.MultipleObjectsReturned(message)
 
     def create(self, **kwargs):
         """Create new object. Returns the saved object instance.
@@ -783,11 +850,21 @@ class QuerySet(object):
             result = None
         return result
 
-    def insert(self, doc_or_docs, load_bulk=True):
+    def insert(self, doc_or_docs, load_bulk=True, safe=False, write_options=None):
         """bulk insert documents
 
+        If ``safe=True`` and the operation is unsuccessful, an
+        :class:`~mongoengine.OperationError` will be raised.
+
         :param docs_or_doc: a document or list of documents to be inserted
         :param load_bulk (optional): If True returns the list of document instances
+        :param safe: check if the operation succeeded before returning
+        :param write_options: Extra keyword arguments are passed down to
+                :meth:`~pymongo.collection.Collection.insert`
+                which will be used as options for the resultant ``getLastError`` command.
+                For example, ``insert(..., {w: 2, fsync: True})`` will wait until at least two
+                servers have recorded the write and will force an fsync on each server being
+                written to.
 
         By default returns document instances, set ``load_bulk`` to False to
         return just ``ObjectIds``
@@ -796,6 +873,10 @@ class QuerySet(object):
         """
         from document import Document
 
+        if not write_options:
+            write_options = {}
+        write_options.update({'safe': safe})
+
         docs = doc_or_docs
         return_one = False
         if isinstance(docs, Document) or issubclass(docs.__class__, Document):
@@ -812,23 +893,39 @@ class QuerySet(object):
                 raise OperationError(msg)
             raw.append(doc.to_mongo())
 
-        ids = self._collection.insert(raw)
+        signals.pre_bulk_insert.send(self._document, documents=docs)
+        try:
+            ids = self._collection.insert(raw, **write_options)
+        except pymongo.errors.OperationFailure, err:
+            message = 'Could not save document (%s)'
+            if u'duplicate key' in unicode(err):
+                message = u'Tried to save duplicate unique keys (%s)'
+            raise OperationError(message % unicode(err))
 
         if not load_bulk:
+            signals.post_bulk_insert.send(
+                    self._document, documents=docs, loaded=False)
             return return_one and ids[0] or ids
 
         documents = self.in_bulk(ids)
         results = []
         for obj_id in ids:
             results.append(documents.get(obj_id))
+        signals.post_bulk_insert.send(
+                self._document, documents=results, loaded=True)
         return return_one and results[0] or results
 
     def with_id(self, object_id):
-        """Retrieve the object matching the id provided.
+        """Retrieve the object matching the id provided.  Uses `object_id` only
+        and raises InvalidQueryError if a filter has been applied.
 
         :param object_id: the value for the id of the document to look up
+
+        .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set
         """
-        return self._document.objects(pk=object_id).first()
+        if not self._query_obj.empty:
+            raise InvalidQueryError("Cannot use a filter whilst using `with_id`")
+        return self.filter(pk=object_id).first()
 
     def in_bulk(self, object_ids):
         """Retrieve a set of documents by their ids.
@@ -843,6 +940,11 @@ class QuerySet(object):
 
         docs = self._collection.find({'_id': {'$in': object_ids}},
                                      **self._cursor_args)
+        if self._scalar:
+            for doc in docs:
+                doc_map[doc['_id']] = self._get_scalar(
+                        self._document._from_son(doc))
+        else:
             for doc in docs:
                 doc_map[doc['_id']] = self._document._from_son(doc)
 
@@ -854,6 +956,9 @@ class QuerySet(object):
         try:
             if self._limit == 0:
                 raise StopIteration
+            if self._scalar:
+                return self._get_scalar(self._document._from_son(
+                        self._cursor.next()))
             return self._document._from_son(self._cursor.next())
         except StopIteration, e:
             self.rewind()
@@ -887,9 +992,9 @@ class QuerySet(object):
         and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced`
         tests in ``tests.queryset.QuerySetTest`` for usage examples.
 
-        :param map_f: map function, as :class:`~pymongo.code.Code` or string
+        :param map_f: map function, as :class:`~bson.code.Code` or string
         :param reduce_f: reduce function, as
-                         :class:`~pymongo.code.Code` or string
+                         :class:`~bson.code.Code` or string
         :param output: output collection name, if set to 'inline' will try to
                        use :class:`~pymongo.collection.Collection.inline_map_reduce`
         :param finalize_f: finalize function, an optional function that
@@ -919,27 +1024,27 @@ class QuerySet(object):
             raise NotImplementedError("Requires MongoDB >= 1.7.1")
 
         map_f_scope = {}
-        if isinstance(map_f, pymongo.code.Code):
+        if isinstance(map_f, Code):
             map_f_scope = map_f.scope
             map_f = unicode(map_f)
-        map_f = pymongo.code.Code(self._sub_js_fields(map_f), map_f_scope)
+        map_f = Code(self._sub_js_fields(map_f), map_f_scope)
 
         reduce_f_scope = {}
-        if isinstance(reduce_f, pymongo.code.Code):
+        if isinstance(reduce_f, Code):
             reduce_f_scope = reduce_f.scope
             reduce_f = unicode(reduce_f)
         reduce_f_code = self._sub_js_fields(reduce_f)
-        reduce_f = pymongo.code.Code(reduce_f_code, reduce_f_scope)
+        reduce_f = Code(reduce_f_code, reduce_f_scope)
 
         mr_args = {'query': self._query}
 
         if finalize_f:
             finalize_f_scope = {}
-            if isinstance(finalize_f, pymongo.code.Code):
+            if isinstance(finalize_f, Code):
                 finalize_f_scope = finalize_f.scope
                 finalize_f = unicode(finalize_f)
             finalize_f_code = self._sub_js_fields(finalize_f)
-            finalize_f = pymongo.code.Code(finalize_f_code, finalize_f_scope)
+            finalize_f = Code(finalize_f_code, finalize_f_scope)
             mr_args['finalize'] = finalize_f
 
         if scope:
@@ -1030,6 +1135,9 @@ class QuerySet(object):
             return self
         # Integer index provided
         elif isinstance(key, int):
+            if self._scalar:
+                return self._get_scalar(self._document._from_son(
+                        self._cursor[key]))
             return self._document._from_son(self._cursor[key])
         raise AttributeError
 
@@ -1039,8 +1147,10 @@ class QuerySet(object):
         :param field: the field to select distinct values from
 
         .. versionadded:: 0.4
+        .. versionchanged:: 0.5 - Fixed handling references
         """
-        return self._cursor.distinct(field)
+        from dereference import DeReference
+        return DeReference()(self._cursor.distinct(field), 1)
 
     def only(self, *fields):
         """Load only a subset of this document's fields. ::
@@ -1209,11 +1319,17 @@ class QuerySet(object):
             document_cls, field_name = rule_entry
             rule = doc._meta['delete_rules'][rule_entry]
             if rule == CASCADE:
-                document_cls.objects(**{field_name + '__in': self}).delete(safe=safe)
+                ref_q = document_cls.objects(**{field_name + '__in': self})
+                if doc != document_cls or (doc == document_cls and ref_q.count() > 0):
+                    ref_q.delete(safe=safe)
             elif rule == NULLIFY:
                 document_cls.objects(**{field_name + '__in': self}).update(
                         safe_update=safe,
                         **{'unset__%s' % field_name: 1})
+            elif rule == PULL:
+                document_cls.objects(**{field_name + '__in': self}).update(
+                        safe_update=safe,
+                        **{'pull_all__%s' % field_name: self})
 
         self._collection.remove(self._query, safe=safe)
 
@@ -1226,6 +1342,9 @@ class QuerySet(object):
 
         mongo_update = {}
         for key, value in update.items():
+            if key == "__raw__":
+                mongo_update.update(value)
+                continue
             parts = key.split('__')
             # Check for an operator and transform to mongo-style if there is
             op = None
@@ -1249,8 +1368,8 @@ class QuerySet(object):
                 parts = []
 
                 cleaned_fields = []
-                append_field = True
                 for field in fields:
+                    append_field = True
                     if isinstance(field, str):
                         # Convert the S operator to $
                         if field == 'S':
@@ -1266,17 +1385,30 @@ class QuerySet(object):
                 field = cleaned_fields[-1]
 
                 if op in (None, 'set', 'push', 'pull', 'addToSet'):
+                    if field.required or value is not None:
                         value = field.prepare_query_value(op, value)
                 elif op in ('pushAll', 'pullAll'):
                     value = [field.prepare_query_value(op, v) for v in value]
 
             key = '.'.join(parts)
 
-            if op:
+            if not op:
+                raise InvalidQueryError("Updates must supply an operation eg: set__FIELD=value")
+
+            if 'pull' in op and '.' in key:
+                # Dot operators don't work on pull operations
+                # it uses nested dict syntax
+                if op == 'pullAll':
+                    raise InvalidQueryError("pullAll operations only support a single field depth")
+
+                parts.reverse()
+                for key in parts:
+                    value = {key: value}
+            else:
                 value = {key: value}
             key = '$' + op
 
-            if op is None or key not in mongo_update:
+            if key not in mongo_update:
                 mongo_update[key] = value
             elif key in mongo_update and isinstance(mongo_update[key], dict):
                 mongo_update[key].update(value)
@@ -1300,8 +1432,15 @@ class QuerySet(object):
             write_options = {}
 
         update = QuerySet._transform_update(self._document, **update)
+        query = self._query
+
+        # SERVER-5247 hack
+        remove_types = "_types" in query and ".$." in unicode(update)
+        if remove_types:
+            del query["_types"]
+
         try:
-            ret = self._collection.update(self._query, update, multi=multi,
+            ret = self._collection.update(query, update, multi=multi,
                                           upsert=upsert, safe=safe_update,
                                           **write_options)
             if ret is not None and 'n' in ret:
@@ -1329,10 +1468,17 @@ class QuerySet(object):
         if not write_options:
             write_options = {}
         update = QuerySet._transform_update(self._document, **update)
+        query = self._query
+
+        # SERVER-5247 hack
+        remove_types = "_types" in query and ".$." in unicode(update)
+        if remove_types:
+            del query["_types"]
+
         try:
             # Explicitly provide 'multi=False' to newer versions of PyMongo
             # as the default may change to 'True'
-            ret = self._collection.update(self._query, update, multi=False,
+            ret = self._collection.update(query, update, multi=False,
                                           upsert=upsert, safe=safe_update,
                                            **write_options)
 
@@ -1342,8 +1488,45 @@ class QuerySet(object):
             raise OperationError(u'Update failed [%s]' % unicode(e))
 
     def __iter__(self):
+        self.rewind()
         return self
 
+    def _get_scalar(self, doc):
+
+        def lookup(obj, name):
+            chunks = name.split('__')
+            for chunk in chunks:
+                obj = getattr(obj, chunk)
+            return obj
+
+        data = [lookup(doc, n) for n in self._scalar]
+        if len(data) == 1:
+            return data[0]
+
+        return tuple(data)
+
+    def scalar(self, *fields):
+        """Instead of returning Document instances, return either a specific
+        value or a tuple of values in order.
+
+        This effects all results and can be unset by calling ``scalar``
+        without arguments. Calls ``only`` automatically.
+
+        :param fields: One or more fields to return instead of a Document.
+        """
+        self._scalar = list(fields)
+
+        if fields:
+            self.only(*fields)
+        else:
+            self.all_fields()
+
+        return self
+
+    def values_list(self, *fields):
+        """An alias for scalar"""
+        return self.scalar(*fields)
+
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where
         *fieldname* is the Python name of a field, *fieldname* will be
@@ -1406,9 +1589,9 @@ class QuerySet(object):
             query['$where'] = self._where_clause
 
         scope['query'] = query
-        code = pymongo.code.Code(code, scope=scope)
+        code = Code(code, scope=scope)
 
-        db = _get_db()
+        db = self._document._get_db()
         return db.eval(code, *fields)
 
     def where(self, where_clause):
@@ -1435,13 +1618,13 @@ class QuerySet(object):
         .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work
             with sharding.
         """
-        map_func = pymongo.code.Code("""
+        map_func = Code("""
             function() {
                 emit(1, this[field] || 0);
             }
         """, scope={'field': field})
 
-        reduce_func = pymongo.code.Code("""
+        reduce_func = Code("""
             function(key, values) {
                 var sum = 0;
                 for (var i in values) {
@@ -1465,14 +1648,14 @@ class QuerySet(object):
         .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work
             with sharding.
         """
-        map_func = pymongo.code.Code("""
+        map_func = Code("""
             function() {
                 if (this.hasOwnProperty(field))
                     emit(1, {t: this[field] || 0, c: 1});
             }
         """, scope={'field': field})
 
-        reduce_func = pymongo.code.Code("""
+        reduce_func = Code("""
             function(key, values) {
                 var out = {t: 0, c: 0};
                 for (var i in values) {
@@ -1484,7 +1667,7 @@ class QuerySet(object):
             }
         """)
 
-        finalize_func = pymongo.code.Code("""
+        finalize_func = Code("""
             function(key, value) {
                 return value.t / value.c;
             }
@@ -1526,13 +1709,20 @@ class QuerySet(object):
             function() {
                 path = '{{~%(field)s}}'.split('.');
                 field = this;
-                for (p in path) { field = field[path[p]]; }
+                for (p in path) {
+                    if (field)
+                       field = field[path[p]];
+                    else
+                       break;
+                }
                 if (field && field.constructor == Array) {
                     field.forEach(function(item) {
                         emit(item, 1);
                     });
-                } else {
+                } else if (field) {
                     emit(field, 1);
+                } else {
+                    emit(null, 1);
                 }
             }
         """ % dict(field=field)
@@ -1572,7 +1762,12 @@ class QuerySet(object):
                     var total = 0.0;
                     db[collection].find(query).forEach(function(doc) {
                         field = doc;
-                        for (p in path) { field = field[path[p]]; }
+                        for (p in path) {
+                            if (field)
+                                field = field[path[p]];
+                            else
+                                break;
+                        }
                         if (field && field.constructor == Array) {
                             total += field.length;
                         } else {
@@ -1588,7 +1783,12 @@ class QuerySet(object):
                 }
                 db[collection].find(query).forEach(function(doc) {
                     field = doc;
-                    for (p in path) { field = field[path[p]]; }
+                    for (p in path) {
+                        if (field)
+                            field = field[path[p]];
+                        else
+                            break;
+                    }
                     if (field && field.constructor == Array) {
                         field.forEach(function(item) {
                             frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]);
@@ -1609,10 +1809,16 @@ class QuerySet(object):
 
     def __repr__(self):
         limit = REPR_OUTPUT_SIZE + 1
-        if self._limit is not None and self._limit < limit:
-            limit = self._limit
+        start = (0 if self._skip is None else self._skip)
+        if self._limit is None:
+            stop = start + limit
+        if self._limit is not None:
+            if self._limit - start > limit:
+                stop = start + limit
+            else:
+                stop = self._limit
         try:
-            data = list(self[self._skip:limit])
+            data = list(self[start:stop])
         except pymongo.errors.InvalidOperation:
             return ".. queryset mid-iteration .."
         if len(data) > REPR_OUTPUT_SIZE:
@@ -1620,13 +1826,15 @@ class QuerySet(object):
         return repr(data)
 
     def select_related(self, max_depth=1):
-        """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to
+        """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to
         a maximum depth in order to cut down the number queries to mongodb.
 
         .. versionadded:: 0.5
         """
-        from dereference import dereference
-        return dereference(self, max_depth=max_depth)
+        from dereference import DeReference
+        # Make select related work the same for querysets
+        max_depth += 1
+        return DeReference()(self, max_depth=max_depth)
 
 
 class QuerySetManager(object):

mongoengine/signals.py

@@ -42,3 +42,5 @@ pre_save = _signals.signal('pre_save')
 post_save = _signals.signal('post_save')
 pre_delete = _signals.signal('pre_delete')
 post_delete = _signals.signal('post_delete')
+pre_bulk_insert = _signals.signal('pre_bulk_insert')
+post_bulk_insert = _signals.signal('post_bulk_insert')

mongoengine/tests.py

@@ -1,4 +1,4 @@
-from mongoengine.connection import _get_db
+from mongoengine.connection import get_db
 
 
 class query_counter(object):
@@ -7,7 +7,7 @@ class query_counter(object):
     def __init__(self):
         """ Construct the query_counter. """
         self.counter = 0
-        self.db = _get_db()
+        self.db = get_db()
 
     def __enter__(self):
         """ On every with block we need to drop the profile collection. """

python-mongoengine.spec

@@ -0,0 +1,54 @@
+# sitelib for noarch packages, sitearch for others (remove the unneeded one)
+%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+
+%define srcname mongoengine
+
+Name:           python-%{srcname}
+Version:        0.6.12
+Release:        1%{?dist}
+Summary:        A Python Document-Object Mapper for working with MongoDB
+
+Group:          Development/Libraries
+License:        MIT
+URL:            https://github.com/MongoEngine/mongoengine
+Source0:        %{srcname}-%{version}.tar.bz2
+
+BuildRequires:  python-devel
+BuildRequires:  python-setuptools
+
+Requires:       mongodb
+Requires:       pymongo
+Requires:       python-blinker
+Requires:       python-imaging
+
+
+%description
+MongoEngine is an ORM-like layer on top of PyMongo.
+
+%prep
+%setup -q -n %{srcname}-%{version}
+
+
+%build
+# Remove CFLAGS=... for noarch packages (unneeded)
+CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root,-)
+%doc docs AUTHORS LICENSE README.rst
+# For noarch packages: sitelib
+ %{python_sitelib}/*
+# For arch-specific packages: sitearch
+# %{python_sitearch}/*
+
+%changelog
+* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html

requirements.txt

@@ -0,0 +1 @@
+pymongo

setup.cfg

@@ -0,0 +1,13 @@
+[aliases]
+test = nosetests
+
+[nosetests]
+verbosity = 2
+detailed-errors = 1
+#with-coverage = 1
+cover-html = 1
+cover-html-dir = ../htmlcov
+cover-package = mongoengine
+cover-erase = 1
+where = tests
+#tests = test_bugfix.py

setup.py

@@ -38,7 +38,9 @@ setup(name='mongoengine',
       packages=find_packages(),
       author='Harry Marr',
       author_email='harry.marr@{nospam}gmail.com',
-      url='http://hmarr.com/mongoengine/',
+      maintainer="Ross Lawley",
+      maintainer_email="ross.lawley@{nospam}gmail.com",
+      url='http://mongoengine.org/',
       license='MIT',
       include_package_data=True,
       description=DESCRIPTION,
@@ -46,6 +48,5 @@ setup(name='mongoengine',
       platforms=['any'],
       classifiers=CLASSIFIERS,
       install_requires=['pymongo'],
-      test_suite='tests',
-      tests_require=['blinker', 'django==1.3']
+      tests_require=['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL']
 )

tests/fixtures.py

@@ -1,9 +1,6 @@
 from datetime import datetime
-import pymongo
 
 from mongoengine import *
-from mongoengine.base import BaseField
-from mongoengine.connection import _get_db
 
 
 class PickleEmbedded(EmbeddedDocument):
@@ -15,6 +12,7 @@ class PickleTest(Document):
     string = StringField(choices=(('One', '1'), ('Two', '2')))
     embedded = EmbeddedDocumentField(PickleEmbedded)
     lists = ListField(StringField())
+    photo = FileField()
 
 
 class Mixin(object):
@@ -22,4 +20,4 @@ class Mixin(object):
 
 
 class Base(Document):
-    pass
+    meta = {'allow_inheritance': True}

tests/test_connection.py

@@ -0,0 +1,98 @@
+import datetime
+import pymongo
+import unittest
+
+import mongoengine.connection
+
+from bson.tz_util import utc
+
+from mongoengine import *
+from mongoengine.connection import get_db, get_connection, ConnectionError
+
+
+class ConnectionTest(unittest.TestCase):
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    def test_connect(self):
+        """Ensure that the connect() method works properly.
+        """
+        connect('mongoenginetest')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+        connect('mongoenginetest2', alias='testdb')
+        conn = get_connection('testdb')
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+    def test_connect_uri(self):
+        """Ensure that the connect() method works properly with uri's
+        """
+        c = connect(db='mongoenginetest', alias='admin')
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
+        c.admin.add_user("admin", "password")
+        c.admin.authenticate("admin", "password")
+        c.mongoenginetest.add_user("username", "password")
+
+        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
+
+        connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+    def test_register_connection(self):
+        """Ensure that connections with different aliases may be registered.
+        """
+        register_connection('testdb', 'mongoenginetest2')
+
+        self.assertRaises(ConnectionError, get_connection)
+        conn = get_connection('testdb')
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db('testdb')
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest2')
+
+    def test_connection_kwargs(self):
+        """Ensure that connection kwargs get passed to pymongo.
+        """
+        connect('mongoenginetest', alias='t1', tz_aware=True)
+        conn = get_connection('t1')
+
+        self.assertTrue(conn.tz_aware)
+
+        connect('mongoenginetest2', alias='t2')
+        conn = get_connection('t2')
+        self.assertFalse(conn.tz_aware)
+
+    def test_datetime(self):
+        connect('mongoenginetest', tz_aware=True)
+        d = datetime.datetime(2010, 5, 5, tzinfo=utc)
+
+        class DateDoc(Document):
+            the_date = DateTimeField(required=True)
+
+        DateDoc.drop_collection()
+        DateDoc(the_date=d).save()
+
+        date_doc = DateDoc.objects.first()
+        self.assertEqual(d, date_doc.the_date)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_dereference.py

@@ -1,7 +1,7 @@
 import unittest
 
 from mongoengine import *
-from mongoengine.connection import _get_db
+from mongoengine.connection import get_db
 from mongoengine.tests import query_counter
 
 
@@ -9,7 +9,7 @@ class FieldTest(unittest.TestCase):
 
     def setUp(self):
         connect(db='mongoenginetest')
-        self.db = _get_db()
+        self.db = get_db()
 
     def test_list_item_dereference(self):
         """Ensure that DBRef items in ListFields are dereferenced.
@@ -760,3 +760,84 @@ class FieldTest(unittest.TestCase):
         UserB.drop_collection()
         UserC.drop_collection()
         Group.drop_collection()
+
+    def test_multidirectional_lists(self):
+
+        class Asset(Document):
+            name = StringField(max_length=250, required=True)
+            parent = GenericReferenceField(default=None)
+            parents = ListField(GenericReferenceField())
+            children = ListField(GenericReferenceField())
+
+        Asset.drop_collection()
+
+        root = Asset(name='', path="/", title="Site Root")
+        root.save()
+
+        company = Asset(name='company', title='Company', parent=root, parents=[root])
+        company.save()
+
+        root.children = [company]
+        root.save()
+
+        root = root.reload()
+        self.assertEquals(root.children, [company])
+        self.assertEquals(company.parents, [root])
+
+    def test_dict_in_dbref_instance(self):
+
+        class Person(Document):
+            name = StringField(max_length=250, required=True)
+
+        class Room(Document):
+            number = StringField(max_length=250, required=True)
+            staffs_with_position = ListField(DictField())
+
+        Person.drop_collection()
+        Room.drop_collection()
+
+        bob = Person.objects.create(name='Bob')
+        bob.save()
+        sarah = Person.objects.create(name='Sarah')
+        sarah.save()
+
+        room_101 = Room.objects.create(number="101")
+        room_101.staffs_with_position = [
+            {'position_key': 'window', 'staff': sarah},
+            {'position_key': 'door', 'staff': bob.to_dbref()}]
+        room_101.save()
+
+        room = Room.objects.first().select_related()
+        self.assertEquals(room.staffs_with_position[0]['staff'], sarah)
+        self.assertEquals(room.staffs_with_position[1]['staff'], bob)
+    
+    def test_document_reload_no_inheritance(self):
+        class Foo(Document):
+            meta = {'allow_inheritance': False}
+            bar = ReferenceField('Bar')
+            baz = ReferenceField('Baz')
+
+        class Bar(Document):
+            meta = {'allow_inheritance': False}
+            msg = StringField(required=True, default='Blammo!')
+
+        class Baz(Document):
+            meta = {'allow_inheritance': False}
+            msg = StringField(required=True, default='Kaboom!')
+
+        Foo.drop_collection()
+        Bar.drop_collection()
+        Baz.drop_collection()
+
+        bar = Bar()
+        bar.save()
+        baz = Baz()
+        baz.save()
+        foo = Foo()
+        foo.bar = bar
+        foo.baz = baz
+        foo.save()
+        foo.reload()
+
+        self.assertEquals(type(foo.bar), Bar)
+        self.assertEquals(type(foo.baz), Baz)

tests/test_django.py

@@ -8,8 +8,14 @@ from mongoengine.django.shortcuts import get_document_or_404
 from django.http import Http404
 from django.template import Context, Template
 from django.conf import settings
+from django.core.paginator import Paginator
+
 settings.configure()
 
+from django.contrib.sessions.tests import SessionTestsMixin
+from mongoengine.django.sessions import SessionStore, MongoSession
+
+
 class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
@@ -67,3 +73,38 @@ class QuerySetTest(unittest.TestCase):
         self.assertRaises(Http404, get_document_or_404, self.Person, pk='1234')
         self.assertEqual(p, get_document_or_404(self.Person, pk=p.pk))
 
+    def test_pagination(self):
+        """Ensure that Pagination works as expected
+        """
+        class Page(Document):
+            name = StringField()
+
+        Page.drop_collection()
+
+        for i in xrange(1, 11):
+            Page(name=str(i)).save()
+
+        paginator = Paginator(Page.objects.all(), 2)
+
+        t = Template("{% for i in page.object_list  %}{{ i.name }}:{% endfor %}")
+        for p in paginator.page_range:
+            d = {"page": paginator.page(p)}
+            end = p * 2
+            start = end - 1
+            self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end))
+
+
+
+class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase):
+    backend = SessionStore
+
+    def setUp(self):
+        connect(db='mongoenginetest')
+        MongoSession.drop_collection()
+        super(MongoDBSessionTest, self).setUp()
+
+    def test_first_save(self):
+        session = SessionStore()
+        session['test'] = True
+        session.save()
+        self.assertTrue('test' in session)

tests/test_dynamic_document.py

@@ -0,0 +1,502 @@
+import unittest
+
+from mongoengine import *
+from mongoengine.connection import get_db
+
+
+class DynamicDocTest(unittest.TestCase):
+
+    def setUp(self):
+        connect(db='mongoenginetest')
+        self.db = get_db()
+
+        class Person(DynamicDocument):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        Person.drop_collection()
+
+        self.Person = Person
+
+    def test_simple_dynamic_document(self):
+        """Ensures simple dynamic documents are saved correctly"""
+
+        p = self.Person()
+        p.name = "James"
+        p.age = 34
+
+        self.assertEquals(p.to_mongo(),
+            {"_types": ["Person"], "_cls": "Person",
+             "name": "James", "age": 34}
+        )
+
+        p.save()
+
+        self.assertEquals(self.Person.objects.first().age, 34)
+
+        # Confirm no changes to self.Person
+        self.assertFalse(hasattr(self.Person, 'age'))
+
+    def test_dynamic_document_delta(self):
+        """Ensures simple dynamic documents can delta correctly"""
+        p = self.Person(name="James", age=34)
+        self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {}))
+
+        p.doc = 123
+        del(p.doc)
+        self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1}))
+
+    def test_change_scope_of_variable(self):
+        """Test changing the scope of a dynamic field has no adverse effects"""
+        p = self.Person()
+        p.name = "Dean"
+        p.misc = 22
+        p.save()
+
+        p = self.Person.objects.get()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertEquals(p.misc, {'hello': 'world'})
+
+    def test_delete_dynamic_field(self):
+        """Test deleting a dynamic field works"""
+        self.Person.drop_collection()
+        p = self.Person()
+        p.name = "Dean"
+        p.misc = 22
+        p.save()
+
+        p = self.Person.objects.get()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertEquals(p.misc, {'hello': 'world'})
+        collection = self.db[self.Person._get_collection_name()]
+        obj = collection.find_one()
+        self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name'])
+
+        del(p.misc)
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertFalse(hasattr(p, 'misc'))
+
+        obj = collection.find_one()
+        self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'name'])
+
+    def test_dynamic_document_queries(self):
+        """Ensure we can query dynamic fields"""
+        p = self.Person()
+        p.name = "Dean"
+        p.age = 22
+        p.save()
+
+        self.assertEquals(1, self.Person.objects(age=22).count())
+        p = self.Person.objects(age=22)
+        p = p.get()
+        self.assertEquals(22, p.age)
+
+    def test_complex_dynamic_document_queries(self):
+        class Person(DynamicDocument):
+            name = StringField()
+
+        Person.drop_collection()
+
+        p = Person(name="test")
+        p.age = "ten"
+        p.save()
+
+        p1 = Person(name="test1")
+        p1.age = "less then ten and a half"
+        p1.save()
+
+        p2 = Person(name="test2")
+        p2.age = 10
+        p2.save()
+
+        self.assertEquals(Person.objects(age__icontains='ten').count(), 2)
+        self.assertEquals(Person.objects(age__gte=10).count(), 1)
+
+    def test_complex_data_lookups(self):
+        """Ensure you can query dynamic document dynamic fields"""
+        p = self.Person()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        self.assertEquals(1, self.Person.objects(misc__hello='world').count())
+
+    def test_inheritance(self):
+        """Ensure that dynamic document plays nice with inheritance"""
+        class Employee(self.Person):
+            salary = IntField()
+
+        Employee.drop_collection()
+
+        self.assertTrue('name' in Employee._fields)
+        self.assertTrue('salary' in Employee._fields)
+        self.assertEqual(Employee._get_collection_name(),
+                         self.Person._get_collection_name())
+
+        joe_bloggs = Employee()
+        joe_bloggs.name = "Joe Bloggs"
+        joe_bloggs.salary = 10
+        joe_bloggs.age = 20
+        joe_bloggs.save()
+
+        self.assertEquals(1, self.Person.objects(age=20).count())
+        self.assertEquals(1, Employee.objects(age=20).count())
+
+        joe_bloggs = self.Person.objects.first()
+        self.assertTrue(isinstance(joe_bloggs, Employee))
+
+    def test_embedded_dynamic_document(self):
+        """Test dynamic embedded documents"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+        embedded_1.list_field = ['1', 2, {'hello': 'world'}]
+        doc.embedded_field = embedded_1
+
+        self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc",
+            "embedded_field": {
+                "_types": ['Embedded'], "_cls": "Embedded",
+                "string_field": "hello",
+                "int_field": 1,
+                "dict_field": {"hello": "world"},
+                "list_field": ['1', 2, {'hello': 'world'}]
+            }
+        })
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEquals(doc.embedded_field.__class__, Embedded)
+        self.assertEquals(doc.embedded_field.string_field, "hello")
+        self.assertEquals(doc.embedded_field.int_field, 1)
+        self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'})
+        self.assertEquals(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}])
+
+    def test_complex_embedded_documents(self):
+        """Test complex dynamic embedded documents setups"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+
+        embedded_2 = Embedded()
+        embedded_2.string_field = 'hello'
+        embedded_2.int_field = 1
+        embedded_2.dict_field = {'hello': 'world'}
+        embedded_2.list_field = ['1', 2, {'hello': 'world'}]
+
+        embedded_1.list_field = ['1', 2, embedded_2]
+        doc.embedded_field = embedded_1
+
+        self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc",
+            "embedded_field": {
+                "_types": ['Embedded'], "_cls": "Embedded",
+                "string_field": "hello",
+                "int_field": 1,
+                "dict_field": {"hello": "world"},
+                "list_field": ['1', 2,
+                    {"_types": ['Embedded'], "_cls": "Embedded",
+                    "string_field": "hello",
+                    "int_field": 1,
+                    "dict_field": {"hello": "world"},
+                    "list_field": ['1', 2, {'hello': 'world'}]}
+                ]
+            }
+        })
+        doc.save()
+        doc = Doc.objects.first()
+        self.assertEquals(doc.embedded_field.__class__, Embedded)
+        self.assertEquals(doc.embedded_field.string_field, "hello")
+        self.assertEquals(doc.embedded_field.int_field, 1)
+        self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'})
+        self.assertEquals(doc.embedded_field.list_field[0], '1')
+        self.assertEquals(doc.embedded_field.list_field[1], 2)
+
+        embedded_field = doc.embedded_field.list_field[2]
+
+        self.assertEquals(embedded_field.__class__, Embedded)
+        self.assertEquals(embedded_field.string_field, "hello")
+        self.assertEquals(embedded_field.int_field, 1)
+        self.assertEquals(embedded_field.dict_field, {'hello': 'world'})
+        self.assertEquals(embedded_field.list_field, ['1', 2, {'hello': 'world'}])
+
+    def test_delta_for_dynamic_documents(self):
+        p = self.Person()
+        p.name = "Dean"
+        p.age = 22
+        p.save()
+
+        p.age = 24
+        self.assertEquals(p.age, 24)
+        self.assertEquals(p._get_changed_fields(), ['age'])
+        self.assertEquals(p._delta(), ({'age': 24}, {}))
+
+        p = self.Person.objects(age=22).get()
+        p.age = 24
+        self.assertEquals(p.age, 24)
+        self.assertEquals(p._get_changed_fields(), ['age'])
+        self.assertEquals(p._delta(), ({'age': 24}, {}))
+
+        p.save()
+        self.assertEquals(1, self.Person.objects(age=24).count())
+
+    def test_delta(self):
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEquals(doc._get_changed_fields(), [])
+        self.assertEquals(doc._delta(), ({}, {}))
+
+        doc.string_field = 'hello'
+        self.assertEquals(doc._get_changed_fields(), ['string_field'])
+        self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {}))
+
+        doc._changed_fields = []
+        doc.int_field = 1
+        self.assertEquals(doc._get_changed_fields(), ['int_field'])
+        self.assertEquals(doc._delta(), ({'int_field': 1}, {}))
+
+        doc._changed_fields = []
+        dict_value = {'hello': 'world', 'ping': 'pong'}
+        doc.dict_field = dict_value
+        self.assertEquals(doc._get_changed_fields(), ['dict_field'])
+        self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {}))
+
+        doc._changed_fields = []
+        list_value = ['1', 2, {'hello': 'world'}]
+        doc.list_field = list_value
+        self.assertEquals(doc._get_changed_fields(), ['list_field'])
+        self.assertEquals(doc._delta(), ({'list_field': list_value}, {}))
+
+        # Test unsetting
+        doc._changed_fields = []
+        doc.dict_field = {}
+        self.assertEquals(doc._get_changed_fields(), ['dict_field'])
+        self.assertEquals(doc._delta(), ({}, {'dict_field': 1}))
+
+        doc._changed_fields = []
+        doc.list_field = []
+        self.assertEquals(doc._get_changed_fields(), ['list_field'])
+        self.assertEquals(doc._delta(), ({}, {'list_field': 1}))
+
+    def test_delta_recursive(self):
+        """Testing deltaing works with dynamic documents"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEquals(doc._get_changed_fields(), [])
+        self.assertEquals(doc._delta(), ({}, {}))
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+        embedded_1.list_field = ['1', 2, {'hello': 'world'}]
+        doc.embedded_field = embedded_1
+
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field'])
+
+        embedded_delta = {
+            'string_field': 'hello',
+            'int_field': 1,
+            'dict_field': {'hello': 'world'},
+            'list_field': ['1', 2, {'hello': 'world'}]
+        }
+        self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {}))
+        embedded_delta.update({
+            '_types': ['Embedded'],
+            '_cls': 'Embedded',
+        })
+        self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {}))
+
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.dict_field = {}
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field'])
+        self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1}))
+
+        self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1}))
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.list_field = []
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1}))
+        self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1}))
+        doc.save()
+        doc.reload()
+
+        embedded_2 = Embedded()
+        embedded_2.string_field = 'hello'
+        embedded_2.int_field = 1
+        embedded_2.dict_field = {'hello': 'world'}
+        embedded_2.list_field = ['1', 2, {'hello': 'world'}]
+
+        doc.embedded_field.list_field = ['1', 2, embedded_2]
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEquals(doc.embedded_field._delta(), ({
+            'list_field': ['1', 2, {
+                '_cls': 'Embedded',
+                '_types': ['Embedded'],
+                'string_field': 'hello',
+                'dict_field': {'hello': 'world'},
+                'int_field': 1,
+                'list_field': ['1', 2, {'hello': 'world'}],
+            }]
+        }, {}))
+
+        self.assertEquals(doc._delta(), ({
+            'embedded_field.list_field': ['1', 2, {
+                '_cls': 'Embedded',
+                 '_types': ['Embedded'],
+                 'string_field': 'hello',
+                 'dict_field': {'hello': 'world'},
+                 'int_field': 1,
+                 'list_field': ['1', 2, {'hello': 'world'}],
+            }]
+        }, {}))
+        doc.save()
+        doc.reload()
+
+        self.assertEquals(doc.embedded_field.list_field[2]._changed_fields, [])
+        self.assertEquals(doc.embedded_field.list_field[0], '1')
+        self.assertEquals(doc.embedded_field.list_field[1], 2)
+        for k in doc.embedded_field.list_field[2]._fields:
+            self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k])
+
+        doc.embedded_field.list_field[2].string_field = 'world'
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field'])
+        self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {}))
+        self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world')
+
+        # Test multiple assignments
+        doc.embedded_field.list_field[2].string_field = 'hello world'
+        doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2]
+        self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEquals(doc.embedded_field._delta(), ({
+            'list_field': ['1', 2, {
+            '_types': ['Embedded'],
+            '_cls': 'Embedded',
+            'string_field': 'hello world',
+            'int_field': 1,
+            'list_field': ['1', 2, {'hello': 'world'}],
+            'dict_field': {'hello': 'world'}}]}, {}))
+        self.assertEquals(doc._delta(), ({
+            'embedded_field.list_field': ['1', 2, {
+                '_types': ['Embedded'],
+                '_cls': 'Embedded',
+                'string_field': 'hello world',
+                'int_field': 1,
+                'list_field': ['1', 2, {'hello': 'world'}],
+                'dict_field': {'hello': 'world'}}
+            ]}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world')
+
+        # Test list native methods
+        doc.embedded_field.list_field[2].list_field.pop(0)
+        self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {}))
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.list_field[2].list_field.append(1)
+        self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1])
+
+        doc.embedded_field.list_field[2].list_field.sort()
+        doc.save()
+        doc.reload()
+        self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}])
+
+        del(doc.embedded_field.list_field[2].list_field[2]['hello'])
+        self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {}))
+        doc.save()
+        doc.reload()
+
+        del(doc.embedded_field.list_field[2].list_field)
+        self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1}))
+
+        doc.save()
+        doc.reload()
+
+        doc.dict_field = {'embedded': embedded_1}
+        doc.save()
+        doc.reload()
+
+        doc.dict_field['embedded'].string_field = 'Hello World'
+        self.assertEquals(doc._get_changed_fields(), ['dict_field.embedded.string_field'])
+        self.assertEquals(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {}))
+
+    def test_indexes(self):
+        """Ensure that indexes are used when meta[indexes] is specified.
+        """
+        class BlogPost(DynamicDocument):
+            meta = {
+                'indexes': [
+                    '-date',
+                    ('category', '-date')
+                ],
+            }
+
+        BlogPost.drop_collection()
+
+        info = BlogPost.objects._collection.index_information()
+        # _id, '-date', ('cat', 'date')
+        # NB: there is no index on _types by itself, since
+        # the indices on -date and tags will both contain
+        # _types as first element in the key
+        self.assertEqual(len(info), 3)
+
+        # Indexes are lazy so use list() to perform query
+        list(BlogPost.objects)
+        info = BlogPost.objects._collection.index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('_types', 1), ('category', 1), ('date', -1)]
+                        in info)
+        self.assertTrue([('_types', 1), ('date', -1)] in info)

tests/test_fields.py

@@ -1,20 +1,29 @@
-import unittest
 import datetime
-from decimal import Decimal
-
-import pymongo
+import os
+import unittest
+import uuid
+import StringIO
+import tempfile
 import gridfs
 
+from decimal import Decimal
+
 from mongoengine import *
-from mongoengine.connection import _get_db
+from mongoengine.connection import get_db
 from mongoengine.base import _document_registry, NotRegistered
 
+TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png')
+
 
 class FieldTest(unittest.TestCase):
 
     def setUp(self):
         connect(db='mongoenginetest')
-        self.db = _get_db()
+        self.db = get_db()
+
+    def tearDown(self):
+        self.db.drop_collection('fs.files')
+        self.db.drop_collection('fs.chunks')
 
     def test_default_values(self):
         """Ensure that default field values are used when creating a document.
@@ -44,6 +53,80 @@ class FieldTest(unittest.TestCase):
         person = Person(age=30)
         self.assertRaises(ValidationError, person.validate)
 
+    def test_not_required_handles_none_in_update(self):
+        """Ensure that every fields should accept None if required is False.
+        """
+
+        class HandleNoneFields(Document):
+            str_fld = StringField()
+            int_fld = IntField()
+            flt_fld = FloatField()
+            comp_dt_fld = ComplexDateTimeField()
+
+        HandleNoneFields.drop_collection()
+
+        doc = HandleNoneFields()
+        doc.str_fld = u'spam ham egg'
+        doc.int_fld = 42
+        doc.flt_fld = 4.2
+        doc.com_dt_fld = datetime.datetime.utcnow()
+        doc.save()
+
+        res = HandleNoneFields.objects(id=doc.id).update(
+            set__str_fld=None,
+            set__int_fld=None,
+            set__flt_fld=None,
+            set__comp_dt_fld=None,
+        )
+        self.assertEqual(res, 1)
+
+        # Retrive data from db and verify it.
+        ret = HandleNoneFields.objects.all()[0]
+        self.assertEqual(ret.str_fld, None)
+        self.assertEqual(ret.int_fld, None)
+        self.assertEqual(ret.flt_fld, None)
+
+        # Return current time if retrived value is None.
+        self.assertTrue(isinstance(ret.comp_dt_fld, datetime.datetime))
+
+    def test_not_required_handles_none_from_database(self):
+        """Ensure that every fields can handle null values from the database.
+        """
+
+        class HandleNoneFields(Document):
+            str_fld = StringField(required=True)
+            int_fld = IntField(required=True)
+            flt_fld = FloatField(required=True)
+            comp_dt_fld = ComplexDateTimeField(required=True)
+
+        HandleNoneFields.drop_collection()
+
+        doc = HandleNoneFields()
+        doc.str_fld = u'spam ham egg'
+        doc.int_fld = 42
+        doc.flt_fld = 4.2
+        doc.com_dt_fld = datetime.datetime.utcnow()
+        doc.save()
+
+        collection = self.db[HandleNoneFields._get_collection_name()]
+        obj = collection.update({"_id": doc.id}, {"$unset": {
+            "str_fld": 1,
+            "int_fld": 1,
+            "flt_fld": 1,
+            "comp_dt_fld": 1}
+        })
+
+        # Retrive data from db and verify it.
+        ret = HandleNoneFields.objects.all()[0]
+
+        self.assertEqual(ret.str_fld, None)
+        self.assertEqual(ret.int_fld, None)
+        self.assertEqual(ret.flt_fld, None)
+        # Return current time if retrived value is None.
+        self.assert_(isinstance(ret.comp_dt_fld, datetime.datetime))
+
+        self.assertRaises(ValidationError, ret.validate)
+
     def test_object_id_validation(self):
         """Ensure that invalid values cannot be assigned to string fields.
         """
@@ -175,6 +258,26 @@ class FieldTest(unittest.TestCase):
         person.admin = 'Yes'
         self.assertRaises(ValidationError, person.validate)
 
+    def test_uuid_validation(self):
+        """Ensure that invalid values cannot be assigned to UUID fields.
+        """
+        class Person(Document):
+            api_key = UUIDField()
+
+        person = Person()
+        # any uuid type is valid
+        person.api_key = uuid.uuid4()
+        person.validate()
+        person.api_key = uuid.uuid1()
+        person.validate()
+
+        # last g cannot belong to an hex number
+        person.api_key = '9d159858-549b-4975-9f98-dd2f987c113g'
+        self.assertRaises(ValidationError, person.validate)
+        # short strings don't validate
+        person.api_key = '9d159858-549b-4975-9f98-dd2f987c113'
+        self.assertRaises(ValidationError, person.validate)
+
     def test_datetime_validation(self):
         """Ensure that invalid values cannot be assigned to datetime fields.
         """
@@ -459,6 +562,31 @@ class FieldTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_reverse_list_sorting(self):
+        '''Ensure that a reverse sorted list field properly sorts values'''
+
+        class Category(EmbeddedDocument):
+            count = IntField()
+            name = StringField()
+
+        class CategoryList(Document):
+            categories = SortedListField(EmbeddedDocumentField(Category), ordering='count', reverse=True)
+            name = StringField()
+
+        catlist = CategoryList(name="Top categories")
+        cat1 = Category(name='posts', count=10)
+        cat2 = Category(name='food', count=100)
+        cat3 = Category(name='drink', count=40)
+        catlist.categories = [cat1, cat2, cat3]
+        catlist.save()
+        catlist.reload()
+
+        self.assertEqual(catlist.categories[0].name, cat2.name)
+        self.assertEqual(catlist.categories[1].name, cat3.name)
+        self.assertEqual(catlist.categories[2].name, cat1.name)
+
+        CategoryList.drop_collection()
+
     def test_list_field(self):
         """Ensure that list types work as expected.
         """
@@ -485,7 +613,6 @@ class FieldTest(unittest.TestCase):
         post.info = [{'test': 3}]
         post.save()
 
-
         self.assertEquals(BlogPost.objects.count(), 3)
         self.assertEquals(BlogPost.objects.filter(info__exact='test').count(), 1)
         self.assertEquals(BlogPost.objects.filter(info__0__test='test').count(), 1)
@@ -495,6 +622,21 @@ class FieldTest(unittest.TestCase):
         self.assertEquals(BlogPost.objects.filter(info__100__test__exact='test').count(), 0)
         BlogPost.drop_collection()
 
+    def test_list_field_passed_in_value(self):
+        class Foo(Document):
+            bars = ListField(ReferenceField("Bar"))
+
+        class Bar(Document):
+            text = StringField()
+
+        bar = Bar(text="hi")
+        bar.save()
+
+        foo = Foo(bars=[])
+        foo.bars.append(bar)
+        self.assertEquals(repr(foo.bars), '[<Bar: Bar object>]')
+
+
     def test_list_field_strict(self):
         """Ensure that list field handles validation if provided a strict field type."""
 
@@ -515,6 +657,39 @@ class FieldTest(unittest.TestCase):
 
         Simple.drop_collection()
 
+    def test_list_field_rejects_strings(self):
+        """Strings aren't valid list field data types"""
+
+        class Simple(Document):
+            mapping = ListField()
+
+        Simple.drop_collection()
+        e = Simple()
+        e.mapping = 'hello world'
+
+        self.assertRaises(ValidationError, e.save)
+
+    def test_complex_field_required(self):
+        """Ensure required cant be None / Empty"""
+
+        class Simple(Document):
+            mapping = ListField(required=True)
+
+        Simple.drop_collection()
+        e = Simple()
+        e.mapping = []
+
+        self.assertRaises(ValidationError, e.save)
+
+        class Simple(Document):
+            mapping = DictField(required=True)
+
+        Simple.drop_collection()
+        e = Simple()
+        e.mapping = {}
+
+        self.assertRaises(ValidationError, e.save)
+
     def test_list_field_complex(self):
         """Ensure that the list fields can handle the complex types."""
 
@@ -582,6 +757,9 @@ class FieldTest(unittest.TestCase):
         post.info = {'the.title': 'test'}
         self.assertRaises(ValidationError, post.validate)
 
+        post.info = {1: 'test'}
+        self.assertRaises(ValidationError, post.validate)
+
         post.info = {'title': 'test'}
         post.save()
 
@@ -600,6 +778,13 @@ class FieldTest(unittest.TestCase):
         # Confirm handles non strings or non existing keys
         self.assertEquals(BlogPost.objects.filter(info__details__test__exact=5).count(), 0)
         self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0)
+
+        post = BlogPost.objects.create(info={'title': 'original'})
+        post.info.update({'title': 'updated'})
+        post.save()
+        post.reload()
+        self.assertEquals('updated', post.info['title'])
+
         BlogPost.drop_collection()
 
     def test_dictfield_strict(self):
@@ -727,6 +912,48 @@ class FieldTest(unittest.TestCase):
 
         Extensible.drop_collection()
 
+    def test_embedded_mapfield_db_field(self):
+
+        class Embedded(EmbeddedDocument):
+            number = IntField(default=0, db_field='i')
+
+        class Test(Document):
+            my_map = MapField(field=EmbeddedDocumentField(Embedded), db_field='x')
+
+        Test.drop_collection()
+
+        test = Test()
+        test.my_map['DICTIONARY_KEY'] = Embedded(number=1)
+        test.save()
+
+        Test.objects.update_one(inc__my_map__DICTIONARY_KEY__number=1)
+
+        test = Test.objects.get()
+        self.assertEqual(test.my_map['DICTIONARY_KEY'].number, 2)
+        doc = self.db.test.find_one()
+        self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2)
+
+    def test_embedded_db_field(self):
+
+        class Embedded(EmbeddedDocument):
+            number = IntField(default=0, db_field='i')
+
+        class Test(Document):
+            embedded = EmbeddedDocumentField(Embedded, db_field='x')
+
+        Test.drop_collection()
+
+        test = Test()
+        test.embedded = Embedded(number=1)
+        test.save()
+
+        Test.objects.update_one(inc__embedded__number=1)
+
+        test = Test.objects.get()
+        self.assertEqual(test.embedded.number, 2)
+        doc = self.db.test.find_one()
+        self.assertEqual(doc['x']['i'], 2)
+
     def test_embedded_document_validation(self):
         """Ensure that invalid embedded documents cannot be assigned to
         embedded document fields.
@@ -942,15 +1169,29 @@ class FieldTest(unittest.TestCase):
         class Company(Document):
             name = StringField()
 
+        Product.drop_collection()
+        Company.drop_collection()
+
         ten_gen = Company(name='10gen')
         ten_gen.save()
         mongodb = Product(name='MongoDB', company=ten_gen)
         mongodb.save()
 
+        me = Product(name='MongoEngine')
+        me.save()
+
         obj = Product.objects(company=ten_gen).first()
         self.assertEqual(obj, mongodb)
         self.assertEqual(obj.company, ten_gen)
 
+        obj = Product.objects(company=None).first()
+        self.assertEqual(obj, me)
+
+        obj, created = Product.objects.get_or_create(company=None)
+
+        self.assertEqual(created, False)
+        self.assertEqual(obj, me)
+
     def test_reference_query_conversion(self):
         """Ensure that ReferenceFields can be queried using objects and values
         of the type of the primary key of the referenced object.
@@ -1062,7 +1303,6 @@ class FieldTest(unittest.TestCase):
         Post.drop_collection()
         User.drop_collection()
 
-
     def test_generic_reference_document_not_registered(self):
         """Ensure dereferencing out of the document registry throws a
         `NotRegistered` error.
@@ -1089,7 +1329,7 @@ class FieldTest(unittest.TestCase):
         user = User.objects.first()
         try:
             user.bookmarks
-            raise AssertionError, "Link was removed from the registry"
+            raise AssertionError("Link was removed from the registry")
         except NotRegistered:
             pass
 
@@ -1108,6 +1348,74 @@ class FieldTest(unittest.TestCase):
         self.assertEquals(repr(Person.objects(city=None)),
                             "[<Person: Person object>]")
 
+
+    def test_generic_reference_choices(self):
+        """Ensure that a GenericReferenceField can handle choices
+        """
+        class Link(Document):
+            title = StringField()
+
+        class Post(Document):
+            title = StringField()
+
+        class Bookmark(Document):
+            bookmark_object = GenericReferenceField(choices=(Post,))
+
+        Link.drop_collection()
+        Post.drop_collection()
+        Bookmark.drop_collection()
+
+        link_1 = Link(title="Pitchfork")
+        link_1.save()
+
+        post_1 = Post(title="Behind the Scenes of the Pavement Reunion")
+        post_1.save()
+
+        bm = Bookmark(bookmark_object=link_1)
+        self.assertRaises(ValidationError, bm.validate)
+
+        bm = Bookmark(bookmark_object=post_1)
+        bm.save()
+
+        bm = Bookmark.objects.first()
+        self.assertEqual(bm.bookmark_object, post_1)
+
+    def test_generic_reference_list_choices(self):
+        """Ensure that a ListField properly dereferences generic references and
+        respects choices.
+        """
+        class Link(Document):
+            title = StringField()
+
+        class Post(Document):
+            title = StringField()
+
+        class User(Document):
+            bookmarks = ListField(GenericReferenceField(choices=(Post,)))
+
+        Link.drop_collection()
+        Post.drop_collection()
+        User.drop_collection()
+
+        link_1 = Link(title="Pitchfork")
+        link_1.save()
+
+        post_1 = Post(title="Behind the Scenes of the Pavement Reunion")
+        post_1.save()
+
+        user = User(bookmarks=[link_1])
+        self.assertRaises(ValidationError, user.validate)
+
+        user = User(bookmarks=[post_1])
+        user.save()
+
+        user = User.objects.first()
+        self.assertEqual(user.bookmarks, [post_1])
+
+        Link.drop_collection()
+        Post.drop_collection()
+        User.drop_collection()
+
     def test_binary_fields(self):
         """Ensure that binary fields can be stored and retrieved.
         """
@@ -1213,6 +1521,53 @@ class FieldTest(unittest.TestCase):
 
         Shirt.drop_collection()
 
+    def test_simple_choices_validation(self):
+        """Ensure that value is in a container of allowed values.
+        """
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL'))
+
+        Shirt.drop_collection()
+
+        shirt = Shirt()
+        shirt.validate()
+
+        shirt.size = "S"
+        shirt.validate()
+
+        shirt.size = "XS"
+        self.assertRaises(ValidationError, shirt.validate)
+
+        Shirt.drop_collection()
+
+    def test_simple_choices_get_field_display(self):
+        """Test dynamic helper for returning the display value of a choices field.
+        """
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL'))
+            style = StringField(max_length=3, choices=('Small', 'Baggy', 'wide'), default='Small')
+
+        Shirt.drop_collection()
+
+        shirt = Shirt()
+
+        self.assertEqual(shirt.get_size_display(), None)
+        self.assertEqual(shirt.get_style_display(), 'Small')
+
+        shirt.size = "XXL"
+        shirt.style = "Baggy"
+        self.assertEqual(shirt.get_size_display(), 'XXL')
+        self.assertEqual(shirt.get_style_display(), 'Baggy')
+
+        # Set as Z - an invalid choice
+        shirt.size = "Z"
+        shirt.style = "Z"
+        self.assertEqual(shirt.get_size_display(), 'Z')
+        self.assertEqual(shirt.get_style_display(), 'Z')
+        self.assertRaises(ValidationError, shirt.validate)
+
+        Shirt.drop_collection()
+
     def test_file_fields(self):
         """Ensure that file fields can be written to and their data retrieved
         """
@@ -1242,6 +1597,21 @@ class FieldTest(unittest.TestCase):
         self.assertEquals(result.file.read(), text)
         self.assertEquals(result.file.content_type, content_type)
         result.file.delete() # Remove file from GridFS
+        PutFile.objects.delete()
+
+        # Ensure file-like objects are stored
+        putfile = PutFile()
+        putstring = StringIO.StringIO()
+        putstring.write(text)
+        putstring.seek(0)
+        putfile.file.put(putstring, content_type=content_type)
+        putfile.save()
+        putfile.validate()
+        result = PutFile.objects.first()
+        self.assertTrue(putfile == result)
+        self.assertEquals(result.file.read(), text)
+        self.assertEquals(result.file.content_type, content_type)
+        result.file.delete()
 
         streamfile = StreamFile()
         streamfile.file.new_file(content_type=content_type)
@@ -1289,7 +1659,50 @@ class FieldTest(unittest.TestCase):
         # Make sure FileField is optional and not required
         class DemoFile(Document):
             file = FileField()
-        d = DemoFile.objects.create()
+        DemoFile.objects.create()
+
+
+    def test_file_field_no_default(self):
+
+        class GridDocument(Document):
+            the_file = FileField()
+
+        GridDocument.drop_collection()
+
+        with tempfile.TemporaryFile() as f:
+            f.write("Hello World!")
+            f.flush()
+
+            # Test without default
+            doc_a = GridDocument()
+            doc_a.save()
+
+
+            doc_b = GridDocument.objects.with_id(doc_a.id)
+            doc_b.the_file.replace(f, filename='doc_b')
+            doc_b.save()
+            self.assertNotEquals(doc_b.the_file.grid_id, None)
+
+            # Test it matches
+            doc_c = GridDocument.objects.with_id(doc_b.id)
+            self.assertEquals(doc_b.the_file.grid_id, doc_c.the_file.grid_id)
+
+            # Test with default
+            doc_d = GridDocument(the_file='')
+            doc_d.save()
+
+            doc_e = GridDocument.objects.with_id(doc_d.id)
+            self.assertEquals(doc_d.the_file.grid_id, doc_e.the_file.grid_id)
+
+            doc_e.the_file.replace(f, filename='doc_e')
+            doc_e.save()
+
+            doc_f = GridDocument.objects.with_id(doc_e.id)
+            self.assertEquals(doc_e.the_file.grid_id, doc_f.the_file.grid_id)
+
+        db = GridDocument._get_db()
+        grid_fs = gridfs.GridFS(db)
+        self.assertEquals(['doc_b', 'doc_e'], grid_fs.list())
 
     def test_file_uniqueness(self):
         """Ensure that each instance of a FileField is unique
@@ -1328,6 +1741,95 @@ class FieldTest(unittest.TestCase):
 
         TestFile.drop_collection()
 
+    def test_image_field(self):
+
+        class TestImage(Document):
+            image = ImageField()
+
+        TestImage.drop_collection()
+
+        t = TestImage()
+        t.image.put(open(TEST_IMAGE_PATH, 'r'))
+        t.save()
+
+        t = TestImage.objects.first()
+
+        self.assertEquals(t.image.format, 'PNG')
+
+        w, h = t.image.size
+        self.assertEquals(w, 371)
+        self.assertEquals(h, 76)
+
+        t.image.delete()
+
+    def test_image_field_resize(self):
+
+        class TestImage(Document):
+            image = ImageField(size=(185, 37))
+
+        TestImage.drop_collection()
+
+        t = TestImage()
+        t.image.put(open(TEST_IMAGE_PATH, 'r'))
+        t.save()
+
+        t = TestImage.objects.first()
+
+        self.assertEquals(t.image.format, 'PNG')
+        w, h = t.image.size
+
+        self.assertEquals(w, 185)
+        self.assertEquals(h, 37)
+
+        t.image.delete()
+
+    def test_image_field_thumbnail(self):
+
+        class TestImage(Document):
+            image = ImageField(thumbnail_size=(92, 18))
+
+        TestImage.drop_collection()
+
+        t = TestImage()
+        t.image.put(open(TEST_IMAGE_PATH, 'r'))
+        t.save()
+
+        t = TestImage.objects.first()
+
+        self.assertEquals(t.image.thumbnail.format, 'PNG')
+        self.assertEquals(t.image.thumbnail.width, 92)
+        self.assertEquals(t.image.thumbnail.height, 18)
+
+        t.image.delete()
+
+
+    def test_file_multidb(self):
+        register_connection('testfiles', 'testfiles')
+        class TestFile(Document):
+            name = StringField()
+            file = FileField(db_alias="testfiles",
+                             collection_name="macumba")
+
+        TestFile.drop_collection()
+
+        # delete old filesystem
+        get_db("testfiles").macumba.files.drop()
+        get_db("testfiles").macumba.chunks.drop()
+
+        # First instance
+        testfile = TestFile()
+        testfile.name = "Hello, World!"
+        testfile.file.put('Hello, World!',
+                          name="hello.txt")
+        testfile.save()
+
+        data = get_db("testfiles").macumba.files.find_one()
+        self.assertEquals(data.get('name'), 'hello.txt')
+
+        testfile = TestFile.objects.first()
+        self.assertEquals(testfile.file.read(),
+                          'Hello, World!')
+
     def test_geo_indexes(self):
         """Ensure that indexes are created automatically for GeoPointFields.
         """
@@ -1488,7 +1990,6 @@ class FieldTest(unittest.TestCase):
         c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'})
         self.assertEqual(c['next'], 10)
 
-
     def test_generic_embedded_document(self):
         class Car(EmbeddedDocument):
             name = StringField()
@@ -1501,6 +2002,8 @@ class FieldTest(unittest.TestCase):
             name = StringField()
             like = GenericEmbeddedDocumentField()
 
+        Person.drop_collection()
+
         person = Person(name='Test User')
         person.like = Car(name='Fiat')
         person.save()
@@ -1514,5 +2017,103 @@ class FieldTest(unittest.TestCase):
         person = Person.objects.first()
         self.assertTrue(isinstance(person.like, Dish))
 
+    def test_generic_embedded_document_choices(self):
+        """Ensure you can limit GenericEmbeddedDocument choices
+        """
+        class Car(EmbeddedDocument):
+            name = StringField()
+
+        class Dish(EmbeddedDocument):
+            food = StringField(required=True)
+            number = IntField()
+
+        class Person(Document):
+            name = StringField()
+            like = GenericEmbeddedDocumentField(choices=(Dish,))
+
+        Person.drop_collection()
+
+        person = Person(name='Test User')
+        person.like = Car(name='Fiat')
+        self.assertRaises(ValidationError, person.validate)
+
+        person.like = Dish(food="arroz", number=15)
+        person.save()
+
+        person = Person.objects.first()
+        self.assertTrue(isinstance(person.like, Dish))
+
+    def test_generic_list_embedded_document_choices(self):
+        """Ensure you can limit GenericEmbeddedDocument choices inside a list
+        field
+        """
+        class Car(EmbeddedDocument):
+            name = StringField()
+
+        class Dish(EmbeddedDocument):
+            food = StringField(required=True)
+            number = IntField()
+
+        class Person(Document):
+            name = StringField()
+            likes = ListField(GenericEmbeddedDocumentField(choices=(Dish,)))
+
+        Person.drop_collection()
+
+        person = Person(name='Test User')
+        person.likes = [Car(name='Fiat')]
+        self.assertRaises(ValidationError, person.validate)
+
+        person.likes = [Dish(food="arroz", number=15)]
+        person.save()
+
+        person = Person.objects.first()
+        self.assertTrue(isinstance(person.likes[0], Dish))
+
+    def test_recursive_validation(self):
+        """Ensure that a validation result to_dict is available.
+        """
+        class Author(EmbeddedDocument):
+            name = StringField(required=True)
+
+        class Comment(EmbeddedDocument):
+            author = EmbeddedDocumentField(Author, required=True)
+            content = StringField(required=True)
+
+        class Post(Document):
+            title = StringField(required=True)
+            comments = ListField(EmbeddedDocumentField(Comment))
+
+        bob = Author(name='Bob')
+        post = Post(title='hello world')
+        post.comments.append(Comment(content='hello', author=bob))
+        post.comments.append(Comment(author=bob))
+
+        try:
+            post.validate()
+        except ValidationError, error:
+            pass
+
+        # ValidationError.errors property
+        self.assertTrue(hasattr(error, 'errors'))
+        self.assertTrue(isinstance(error.errors, dict))
+        self.assertTrue('comments' in error.errors)
+        self.assertTrue(1 in error.errors['comments'])
+        self.assertTrue(isinstance(error.errors['comments'][1]['content'],
+                        ValidationError))
+
+        # ValidationError.schema property
+        error_dict = error.to_dict()
+        self.assertTrue(isinstance(error_dict, dict))
+        self.assertTrue('comments' in error_dict)
+        self.assertTrue(1 in error_dict['comments'])
+        self.assertTrue('content' in error_dict['comments'][1])
+        self.assertEquals(error_dict['comments'][1]['content'],
+                          u'Field is required ("content")')
+
+        post.comments[1].content = 'here we go'
+        post.validate()
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_queryset.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
 import unittest
 import pymongo
+from bson import ObjectId
 from datetime import datetime, timedelta
 
 from mongoengine.queryset import (QuerySet, QuerySetManager,
                                   MultipleObjectsReturned, DoesNotExist,
                                   QueryFieldList)
 from mongoengine import *
-from mongoengine.connection import _get_connection
+from mongoengine.connection import get_connection
 from mongoengine.tests import query_counter
 
 
@@ -19,6 +20,7 @@ class QuerySetTest(unittest.TestCase):
         class Person(Document):
             name = StringField()
             age = IntField()
+            meta = {'allow_inheritance': True}
         self.Person = Person
 
     def test_initialisation(self):
@@ -59,8 +61,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(len(people), 2)
         results = list(people)
         self.assertTrue(isinstance(results[0], self.Person))
-        self.assertTrue(isinstance(results[0].id, (pymongo.objectid.ObjectId,
-                                                   str, unicode)))
+        self.assertTrue(isinstance(results[0].id, (ObjectId, str, unicode)))
         self.assertEqual(results[0].name, "User A")
         self.assertEqual(results[0].age, 20)
         self.assertEqual(results[1].name, "User B")
@@ -110,6 +111,16 @@ class QuerySetTest(unittest.TestCase):
         people = list(self.Person.objects[80000:80001])
         self.assertEqual(len(people), 0)
 
+        # Test larger slice __repr__
+        self.Person.objects.delete()
+        for i in xrange(55):
+            self.Person(name='A%s' % i, age=i).save()
+
+        self.assertEqual(len(self.Person.objects), 55)
+        self.assertEqual("Person object", "%s" % self.Person.objects[0])
+        self.assertEqual("[<Person: Person object>, <Person: Person object>]",  "%s" % self.Person.objects[1:3])
+        self.assertEqual("[<Person: Person object>, <Person: Person object>]",  "%s" % self.Person.objects[51:53])
+
     def test_find_one(self):
         """Ensure that a query using find_one returns a valid result.
         """
@@ -144,6 +155,8 @@ class QuerySetTest(unittest.TestCase):
         person = self.Person.objects.with_id(person1.id)
         self.assertEqual(person.name, "User A")
 
+        self.assertRaises(InvalidQueryError, self.Person.objects(name="User A").with_id, person1.id)
+
     def test_find_only_one(self):
         """Ensure that a query using ``get`` returns at most one result.
         """
@@ -316,11 +329,11 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost(title="ABC", comments=[c1, c2]).save()
 
-        BlogPost.objects(comments__by="joe").update(inc__comments__S__votes=1)
+        BlogPost.objects(comments__by="jane").update(inc__comments__S__votes=1)
 
         post = BlogPost.objects.first()
-        self.assertEquals(post.comments[0].by, 'joe')
-        self.assertEquals(post.comments[0].votes, 4)
+        self.assertEquals(post.comments[1].by, 'jane')
+        self.assertEquals(post.comments[1].votes, 8)
 
         # Currently the $ operator only applies to the first matched item in
         # the query
@@ -368,6 +381,34 @@ class QuerySetTest(unittest.TestCase):
         self.assertRaises(OperationError, update_nested)
         Simple.drop_collection()
 
+    def test_update_using_positional_operator_embedded_document(self):
+        """Ensure that the embedded documents can be updated using the positional
+        operator."""
+
+        class Vote(EmbeddedDocument):
+            score = IntField()
+
+        class Comment(EmbeddedDocument):
+            by = StringField()
+            votes = EmbeddedDocumentField(Vote)
+
+        class BlogPost(Document):
+            title = StringField()
+            comments = ListField(EmbeddedDocumentField(Comment))
+
+        BlogPost.drop_collection()
+
+        c1 = Comment(by="joe", votes=Vote(score=3))
+        c2 = Comment(by="jane", votes=Vote(score=7))
+
+        BlogPost(title="ABC", comments=[c1, c2]).save()
+
+        BlogPost.objects(comments__by="joe").update(set__comments__S__votes=Vote(score=4))
+
+        post = BlogPost.objects.first()
+        self.assertEquals(post.comments[0].by, 'joe')
+        self.assertEquals(post.comments[0].votes.score, 4)
+
     def test_mapfield_update(self):
         """Ensure that the MapField can be updated."""
         class Member(EmbeddedDocument):
@@ -439,7 +480,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(person.name, "User C")
 
     def test_bulk_insert(self):
-        """Ensure that query by array position works.
+        """Ensure that bulk insert works
         """
 
         class Comment(EmbeddedDocument):
@@ -449,12 +490,15 @@ class QuerySetTest(unittest.TestCase):
             comments = ListField(EmbeddedDocumentField(Comment))
 
         class Blog(Document):
-            title = StringField()
+            title = StringField(unique=True)
             tags = ListField(StringField())
             posts = ListField(EmbeddedDocumentField(Post))
 
         Blog.drop_collection()
 
+        # Recreates the collection
+        self.assertEqual(0, Blog.objects.count())
+
         with query_counter() as q:
             self.assertEqual(q, 0)
 
@@ -468,10 +512,10 @@ class QuerySetTest(unittest.TestCase):
                 blogs.append(Blog(title="post %s" % i, posts=[post1, post2]))
 
             Blog.objects.insert(blogs, load_bulk=False)
-            self.assertEqual(q, 2) # 1 for the inital connection and 1 for the insert
+            self.assertEqual(q, 1) # 1 for the insert
 
             Blog.objects.insert(blogs)
-            self.assertEqual(q, 4) # 1 for insert, and 1 for in bulk
+            self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total)
 
         Blog.drop_collection()
 
@@ -519,6 +563,23 @@ class QuerySetTest(unittest.TestCase):
         obj_id = Blog.objects.insert(blog1, load_bulk=False)
         self.assertEquals(obj_id.__class__.__name__, 'ObjectId')
 
+        Blog.drop_collection()
+        post3 = Post(comments=[comment1, comment1])
+        blog1 = Blog(title="foo", posts=[post1, post2])
+        blog2 = Blog(title="bar", posts=[post2, post3])
+        blog3 = Blog(title="baz", posts=[post1, post2])
+        Blog.objects.insert([blog1, blog2])
+
+        def throw_operation_error_not_unique():
+            Blog.objects.insert([blog2, blog3], safe=True)
+
+        self.assertRaises(OperationError, throw_operation_error_not_unique)
+        self.assertEqual(Blog.objects.count(), 2)
+
+        Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True})
+        self.assertEqual(Blog.objects.count(), 3)
+
+
     def test_slave_okay(self):
         """Ensures that a query can take slave_okay syntax
         """
@@ -567,7 +628,13 @@ class QuerySetTest(unittest.TestCase):
         people1 = [person for person in queryset]
         people2 = [person for person in queryset]
 
+        # Check that it still works even if iteration is interrupted.
+        for person in queryset:
+            break
+        people3 = [person for person in queryset]
+
         self.assertEqual(people1, people2)
+        self.assertEqual(people1, people3)
 
     def test_repr_iteration(self):
         """Ensure that QuerySet __repr__ can handle loops
@@ -1277,6 +1344,37 @@ class QuerySetTest(unittest.TestCase):
         self.Person.objects(name='Test User').delete()
         self.assertEqual(1, BlogPost.objects.count())
 
+    def test_reverse_delete_rule_cascade_self_referencing(self):
+        """Ensure self-referencing CASCADE deletes do not result in infinite loop
+        """
+        class Category(Document):
+            name = StringField()
+            parent = ReferenceField('self', reverse_delete_rule=CASCADE)
+
+        num_children = 3
+        base = Category(name='Root')
+        base.save()
+
+        # Create a simple parent-child tree
+        for i in range(num_children):
+            child_name = 'Child-%i' % i
+            child = Category(name=child_name, parent=base)
+            child.save()
+
+            for i in range(num_children):
+                child_child_name = 'Child-Child-%i' % i
+                child_child = Category(name=child_child_name, parent=child)
+                child_child.save()
+
+        tree_size = 1 + num_children + (num_children * num_children)
+        self.assertEquals(tree_size, Category.objects.count())
+        self.assertEquals(num_children, Category.objects(parent=base).count())
+
+        # The delete should effectively wipe out the Category collection
+        # without resulting in infinite parent-child cascade recursion
+        base.delete()
+        self.assertEquals(0, Category.objects.count())
+
     def test_reverse_delete_rule_nullify(self):
         """Ensure nullification of references to deleted documents.
         """
@@ -1321,6 +1419,36 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(OperationError, self.Person.objects.delete)
 
+    def test_reverse_delete_rule_pull(self):
+        """Ensure pulling of references to deleted documents.
+        """
+        class BlogPost(Document):
+            content = StringField()
+            authors = ListField(ReferenceField(self.Person,
+                reverse_delete_rule=PULL))
+
+        BlogPost.drop_collection()
+        self.Person.drop_collection()
+
+        me = self.Person(name='Test User')
+        me.save()
+
+        someoneelse = self.Person(name='Some-one Else')
+        someoneelse.save()
+
+        post = BlogPost(content='Watching TV', authors=[me, someoneelse])
+        post.save()
+
+        another = BlogPost(content='Chilling Out', authors=[someoneelse])
+        another.save()
+
+        someoneelse.delete()
+        post.reload()
+        another.reload()
+
+        self.assertEqual(post.authors, [me])
+        self.assertEqual(another.authors, [])
+
     def test_update(self):
         """Ensure that atomic updates work properly.
         """
@@ -1371,20 +1499,68 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-    def test_update_pull(self):
+    def test_update_push_and_pull(self):
         """Ensure that the 'pull' update operation works correctly.
         """
         class BlogPost(Document):
             slug = StringField()
             tags = ListField(StringField())
 
-        post = BlogPost(slug="test", tags=['code', 'mongodb', 'code'])
+        BlogPost.drop_collection()
+
+        post = BlogPost(slug="test")
         post.save()
 
+        BlogPost.objects.filter(id=post.id).update(push__tags="code")
+        post.reload()
+        self.assertEqual(post.tags, ["code"])
+
+        BlogPost.objects.filter(id=post.id).update(push_all__tags=["mongodb", "code"])
+        post.reload()
+        self.assertEqual(post.tags, ["code", "mongodb", "code"])
+
         BlogPost.objects(slug="test").update(pull__tags="code")
         post.reload()
-        self.assertTrue('code' not in post.tags)
-        self.assertEqual(len(post.tags), 1)
+        self.assertEqual(post.tags, ["mongodb"])
+
+
+        BlogPost.objects(slug="test").update(pull_all__tags=["mongodb", "code"])
+        post.reload()
+        self.assertEqual(post.tags, [])
+
+        BlogPost.objects(slug="test").update(__raw__={"$addToSet": {"tags": {"$each": ["code", "mongodb", "code"]}}})
+        post.reload()
+        self.assertEqual(post.tags, ["code", "mongodb"])
+
+    def test_pull_nested(self):
+
+        class User(Document):
+            name = StringField()
+
+        class Collaborator(EmbeddedDocument):
+            user = StringField()
+
+            def __unicode__(self):
+                return '%s' % self.user
+
+        class Site(Document):
+            name = StringField(max_length=75, unique=True, required=True)
+            collaborators = ListField(EmbeddedDocumentField(Collaborator))
+
+
+        Site.drop_collection()
+
+        c = Collaborator(user='Esteban')
+        s = Site(name="test", collaborators=[c])
+        s.save()
+
+        Site.objects(id=s.id).update_one(pull__collaborators__user='Esteban')
+        self.assertEqual(Site.objects.first().collaborators, [])
+
+        def pull_all():
+            Site.objects(id=s.id).update_one(pull_all__collaborators__user=['Ross'])
+
+        self.assertRaises(InvalidQueryError, pull_all)
 
     def test_update_one_pop_generic_reference(self):
 
@@ -1449,6 +1625,37 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+
+    def test_set_list_embedded_documents(self):
+
+        class Author(EmbeddedDocument):
+            name = StringField()
+
+        class Message(Document):
+            title = StringField()
+            authors = ListField(EmbeddedDocumentField('Author'))
+
+        Message.drop_collection()
+
+        message = Message(title="hello", authors=[Author(name="Harry")])
+        message.save()
+
+        Message.objects(authors__name="Harry").update_one(
+            set__authors__S=Author(name="Ross"))
+
+        message = message.reload()
+        self.assertEquals(message.authors[0].name, "Ross")
+
+        Message.objects(authors__name="Ross").update_one(
+            set__authors=[Author(name="Harry"),
+                          Author(name="Ross"),
+                          Author(name="Adam")])
+
+        message = message.reload()
+        self.assertEquals(message.authors[0].name, "Harry")
+        self.assertEquals(message.authors[1].name, "Ross")
+        self.assertEquals(message.authors[2].name, "Adam")
+
     def test_order_by(self):
         """Ensure that QuerySets may be ordered.
         """
@@ -1840,6 +2047,35 @@ class QuerySetTest(unittest.TestCase):
         freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True)
         self.assertEquals(freq, {'CRB': 0.5, None: 0.5})
 
+    def test_item_frequencies_with_null_embedded(self):
+        class Data(EmbeddedDocument):
+            name = StringField()
+
+        class Extra(EmbeddedDocument):
+            tag = StringField()
+
+        class Person(Document):
+            data = EmbeddedDocumentField(Data, required=True)
+            extra = EmbeddedDocumentField(Extra)
+
+
+        Person.drop_collection()
+
+        p = Person()
+        p.data = Data(name="Wilson Jr")
+        p.save()
+
+        p = Person()
+        p.data = Data(name="Wesley")
+        p.extra = Extra(tag="friend")
+        p.save()
+
+        ot = Person.objects.item_frequencies('extra.tag', map_reduce=False)
+        self.assertEquals(ot, {None: 1.0, u'friend': 1.0})
+
+        ot = Person.objects.item_frequencies('extra.tag', map_reduce=True)
+        self.assertEquals(ot, {None: 1.0, u'friend': 1.0})
+
     def test_average(self):
         """Ensure that field can be averaged correctly.
         """
@@ -1882,6 +2118,24 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(set(self.Person.objects(age=30).distinct('name')),
                          set(['Mr Orange', 'Mr Pink']))
 
+    def test_distinct_handles_references(self):
+        class Foo(Document):
+            bar = ReferenceField("Bar")
+
+        class Bar(Document):
+            text = StringField()
+
+        Bar.drop_collection()
+        Foo.drop_collection()
+
+        bar = Bar(text="hi")
+        bar.save()
+
+        foo = Foo(bar=bar)
+        foo.save()
+
+        self.assertEquals(Foo.objects.distinct("bar"), [bar])
+
     def test_custom_manager(self):
         """Ensure that custom QuerySetManager instances work as expected.
         """
@@ -2200,7 +2454,7 @@ class QuerySetTest(unittest.TestCase):
 
         # check that polygon works for users who have a server >= 1.9
         server_version = tuple(
-            _get_connection().server_info()['version'].split('.')
+            get_connection().server_info()['version'].split('.')
         )
         required_version = tuple("1.9.0".split("."))
         if server_version >= required_version:
@@ -2569,6 +2823,278 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(TypeError, invalid_where)
 
+    def test_scalar(self):
+
+        class Organization(Document):
+            id = ObjectIdField('_id')
+            name = StringField()
+
+        class User(Document):
+            id = ObjectIdField('_id')
+            name = StringField()
+            organization = ObjectIdField()
+
+        User.drop_collection()
+        Organization.drop_collection()
+
+        whitehouse = Organization(name="White House")
+        whitehouse.save()
+        User(name="Bob Dole", organization=whitehouse.id).save()
+
+        # Efficient way to get all unique organization names for a given
+        # set of users (Pretend this has additional filtering.)
+        user_orgs = set(User.objects.scalar('organization'))
+        orgs = Organization.objects(id__in=user_orgs).scalar('name')
+        self.assertEqual(list(orgs), ['White House'])
+
+        # Efficient for generating listings, too.
+        orgs = Organization.objects.scalar('name').in_bulk(list(user_orgs))
+        user_map = User.objects.scalar('name', 'organization')
+        user_listing = [(user, orgs[org]) for user, org in user_map]
+        self.assertEqual([("Bob Dole", "White House")], user_listing)
+
+    def test_scalar_simple(self):
+        class TestDoc(Document):
+            x = IntField()
+            y = BooleanField()
+
+        TestDoc.drop_collection()
+
+        TestDoc(x=10, y=True).save()
+        TestDoc(x=20, y=False).save()
+        TestDoc(x=30, y=True).save()
+
+        plist = list(TestDoc.objects.scalar('x', 'y'))
+
+        self.assertEqual(len(plist), 3)
+        self.assertEqual(plist[0], (10, True))
+        self.assertEqual(plist[1], (20, False))
+        self.assertEqual(plist[2], (30, True))
+
+        class UserDoc(Document):
+            name = StringField()
+            age = IntField()
+
+        UserDoc.drop_collection()
+
+        UserDoc(name="Wilson Jr", age=19).save()
+        UserDoc(name="Wilson", age=43).save()
+        UserDoc(name="Eliana", age=37).save()
+        UserDoc(name="Tayza", age=15).save()
+
+        ulist = list(UserDoc.objects.scalar('name', 'age'))
+
+        self.assertEqual(ulist, [
+                (u'Wilson Jr', 19),
+                (u'Wilson', 43),
+                (u'Eliana', 37),
+                (u'Tayza', 15)])
+
+        ulist = list(UserDoc.objects.scalar('name').order_by('age'))
+
+        self.assertEqual(ulist, [
+                (u'Tayza'),
+                (u'Wilson Jr'),
+                (u'Eliana'),
+                (u'Wilson')])
+
+    def test_scalar_embedded(self):
+        class Profile(EmbeddedDocument):
+            name = StringField()
+            age = IntField()
+
+        class Locale(EmbeddedDocument):
+            city = StringField()
+            country = StringField()
+
+        class Person(Document):
+            profile = EmbeddedDocumentField(Profile)
+            locale = EmbeddedDocumentField(Locale)
+
+        Person.drop_collection()
+
+        Person(profile=Profile(name="Wilson Jr", age=19),
+               locale=Locale(city="Corumba-GO", country="Brazil")).save()
+
+        Person(profile=Profile(name="Gabriel Falcao", age=23),
+               locale=Locale(city="New York", country="USA")).save()
+
+        Person(profile=Profile(name="Lincoln de souza", age=28),
+               locale=Locale(city="Belo Horizonte", country="Brazil")).save()
+
+        Person(profile=Profile(name="Walter cruz", age=30),
+               locale=Locale(city="Brasilia", country="Brazil")).save()
+
+        self.assertEqual(
+            list(Person.objects.order_by('profile__age').scalar('profile__name')),
+            [u'Wilson Jr', u'Gabriel Falcao', u'Lincoln de souza', u'Walter cruz'])
+
+        ulist = list(Person.objects.order_by('locale.city')
+                     .scalar('profile__name', 'profile__age', 'locale__city'))
+        self.assertEqual(ulist,
+                         [(u'Lincoln de souza', 28, u'Belo Horizonte'),
+                          (u'Walter cruz', 30, u'Brasilia'),
+                          (u'Wilson Jr', 19, u'Corumba-GO'),
+                          (u'Gabriel Falcao', 23, u'New York')])
+
+    def test_scalar_decimal(self):
+        from decimal import Decimal
+        class Person(Document):
+            name = StringField()
+            rating = DecimalField()
+
+        Person.drop_collection()
+        Person(name="Wilson Jr", rating=Decimal('1.0')).save()
+
+        ulist = list(Person.objects.scalar('name', 'rating'))
+        self.assertEqual(ulist, [(u'Wilson Jr', Decimal('1.0'))])
+
+
+    def test_scalar_reference_field(self):
+        class State(Document):
+            name = StringField()
+
+        class Person(Document):
+            name = StringField()
+            state = ReferenceField(State)
+
+        State.drop_collection()
+        Person.drop_collection()
+
+        s1 = State(name="Goias")
+        s1.save()
+
+        Person(name="Wilson JR", state=s1).save()
+
+        plist = list(Person.objects.scalar('name', 'state'))
+        self.assertEqual(plist, [(u'Wilson JR', s1)])
+
+    def test_scalar_generic_reference_field(self):
+        class State(Document):
+            name = StringField()
+
+        class Person(Document):
+            name = StringField()
+            state = GenericReferenceField()
+
+        State.drop_collection()
+        Person.drop_collection()
+
+        s1 = State(name="Goias")
+        s1.save()
+
+        Person(name="Wilson JR", state=s1).save()
+
+        plist = list(Person.objects.scalar('name', 'state'))
+        self.assertEqual(plist, [(u'Wilson JR', s1)])
+
+    def test_scalar_db_field(self):
+
+        class TestDoc(Document):
+            x = IntField()
+            y = BooleanField()
+
+        TestDoc.drop_collection()
+
+        TestDoc(x=10, y=True).save()
+        TestDoc(x=20, y=False).save()
+        TestDoc(x=30, y=True).save()
+
+        plist = list(TestDoc.objects.scalar('x', 'y'))
+        self.assertEqual(len(plist), 3)
+        self.assertEqual(plist[0], (10, True))
+        self.assertEqual(plist[1], (20, False))
+        self.assertEqual(plist[2], (30, True))
+
+    def test_scalar_primary_key(self):
+
+        class SettingValue(Document):
+            key = StringField(primary_key=True)
+            value = StringField()
+
+        SettingValue.drop_collection()
+        s = SettingValue(key="test", value="test value")
+        s.save()
+
+        val = SettingValue.objects.scalar('key', 'value')
+        self.assertEqual(list(val), [('test', 'test value')])
+
+    def test_scalar_cursor_behaviour(self):
+        """Ensure that a query returns a valid set of results.
+        """
+        person1 = self.Person(name="User A", age=20)
+        person1.save()
+        person2 = self.Person(name="User B", age=30)
+        person2.save()
+
+        # Find all people in the collection
+        people = self.Person.objects.scalar('name')
+        self.assertEqual(len(people), 2)
+        results = list(people)
+        self.assertEqual(results[0], "User A")
+        self.assertEqual(results[1], "User B")
+
+        # Use a query to filter the people found to just person1
+        people = self.Person.objects(age=20).scalar('name')
+        self.assertEqual(len(people), 1)
+        person = people.next()
+        self.assertEqual(person, "User A")
+
+        # Test limit
+        people = list(self.Person.objects.limit(1).scalar('name'))
+        self.assertEqual(len(people), 1)
+        self.assertEqual(people[0], 'User A')
+
+        # Test skip
+        people = list(self.Person.objects.skip(1).scalar('name'))
+        self.assertEqual(len(people), 1)
+        self.assertEqual(people[0], 'User B')
+
+        person3 = self.Person(name="User C", age=40)
+        person3.save()
+
+        # Test slice limit
+        people = list(self.Person.objects[:2].scalar('name'))
+        self.assertEqual(len(people), 2)
+        self.assertEqual(people[0], 'User A')
+        self.assertEqual(people[1], 'User B')
+
+        # Test slice skip
+        people = list(self.Person.objects[1:].scalar('name'))
+        self.assertEqual(len(people), 2)
+        self.assertEqual(people[0], 'User B')
+        self.assertEqual(people[1], 'User C')
+
+        # Test slice limit and skip
+        people = list(self.Person.objects[1:2].scalar('name'))
+        self.assertEqual(len(people), 1)
+        self.assertEqual(people[0], 'User B')
+
+        people = list(self.Person.objects[1:1].scalar('name'))
+        self.assertEqual(len(people), 0)
+
+        # Test slice out of range
+        people = list(self.Person.objects.scalar('name')[80000:80001])
+        self.assertEqual(len(people), 0)
+
+        # Test larger slice __repr__
+        self.Person.objects.delete()
+        for i in xrange(55):
+            self.Person(name='A%s' % i, age=i).save()
+
+        self.assertEqual(len(self.Person.objects.scalar('name')), 55)
+        self.assertEqual("A0", "%s" % self.Person.objects.order_by('name').scalar('name').first())
+        self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').order_by('name')[0])
+        self.assertEqual("[u'A1', u'A2']",  "%s" % self.Person.objects.order_by('age').scalar('name')[1:3])
+        self.assertEqual("[u'A51', u'A52']",  "%s" % self.Person.objects.order_by('age').scalar('name')[51:53])
+
+        # with_id and in_bulk
+        person = self.Person.objects.order_by('name').first()
+        self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').with_id(person.id))
+
+        pks = self.Person.objects.order_by('age').scalar('pk')[1:3]
+        self.assertEqual("[u'A1', u'A2']",  "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values()))
+
 
 class QTest(unittest.TestCase):
 
@@ -2790,6 +3316,30 @@ class QueryFieldListTest(unittest.TestCase):
         q += QueryFieldList(fields=['a'], value={"$slice": 5})
         self.assertEqual(q.as_dict(), {'a': {"$slice": 5}})
 
+    def test_elem_match(self):
+        class Foo(EmbeddedDocument):
+            shape = StringField()
+            color = StringField()
+            trick = BooleanField()
+            meta = {'allow_inheritance': False}
+
+        class Bar(Document):
+            foo = ListField(EmbeddedDocumentField(Foo))
+            meta = {'allow_inheritance': False}
+
+        Bar.drop_collection()
+
+        b1 = Bar(foo=[Foo(shape= "square", color ="purple", thick = False),
+                      Foo(shape= "circle", color ="red", thick = True)])
+        b1.save()
+
+        b2 = Bar(foo=[Foo(shape= "square", color ="red", thick = True),
+                      Foo(shape= "circle", color ="purple", thick = False)])
+        b2.save()
+
+        ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"}))
+        self.assertEqual([b1], ak)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_replicaset_connection.py

@@ -0,0 +1,32 @@
+import unittest
+import pymongo
+from pymongo import ReadPreference, ReplicaSetConnection
+
+import mongoengine
+from mongoengine import *
+from mongoengine.connection import get_db, get_connection, ConnectionError
+
+
+class ConnectionTest(unittest.TestCase):
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    def test_replicaset_uri_passes_read_preference(self):
+        """Requires a replica set called "rs" on port 27017
+        """
+
+        try:
+            conn = connect(db='mongoenginetest', host="mongodb://localhost/mongoenginetest?replicaSet=rs", read_preference=ReadPreference.SECONDARY_ONLY)
+        except ConnectionError, e:
+            return
+
+        if not isinstance(conn, ReplicaSetConnection):
+            return
+
+        self.assertEquals(conn.read_preference, ReadPreference.SECONDARY_ONLY)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_signals.py

@@ -56,6 +56,18 @@ class SignalTests(unittest.TestCase):
             @classmethod
             def post_delete(cls, sender, document, **kwargs):
                 signal_output.append('post_delete signal, %s' % document)
+
+            @classmethod
+            def pre_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('pre_bulk_insert signal, %s' % documents)
+
+            @classmethod
+            def post_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('post_bulk_insert signal, %s' % documents)
+                if kwargs.get('loaded', False):
+                    signal_output.append('Is loaded')
+                else:
+                    signal_output.append('Not loaded')
         self.Author = Author
 
 
@@ -104,7 +116,9 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_save.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
-            len(signals.post_delete.receivers)
+            len(signals.post_delete.receivers),
+            len(signals.pre_bulk_insert.receivers),
+            len(signals.post_bulk_insert.receivers),
         )
 
         signals.pre_init.connect(Author.pre_init, sender=Author)
@@ -113,6 +127,8 @@ class SignalTests(unittest.TestCase):
         signals.post_save.connect(Author.post_save, sender=Author)
         signals.pre_delete.connect(Author.pre_delete, sender=Author)
         signals.post_delete.connect(Author.post_delete, sender=Author)
+        signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author)
+        signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author)
 
         signals.pre_init.connect(Another.pre_init, sender=Another)
         signals.post_init.connect(Another.post_init, sender=Another)
@@ -128,6 +144,8 @@ class SignalTests(unittest.TestCase):
         signals.pre_delete.disconnect(self.Author.pre_delete)
         signals.post_save.disconnect(self.Author.post_save)
         signals.pre_save.disconnect(self.Author.pre_save)
+        signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert)
+        signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert)
 
         signals.pre_init.disconnect(self.Another.pre_init)
         signals.post_init.disconnect(self.Another.post_init)
@@ -143,7 +161,9 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_save.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
-            len(signals.post_delete.receivers)
+            len(signals.post_delete.receivers),
+            len(signals.pre_bulk_insert.receivers),
+            len(signals.post_bulk_insert.receivers),
         )
 
         self.assertEqual(self.pre_signals, post_signals)
@@ -154,6 +174,14 @@ class SignalTests(unittest.TestCase):
         def create_author():
             a1 = self.Author(name='Bill Shakespeare')
 
+        def bulk_create_author_with_load():
+            a1 = self.Author(name='Bill Shakespeare')
+            self.Author.objects.insert([a1], load_bulk=True)
+
+        def bulk_create_author_without_load():
+            a1 = self.Author(name='Bill Shakespeare')
+            self.Author.objects.insert([a1], load_bulk=False)
+
         self.assertEqual(self.get_signal_output(create_author), [
             "pre_init signal, Author",
             "{'name': 'Bill Shakespeare'}",
@@ -179,3 +207,24 @@ class SignalTests(unittest.TestCase):
             'pre_delete signal, William Shakespeare',
             'post_delete signal, William Shakespeare',
         ])
+
+        signal_output = self.get_signal_output(bulk_create_author_with_load)
+
+        # The output of this signal is not entirely deterministic. The reloaded
+        # object will have an object ID. Hence, we only check part of the output
+        self.assertEquals(signal_output[3],
+            "pre_bulk_insert signal, [<Author: Bill Shakespeare>]")
+        self.assertEquals(signal_output[-2:],
+            ["post_bulk_insert signal, [<Author: Bill Shakespeare>]",
+             "Is loaded",])
+
+        self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [
+            "pre_init signal, Author",
+            "{'name': 'Bill Shakespeare'}",
+            "post_init signal, Bill Shakespeare",
+            "pre_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            "post_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            "Not loaded",
+        ])
+
+        self.Author.objects.delete()