Updated version

Conflicts:
	mongoengine/base.py

.gitignore

@@ -1,7 +1,8 @@
 .*
 !.gitignore
-*.pyc
-.*.swp
+*~
+*.py[co]
+.*.sw[po]
 *.egg
 docs/.build
 docs/_build
@@ -12,4 +13,5 @@ env/
 .settings
 .project
 .pydevproject
-tests/bugfix.py
+tests/test_bugfix.py
+htmlcov/

.travis.yml

@@ -0,0 +1,28 @@
+# http://travis-ci.org/#!/MongoEngine/mongoengine
+language: python
+python:
+    - 2.5
+    - 2.6
+    - 2.7
+    - 3.1
+    - 3.2
+env:
+  - PYMONGO=dev
+  - PYMONGO=2.3
+  - PYMONGO=2.2
+install:
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi
+    - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
+    - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
+    - python setup.py install
+script:
+    - python setup.py test
+notifications:
+  irc: "irc.freenode.org#mongoengine"
+branches:
+  only:
+    - master
+    - 0.7

AUTHORS

@@ -1,13 +1,14 @@
 The PRIMARY AUTHORS are (and/or have been):
 
+Ross Lawley <ross.lawley@gmail.com>
 Harry Marr <harry@hmarr.com>
 Matt Dennewitz <mattdennewitz@gmail.com>
 Deepak Thukral <iapain@yahoo.com>
 Florian Schlachter <flori@n-schlachter.de>
 Steve Challis <steve@stevechallis.com>
-Ross Lawley <ross.lawley@gmail.com>
 Wilson Júnior <wilsonpjunior@gmail.com>
 Dan Crosta https://github.com/dcrosta
+Laine Herron https://github.com/LaineHerron
 
 CONTRIBUTORS
 
@@ -67,5 +68,58 @@ that much better:
  * Gareth Lloyd
  * Albert Choi
  * John Arnfield
+ * grubberr
+ * Paul Aliagas
+ * Paul Cunnane
  * Julien Rebetez
-
+ * Marc Tamlyn
+ * Karim Allah
+ * Adam Parrish
+ * jpfarias
+ * jonrscott
+ * Alice Zoë Bevan-McGregor
+ * Stephen Young
+ * tkloc
+ * aid
+ * yamaneko1212
+ * dave mankoff
+ * Alexander G. Morano
+ * jwilder
+ * Joe Shaw
+ * Adam Flynn
+ * Ankhbayar
+ * Jan Schrewe
+ * David Koblas
+ * Crittercism
+ * Alvin Liang
+ * andrewmlevy
+ * Chris Faulkner
+ * Ashwin Purohit
+ * Shalabh Aggarwal
+ * Chris Williams
+ * Robert Kajic
+ * Jacob Peddicord
+ * Nils Hasenbanck
+ * mostlystatic
+ * Greg Banks
+ * swashbuckler
+ * Adam Reeve
+ * Anthony Nemitz
+ * deignacio
+ * shaunduncan
+ * Meir Kriheli
+ * Andrey Fedoseev
+ * aparajita
+ * Tristan Escalada
+ * Alexander Koshelev
+ * Jaime Irurzun
+ * Alexandre González
+ * Thomas Steinacher
+ * Tommi Komulainen
+ * Peter Landry
+ * biszkoptwielki
+ * Anton Kolechkin
+ * Sergey Nikitin
+ * psychogenic
+ * Stefan Wójcik
+ * dimonb

LICENSE

@@ -1,5 +1,5 @@
-Copyright (c) 2009-2010 Harry Marr
- 
+Copyright (c) 2009-2012 See AUTHORS
+
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation
 files (the "Software"), to deal in the Software without
@@ -8,10 +8,10 @@ copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the
 Software is furnished to do so, subject to the following
 conditions:
- 
+
 The above copyright notice and this permission notice shall be
 included in all copies or substantial portions of the Software.
- 
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

README.rst

@@ -2,26 +2,31 @@
 MongoEngine
 ===========
 :Info: MongoEngine is an ORM-like layer on top of PyMongo.
+:Repository: https://github.com/MongoEngine/mongoengine
 :Author: Harry Marr (http://github.com/hmarr)
+:Maintainer: Ross Lawley (http://github.com/rozza)
+
+.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master
+  :target: http://travis-ci.org/MongoEngine/mongoengine
 
 About
 =====
-MongoEngine is a Python Object-Document Mapper for working with MongoDB. 
-Documentation available at http://hmarr.com/mongoengine/ - there is currently 
-a `tutorial <http://hmarr.com/mongoengine/tutorial.html>`_, a `user guide 
-<http://hmarr.com/mongoengine/userguide.html>`_ and an `API reference
-<http://hmarr.com/mongoengine/apireference.html>`_.
+MongoEngine is a Python Object-Document Mapper for working with MongoDB.
+Documentation available at http://mongoengine-odm.rtfd.org - there is currently
+a `tutorial <http://readthedocs.org/docs/mongoengine-odm/en/latest/tutorial.html>`_, a `user guide
+<http://readthedocs.org/docs/mongoengine-odm/en/latest/userguide.html>`_ and an `API reference
+<http://readthedocs.org/docs/mongoengine-odm/en/latest/apireference.html>`_.
 
 Installation
 ============
 If you have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
 you can use ``easy_install -U mongoengine``. Otherwise, you can download the
-source from `GitHub <http://github.com/hmarr/mongoengine>`_ and run ``python
+source from `GitHub <http://github.com/MongoEngine/mongoengine>`_ and run ``python
 setup.py install``.
 
 Dependencies
 ============
-- pymongo 1.1+
+- pymongo 2.1.1+
 - sphinx (optional - for documentation generation)
 
 Examples
@@ -58,11 +63,6 @@ Some simple examples of what MongoEngine code looks like::
     ...         print 'Link:', post.url
     ...     print
     ...
-    === Using MongoEngine ===
-    See the tutorial
-
-    === MongoEngine Docs ===
-    Link: hmarr.com/mongoengine
 
     >>> len(BlogPost.objects)
     2
@@ -80,18 +80,18 @@ Some simple examples of what MongoEngine code looks like::
 Tests
 =====
 To run the test suite, ensure you are running a local instance of MongoDB on
-the standard port, and run ``python setup.py test``.
+the standard port, and run: ``python setup.py test``.
 
 Community
 =========
-- `MongoEngine Users mailing list 
+- `MongoEngine Users mailing list
   <http://groups.google.com/group/mongoengine-users>`_
-- `MongoEngine Developers mailing list 
+- `MongoEngine Developers mailing list
   <http://groups.google.com/group/mongoengine-dev>`_
-- `#mongoengine IRC channel <irc://irc.freenode.net/mongoengine>`_
+- `#mongoengine IRC channel <http://webchat.freenode.net/?channels=mongoengine>`_
 
 Contributing
 ============
-The source is available on `GitHub <http://github.com/hmarr/mongoengine>`_ - to
+The source is available on `GitHub <http://github.com/MongoEngine/mongoengine>`_ - to
 contribute to the project, fork it on GitHub and send a pull request, all
 contributions and suggestions are welcome!

benchmark.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+import timeit
+
+
+def cprofile_main():
+    from pymongo import Connection
+    connection = Connection()
+    connection.drop_database('timeit_test')
+    connection.disconnect()
+
+    from mongoengine import Document, DictField, connect
+    connect("timeit_test")
+
+    class Noddy(Document):
+        fields = DictField()
+
+    for i in xrange(1):
+        noddy = Noddy()
+        for j in range(20):
+            noddy.fields["key" + str(j)] = "value " + str(j)
+        noddy.save()
+
+
+def main():
+    """
+    0.4 Performance Figures ...
+
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    3.86744189262
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    6.23374891281
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    5.33027005196
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    pass - No Cascade
+
+    0.5.X
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    3.89597702026
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    21.7735359669
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    19.8670389652
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    pass - No Cascade
+
+    0.6.X
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    3.81559205055
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    10.0446798801
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    9.51354718208
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    9.02567505836
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, force=True
+    8.44933390617
+
+    0.7.X
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - Pymongo
+    3.78801012039
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine
+    9.73050498962
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
+    8.33456707001
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
+    8.37778115273
+    ----------------------------------------------------------------------------------------------------
+    Creating 10000 dictionaries - MongoEngine, force=True
+    8.36906409264
+    """
+
+    setup = """
+from pymongo import Connection
+connection = Connection()
+connection.drop_database('timeit_test')
+"""
+
+    stmt = """
+from pymongo import Connection
+connection = Connection()
+
+db = connection.timeit_test
+noddy = db.noddy
+
+for i in xrange(10000):
+    example = {'fields': {}}
+    for j in range(20):
+        example['fields']["key"+str(j)] = "value "+str(j)
+
+    noddy.insert(example)
+
+myNoddys = noddy.find()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - Pymongo"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    setup = """
+from pymongo import Connection
+connection = Connection()
+connection.drop_database('timeit_test')
+connection.disconnect()
+
+from mongoengine import Document, DictField, connect
+connect("timeit_test")
+
+class Noddy(Document):
+    fields = DictField()
+"""
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save()
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(safe=False, validate=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(safe=False, validate=False, cascade=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+    stmt = """
+for i in xrange(10000):
+    noddy = Noddy()
+    for j in range(20):
+        noddy.fields["key"+str(j)] = "value "+str(j)
+    noddy.save(force_insert=True, safe=False, validate=False, cascade=False)
+
+myNoddys = Noddy.objects()
+[n for n in myNoddys] # iterate
+"""
+
+    print "-" * 100
+    print """Creating 10000 dictionaries - MongoEngine, force=True"""
+    t = timeit.Timer(stmt=stmt, setup=setup)
+    print t.timeit(1)
+
+if __name__ == "__main__":
+    main()

docs/apireference.rst

@@ -6,6 +6,7 @@ Connecting
 ==========
 
 .. autofunction:: mongoengine.connect
+.. autofunction:: mongoengine.register_connection
 
 Documents
 =========
@@ -21,9 +22,18 @@ Documents
 .. autoclass:: mongoengine.EmbeddedDocument
    :members:
 
+.. autoclass:: mongoengine.DynamicDocument
+   :members:
+
+.. autoclass:: mongoengine.DynamicEmbeddedDocument
+   :members:
+
 .. autoclass:: mongoengine.document.MapReduceDocument
   :members:
 
+.. autoclass:: mongoengine.ValidationError
+  :members:
+
 Querying
 ========
 
@@ -37,25 +47,28 @@ Querying
 Fields
 ======
 
-.. autoclass:: mongoengine.StringField
-.. autoclass:: mongoengine.URLField
-.. autoclass:: mongoengine.EmailField
-.. autoclass:: mongoengine.IntField
-.. autoclass:: mongoengine.FloatField
-.. autoclass:: mongoengine.DecimalField
-.. autoclass:: mongoengine.DateTimeField
+.. autoclass:: mongoengine.BinaryField
+.. autoclass:: mongoengine.BooleanField
 .. autoclass:: mongoengine.ComplexDateTimeField
-.. autoclass:: mongoengine.ListField
-.. autoclass:: mongoengine.SortedListField
+.. autoclass:: mongoengine.DateTimeField
+.. autoclass:: mongoengine.DecimalField
 .. autoclass:: mongoengine.DictField
+.. autoclass:: mongoengine.DynamicField
+.. autoclass:: mongoengine.EmailField
+.. autoclass:: mongoengine.EmbeddedDocumentField
+.. autoclass:: mongoengine.FileField
+.. autoclass:: mongoengine.FloatField
+.. autoclass:: mongoengine.GenericEmbeddedDocumentField
+.. autoclass:: mongoengine.GenericReferenceField
+.. autoclass:: mongoengine.GeoPointField
+.. autoclass:: mongoengine.ImageField
+.. autoclass:: mongoengine.IntField
+.. autoclass:: mongoengine.ListField
 .. autoclass:: mongoengine.MapField
 .. autoclass:: mongoengine.ObjectIdField
 .. autoclass:: mongoengine.ReferenceField
-.. autoclass:: mongoengine.GenericReferenceField
-.. autoclass:: mongoengine.EmbeddedDocumentField
-.. autoclass:: mongoengine.GenericEmbeddedDocumentField
-.. autoclass:: mongoengine.BooleanField
-.. autoclass:: mongoengine.FileField
-.. autoclass:: mongoengine.BinaryField
-.. autoclass:: mongoengine.GeoPointField
 .. autoclass:: mongoengine.SequenceField
+.. autoclass:: mongoengine.SortedListField
+.. autoclass:: mongoengine.StringField
+.. autoclass:: mongoengine.URLField
+.. autoclass:: mongoengine.UUIDField

docs/changelog.rst

@@ -2,15 +2,246 @@
 Changelog
 =========
 
+Changes in 0.7.2
+================
+- Update index spec generation so its not destructive (MongoEngine/mongoengine#113)
+
+Changes in 0.7.1
+=================
+- Fixed index spec inheritance (MongoEngine/mongoengine#111)
+
+Changes in 0.7.0
+=================
+- Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107)
+- Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104)
+- Fixed Q object merge edge case (MongoEngine/mongoengine#109)
+- Fixed reloading on sharded documents (hmarr/mongoengine#569)
+- Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62)
+- Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92)
+- Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88)
+- Updated ReferenceField's to optionally store ObjectId strings
+  this will become the default in 0.8 (MongoEngine/mongoengine#89)
+- Added FutureWarning - save will default to `cascade=False` in 0.8
+- Added example of indexing embedded document fields (MongoEngine/mongoengine#75)
+- Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80)
+- Add flexibility for fields handling bad data (MongoEngine/mongoengine#78)
+- Embedded Documents no longer handle meta definitions
+- Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74)
+- Improved queryset filtering (hmarr/mongoengine#554)
+- Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561)
+- Fixed abstract classes and shard keys (MongoEngine/mongoengine#64)
+- Fixed Python 2.5 support
+- Added Python 3 support (thanks to Laine Heron)
+
+Changes in 0.6.20
+=================
+- Added support for distinct and db_alias (MongoEngine/mongoengine#59)
+- Improved support for chained querysets when constraining the same fields (hmarr/mongoengine#554)
+- Fixed BinaryField lookup re (MongoEngine/mongoengine#48)
+
+Changes in 0.6.19
+=================
+
+- Added Binary support to UUID (MongoEngine/mongoengine#47)
+- Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46)
+- Fixed BinaryField python value issue (MongoEngine/mongoengine#48)
+- Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41)
+- Fixed queryset manager issue (MongoEngine/mongoengine#52)
+- Fixed FileField comparision (hmarr/mongoengine#547)
+
+Changes in 0.6.18
+=================
+- Fixed recursion loading bug in _get_changed_fields
+
+Changes in 0.6.17
+=================
+- Fixed issue with custom queryset manager expecting explict variable names
+
+Changes in 0.6.16
+=================
+- Fixed issue where db_alias wasn't inherited
+
+Changes in 0.6.15
+=================
+- Updated validation error messages
+- Added support for null / zero / false values in item_frequencies
+- Fixed cascade save edge case
+- Fixed geo index creation through reference fields
+- Added support for args / kwargs when using @queryset_manager
+- Deref list custom id fix
+
+Changes in 0.6.14
+=================
+- Fixed error dict with nested validation
+- Fixed Int/Float fields and not equals None
+- Exclude tests from installation
+- Allow tuples for index meta
+- Fixed use of str in instance checks
+- Fixed unicode support in transform update
+- Added support for add_to_set and each
+
+Changes in 0.6.13
+=================
+- Fixed EmbeddedDocument db_field validation issue
+- Fixed StringField unicode issue
+- Fixes __repr__ modifying the cursor
+
+Changes in 0.6.12
+=================
+- Fixes scalar lookups for primary_key
+- Fixes error with _delta handling DBRefs
+
+Changes in 0.6.11
+==================
+- Fixed inconsistency handling None values field attrs
+- Fixed map_field embedded db_field issue
+- Fixed .save() _delta issue with DbRefs
+- Fixed Django TestCase
+- Added cmp to Embedded Document
+- Added PULL reverse_delete_rule
+- Fixed CASCADE delete bug
+- Fixed db_field data load error
+- Fixed recursive save with FileField
+
+Changes in 0.6.10
+=================
+- Fixed basedict / baselist to return super(..)
+- Promoted BaseDynamicField to DynamicField
+
+Changes in 0.6.9
+================
+- Fixed sparse indexes on inherited docs
+- Removed FileField auto deletion, needs more work maybe 0.7
+
+Changes in 0.6.8
+================
+- Fixed FileField losing reference when no default set
+- Removed possible race condition from FileField (grid_file)
+- Added assignment to save, can now do: `b = MyDoc(**kwargs).save()`
+- Added support for pull operations on nested EmbeddedDocuments
+- Added support for choices with GenericReferenceFields
+- Added support for choices with GenericEmbeddedDocumentFields
+- Fixed Django 1.4 sessions first save data loss
+- FileField now automatically delete files on .delete()
+- Fix for GenericReference to_mongo method
+- Fixed connection regression
+- Updated Django User document, now allows inheritance
+
+Changes in 0.6.7
+================
+- Fixed indexing on '_id' or 'pk' or 'id'
+- Invalid data from the DB now raises a InvalidDocumentError
+- Cleaned up the Validation Error - docs and code
+- Added meta `auto_create_index` so you can disable index creation
+- Added write concern options to inserts
+- Fixed typo in meta for index options
+- Bug fix Read preference now passed correctly
+- Added support for File like objects for GridFS
+- Fix for #473 - Dereferencing abstracts
+
+Changes in 0.6.6
+================
+- Django 1.4 fixed (finally)
+- Added tests for Django
+
+Changes in 0.6.5
+================
+- More Django updates
+
+Changes in 0.6.4
+================
+
+- Refactored connection / fixed replicasetconnection
+- Bug fix for unknown connection alias error message
+- Sessions support Django 1.3 and Django 1.4
+- Minor fix for ReferenceField
+
+Changes in 0.6.3
+================
+- Updated sessions for Django 1.4
+- Bug fix for updates where listfields contain embedded documents
+- Bug fix for collection naming and mixins
+
+Changes in 0.6.2
+================
+- Updated documentation for ReplicaSet connections
+- Hack round _types issue with SERVER-5247 - querying other arrays may also cause problems.
+
+Changes in 0.6.1
+================
+- Fix for replicaSet connections
+
+Changes in 0.6
+================
+
+- Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7
+- Added support for covered indexes when inheritance is off
+- No longer always upsert on save for items with a '_id'
+- Error raised if update doesn't have an operation
+- DeReferencing is now thread safe
+- Errors raised if trying to perform a join in a query
+- Updates can now take __raw__ queries
+- Added custom 2D index declarations
+- Added replicaSet connection support
+- Updated deprecated imports from pymongo (safe for pymongo 2.2)
+- Added uri support for connections
+- Added scalar for efficiently returning partial data values (aliased to values_list)
+- Fixed limit skip bug
+- Improved Inheritance / Mixin
+- Added sharding support
+- Added pymongo 2.1 support
+- Fixed Abstract documents can now declare indexes
+- Added db_alias support to individual documents
+- Fixed GridFS documents can now be pickled
+- Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field
+- Added InvalidQueryError when calling with_id with a filter
+- Added support for DBRefs in distinct()
+- Fixed issue saving False booleans
+- Fixed issue with dynamic documents deltas
+- Added Reverse Delete Rule support to ListFields - MapFields aren't supported
+- Added customisable cascade kwarg options
+- Fixed Handle None values for non-required fields
+- Removed Document._get_subclasses() - no longer required
+- Fixed bug requiring subclasses when not actually needed
+- Fixed deletion of dynamic data
+- Added support for the $elementMatch operator
+- Added reverse option to SortedListFields
+- Fixed dereferencing - multi directional list dereferencing
+- Fixed issue creating indexes with recursive embedded documents
+- Fixed recursive lookup in _unique_with_indexes
+- Fixed passing ComplexField defaults to constructor for ReferenceFields
+- Fixed validation of DictField Int keys
+- Added optional cascade saving
+- Fixed dereferencing - max_depth now taken into account
+- Fixed document mutation saving issue
+- Fixed positional operator when replacing embedded documents
+- Added Non-Django Style choices back (you can have either)
+- Fixed __repr__ of a sliced queryset
+- Added recursive validation error of documents / complex fields
+- Fixed breaking during queryset iteration
+- Added pre and post bulk-insert signals
+- Added ImageField - requires PIL
+- Fixed Reference Fields can be None in get_or_create / queries
+- Fixed accessing pk on an embedded document
+- Fixed calling a queryset after drop_collection now recreates the collection
+- Add field name to validation exception messages
+- Added UUID field
+- Improved efficiency of .get()
+- Updated ComplexFields so if required they won't accept empty lists / dicts
+- Added spec file for rpm-based distributions
+- Fixed ListField so it doesnt accept strings
+- Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas
+
 Changes in v0.5.2
 =================
 
 - A Robust Circular reference bugfix
 
+
 Changes in v0.5.1
 =================
 
-- Circular reference bugfix
+- Fixed simple circular reference bug
 
 Changes in v0.5
 ===============

docs/conf.py

@@ -38,7 +38,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'MongoEngine'
-copyright = u'2009-2011, Harry Marr'
+copyright = u'2009-2012, MongoEngine Authors'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -121,7 +121,7 @@ html_theme_path = ['_themes']
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

docs/django.rst

@@ -2,19 +2,21 @@
 Using MongoEngine with Django
 =============================
 
+.. note :: Updated to support Django 1.4
+
 Connecting
 ==========
 In your **settings.py** file, ignore the standard database settings (unless you
-also plan to use the ORM in your project), and instead call 
+also plan to use the ORM in your project), and instead call
 :func:`~mongoengine.connect` somewhere in the settings module.
 
 Authentication
 ==============
 MongoEngine includes a Django authentication backend, which uses MongoDB. The
-:class:`~mongoengine.django.auth.User` model is a MongoEngine 
-:class:`~mongoengine.Document`, but implements most of the methods and 
+:class:`~mongoengine.django.auth.User` model is a MongoEngine
+:class:`~mongoengine.Document`, but implements most of the methods and
 attributes that the standard Django :class:`User` model does - so the two are
-moderately compatible. Using this backend will allow you to store users in 
+moderately compatible. Using this backend will allow you to store users in
 MongoDB but still use many of the Django authentication infrastucture (such as
 the :func:`login_required` decorator and the :func:`authenticate` function). To
 enable the MongoEngine auth backend, add the following to you **settings.py**
@@ -24,7 +26,7 @@ file::
         'mongoengine.django.auth.MongoEngineBackend',
     )
 
-The :mod:`~mongoengine.django.auth` module also contains a 
+The :mod:`~mongoengine.django.auth` module also contains a
 :func:`~mongoengine.django.auth.get_user` helper function, that takes a user's
 :attr:`id` and returns a :class:`~mongoengine.django.auth.User` object.
 
@@ -49,9 +51,9 @@ Storage
 =======
 With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`,
 it is useful to have a Django file storage backend that wraps this. The new
-storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. 
+storage module is called :class:`~mongoengine.django.storage.GridFSStorage`.
 Using it is very similar to using the default FileSystemStorage.::
-    
+
     from mongoengine.django.storage import GridFSStorage
     fs = GridFSStorage()
 

docs/guide/connecting.rst

@@ -3,6 +3,7 @@
 =====================
 Connecting to MongoDB
 =====================
+
 To connect to a running instance of :program:`mongod`, use the
 :func:`~mongoengine.connect` function. The first argument is the name of the
 database to connect to. If the database does not exist, it will be created. If
@@ -18,3 +19,47 @@ provide :attr:`host` and :attr:`port` arguments to
 :func:`~mongoengine.connect`::
 
     connect('project1', host='192.168.1.35', port=12345)
+
+Uri style connections are also supported as long as you include the database
+name - just supply the uri as the :attr:`host` to
+:func:`~mongoengine.connect`::
+
+    connect('project1', host='mongodb://localhost/database_name')
+
+ReplicaSets
+===========
+
+MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnection`
+to use them please use a URI style connection and provide the `replicaSet` name in the
+connection kwargs.
+
+Multiple Databases
+==================
+
+Multiple database support was added in MongoEngine 0.6. To use multiple
+databases you can use :func:`~mongoengine.connect` and provide an `alias` name
+for the connection - if no `alias` is provided then "default" is used.
+
+In the background this uses :func:`~mongoengine.register_connection` to
+store the data and you can register all aliases up front if required.
+
+Individual documents can also support multiple databases by providing a
+`db_alias` in their meta data.  This allows :class:`~pymongo.dbref.DBRef` objects
+to point across databases and collections.  Below is an example schema, using
+3 different databases to store data::
+
+        class User(Document):
+            name = StringField()
+
+            meta = {"db_alias": "user-db"}
+
+        class Book(Document):
+            name = StringField()
+
+            meta = {"db_alias": "book-db"}
+
+        class AuthorBooks(Document):
+            author = ReferenceField(User)
+            book = ReferenceField(Book)
+
+            meta = {"db_alias": "users-books-db"}

docs/guide/defining-documents.rst

@@ -24,6 +24,34 @@ objects** as class attributes to the document class::
         title = StringField(max_length=200, required=True)
         date_modified = DateTimeField(default=datetime.datetime.now)
 
+Dynamic document schemas
+========================
+One of the benefits of MongoDb is dynamic schemas for a collection, whilst data
+should be planned and organised (after all explicit is better than implicit!)
+there are scenarios where having dynamic / expando style documents is desirable.
+
+:class:`~mongoengine.DynamicDocument` documents work in the same way as
+:class:`~mongoengine.Document` but any data / attributes set to them will also
+be saved ::
+
+    from mongoengine import *
+
+    class Page(DynamicDocument):
+        title = StringField(max_length=200, required=True)
+
+    # Create a new page and add tags
+    >>> page = Page(title='Using MongoEngine')
+    >>> page.tags = ['mongodb', 'mongoengine']
+    >>> page.save()
+
+    >>> Page.objects(tags='mongoengine').count()
+    >>> 1
+
+..note::
+
+   There is one caveat on Dynamic Documents: fields cannot start with `_`
+
+
 Fields
 ======
 By default, fields are not required. To make a field mandatory, set the
@@ -34,28 +62,31 @@ not provided. Default values may optionally be a callable, which will be called
 to retrieve the value (such as in the above example). The field types available
 are as follows:
 
-* :class:`~mongoengine.StringField`
-* :class:`~mongoengine.URLField`
-* :class:`~mongoengine.EmailField`
-* :class:`~mongoengine.IntField`
-* :class:`~mongoengine.FloatField`
-* :class:`~mongoengine.DecimalField`
-* :class:`~mongoengine.DateTimeField`
+* :class:`~mongoengine.BinaryField`
+* :class:`~mongoengine.BooleanField`
 * :class:`~mongoengine.ComplexDateTimeField`
-* :class:`~mongoengine.ListField`
-* :class:`~mongoengine.SortedListField`
+* :class:`~mongoengine.DateTimeField`
+* :class:`~mongoengine.DecimalField`
 * :class:`~mongoengine.DictField`
+* :class:`~mongoengine.DynamicField`
+* :class:`~mongoengine.EmailField`
+* :class:`~mongoengine.EmbeddedDocumentField`
+* :class:`~mongoengine.FileField`
+* :class:`~mongoengine.FloatField`
+* :class:`~mongoengine.GenericEmbeddedDocumentField`
+* :class:`~mongoengine.GenericReferenceField`
+* :class:`~mongoengine.GeoPointField`
+* :class:`~mongoengine.ImageField`
+* :class:`~mongoengine.IntField`
+* :class:`~mongoengine.ListField`
 * :class:`~mongoengine.MapField`
 * :class:`~mongoengine.ObjectIdField`
 * :class:`~mongoengine.ReferenceField`
-* :class:`~mongoengine.GenericReferenceField`
-* :class:`~mongoengine.EmbeddedDocumentField`
-* :class:`~mongoengine.GenericEmbeddedDocumentField`
-* :class:`~mongoengine.BooleanField`
-* :class:`~mongoengine.FileField`
-* :class:`~mongoengine.BinaryField`
-* :class:`~mongoengine.GeoPointField`
 * :class:`~mongoengine.SequenceField`
+* :class:`~mongoengine.SortedListField`
+* :class:`~mongoengine.StringField`
+* :class:`~mongoengine.URLField`
+* :class:`~mongoengine.UUIDField`
 
 Field arguments
 ---------------
@@ -70,7 +101,7 @@ arguments can be set on all fields:
 
 :attr:`required` (Default: False)
     If set to True and the field is not set on the document instance, a
-    :class:`~mongoengine.base.ValidationError` will be raised when the document is
+    :class:`~mongoengine.ValidationError` will be raised when the document is
     validated.
 
 :attr:`default` (Default: None)
@@ -107,12 +138,33 @@ arguments can be set on all fields:
     When True, use this field as a primary key for the collection.
 
 :attr:`choices` (Default: None)
-    An iterable of choices to which the value of this field should be limited.
+    An iterable (e.g. a list or tuple) of choices to which the value of this
+    field should be limited.
+
+    Can be either be a nested tuples of value (stored in mongo) and a
+    human readable key ::
+
+        SIZE = (('S', 'Small'),
+                ('M', 'Medium'),
+                ('L', 'Large'),
+                ('XL', 'Extra Large'),
+                ('XXL', 'Extra Extra Large'))
+
+
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=SIZE)
+
+    Or a flat iterable just containing values ::
+
+        SIZE = ('S', 'M', 'L', 'XL', 'XXL')
+
+        class Shirt(Document):
+            size = StringField(max_length=3, choices=SIZE)
 
 :attr:`help_text` (Default: None)
     Optional help text to output with the field - used by form libraries
 
-:attr:`verbose` (Default: None)
+:attr:`verbose_name` (Default: None)
     Optional human-readable name for the field - used by form libraries
 
 
@@ -207,6 +259,35 @@ as the constructor's argument::
         content = StringField()
 
 
+.. _one-to-many-with-listfields:
+
+One to Many with ListFields
+'''''''''''''''''''''''''''
+
+If you are implementing a one to many relationship via a list of references,
+then the references are stored as DBRefs and to query you need to pass an
+instance of the object to the query::
+
+    class User(Document):
+        name = StringField()
+
+    class Page(Document):
+        content = StringField()
+        authors = ListField(ReferenceField(User))
+
+    bob = User(name="Bob Jones").save()
+    john = User(name="John Smith").save()
+
+    Page(content="Test Page", authors=[bob, john]).save()
+    Page(content="Another Page", authors=[john]).save()
+
+    # Find all pages Bob authored
+    Page.objects(authors__in=[bob])
+
+    # Find all pages that both Bob and John have authored
+    Page.objects(authors__all=[bob, john])
+
+
 Dealing with deletion of referred documents
 '''''''''''''''''''''''''''''''''''''''''''
 By default, MongoDB doesn't check the integrity of your data, so deleting
@@ -240,6 +321,10 @@ Its value can take any of the following constants:
 :const:`mongoengine.CASCADE`
   Any object containing fields that are refererring to the object being deleted
   are deleted first.
+:const:`mongoengine.PULL`
+  Removes the reference to the object (using MongoDB's "pull" operation)
+  from any object's fields of
+  :class:`~mongoengine.ListField` (:class:`~mongoengine.ReferenceField`).
 
 
 .. warning::
@@ -382,10 +467,36 @@ If a dictionary is passed then the following options are available:
 :attr:`unique` (Default: False)
     Whether the index should be sparse.
 
-.. note::
+.. note ::
 
-   Geospatial indexes will be automatically created for all
-   :class:`~mongoengine.GeoPointField`\ s
+    To index embedded files / dictionary fields use 'dot' notation eg:
+    `rank.title`
+
+.. warning::
+
+    Inheritance adds extra indices.
+    If don't need inheritance for a document turn inheritance off -
+    see :ref:`document-inheritance`.
+
+
+Geospatial indexes
+---------------------------
+Geospatial indexes will be automatically created for all
+:class:`~mongoengine.GeoPointField`\ s
+
+It is also possible to explicitly define geospatial indexes. This is
+useful if you need to define a geospatial index on a subfield of a
+:class:`~mongoengine.DictField` or a custom field that contains a
+point. To create a geospatial index you must prefix the field with the
+***** sign. ::
+
+    class Place(Document):
+        location = DictField()
+        meta = {
+            'indexes': [
+                '*location.point',
+            ],
+        }
 
 Ordering
 ========
@@ -427,8 +538,31 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. ::
     first_post = BlogPost.objects.order_by("+published_date").first()
     assert first_post.title == "Blog Post #1"
 
+Shard keys
+==========
+
+If your collection is sharded, then you need to specify the shard key as a tuple,
+using the :attr:`shard_key` attribute of :attr:`-mongoengine.Document.meta`.
+This ensures that the shard key is sent with the query when calling the
+:meth:`~mongoengine.document.Document.save` or
+:meth:`~mongoengine.document.Document.update` method on an existing
+:class:`-mongoengine.Document` instance::
+
+    class LogEntry(Document):
+        machine = StringField()
+        app = StringField()
+        timestamp = DateTimeField()
+        data = StringField()
+
+        meta = {
+            'shard_key': ('machine', 'timestamp',)
+        }
+
+.. _document-inheritance:
+
 Document inheritance
 ====================
+
 To create a specialised type of a :class:`~mongoengine.Document` you have
 defined, you may subclass it and add any extra fields or methods you may need.
 As this is new class is not a direct subclass of
@@ -440,10 +574,15 @@ convenient and efficient retrieval of related documents::
     class Page(Document):
         title = StringField(max_length=200, required=True)
 
+        meta = {'allow_inheritance': True}
+
     # Also stored in the collection named 'page'
     class DatedPage(Page):
         date = DateTimeField()
 
+.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document meta.
+
+
 Working with existing data
 --------------------------
 To enable correct retrieval of documents involved in this kind of heirarchy,

docs/guide/document-instances.rst

@@ -35,13 +35,23 @@ already exist, then any changes will be updated atomically.  For example::
     * ``list_field.pop(0)`` - *sets* the resulting list
     * ``del(list_field)``   - *unsets* whole list
 
-To delete a document, call the :meth:`~mongoengine.Document.delete` method.
-Note that this will only work if the document exists in the database and has a
-valide :attr:`id`.
-
 .. seealso::
     :ref:`guide-atomic-updates`
 
+Cascading Saves
+---------------
+If your document contains :class:`~mongoengine.ReferenceField` or
+:class:`~mongoengine.GenericReferenceField` objects, then by default the
+:meth:`~mongoengine.Document.save` method will automatically save any changes to
+those objects as well.  If this is not desired passing :attr:`cascade` as False
+to the save method turns this feature off.
+
+Deleting documents
+------------------
+To delete a document, call the :meth:`~mongoengine.Document.delete` method.
+Note that this will only work if the document exists in the database and has a
+valid :attr:`id`.
+
 Document IDs
 ============
 Each document in the database has a unique id. This may be accessed through the
@@ -81,5 +91,5 @@ is an alias to :attr:`id`::
 .. note::
 
    If you define your own primary key field, the field implicitly becomes
-   required, so a :class:`ValidationError` will be thrown if you don't provide
-   it.
+   required, so a :class:`~mongoengine.ValidationError` will be thrown if
+   you don't provide it.

docs/guide/gridfs.rst

@@ -65,7 +65,7 @@ Deleting stored files is achieved with the :func:`delete` method::
 
     marmot.photo.delete()
 
-.. note::
+.. warning::
 
     The FileField in a Document actually only stores the ID of a file in a
     separate GridFS collection. This means that deleting a document

docs/guide/querying.rst

@@ -76,6 +76,7 @@ expressions:
 * ``istartswith`` -- string field starts with value (case insensitive)
 * ``endswith`` -- string field ends with value
 * ``iendswith`` -- string field ends with value (case insensitive)
+* ``match``  -- performs an $elemMatch so you can match an entire document within an array
 
 There are a few special operators for performing geographical queries, that
 may used with :class:`~mongoengine.GeoPointField`\ s:
@@ -194,22 +195,6 @@ to be created::
     >>> a.name == b.name and a.age == b.age
     True
 
-Dereferencing results
----------------------
-When iterating the results of :class:`~mongoengine.ListField` or
-:class:`~mongoengine.DictField` we automatically dereference any
-:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the
-number the queries to mongo.
-
-There are times when that efficiency is not enough, documents that have
-:class:`~mongoengine.ReferenceField` objects or
-:class:`~mongoengine.GenericReferenceField` objects at the top level are
-expensive as the number of queries to MongoDB can quickly rise.
-
-To limit the number of queries use
-:func:`~mongoengine.queryset.QuerySet.select_related` which converts the
-QuerySet to a list and dereferences as efficiently as possible.
-
 Default Document queries
 ========================
 By default, the objects :attr:`~mongoengine.Document.objects` attribute on a
@@ -247,7 +232,7 @@ custom manager methods as you like::
     BlogPost(title='test1', published=False).save()
     BlogPost(title='test2', published=True).save()
     assert len(BlogPost.objects) == 2
-    assert len(BlogPost.live_posts) == 1
+    assert len(BlogPost.live_posts()) == 1
 
 Custom QuerySets
 ================
@@ -258,11 +243,16 @@ a document, set ``queryset_class`` to the custom class in a
 :class:`~mongoengine.Document`\ s ``meta`` dictionary::
 
     class AwesomerQuerySet(QuerySet):
-        pass
+
+        def get_awesome(self):
+            return self.filter(awesome=True)
 
     class Page(Document):
         meta = {'queryset_class': AwesomerQuerySet}
 
+    # To call:
+    Page.objects.get_awesome()
+
 .. versionadded:: 0.4
 
 Aggregation
@@ -312,8 +302,16 @@ would be generating "tag-clouds"::
     from operator import itemgetter
     top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10]
 
+
+Query efficiency and performance
+================================
+
+There are a couple of methods to improve efficiency when querying, reducing the
+information returned by the query or efficient dereferencing .
+
 Retrieving a subset of fields
-=============================
+-----------------------------
+
 Sometimes a subset of fields on a :class:`~mongoengine.Document` is required,
 and for efficiency only these should be retrieved from the database. This issue
 is especially important for MongoDB, as fields may often be extremely large
@@ -346,6 +344,27 @@ will be given::
 If you later need the missing fields, just call
 :meth:`~mongoengine.Document.reload` on your document.
 
+Getting related data
+--------------------
+
+When iterating the results of :class:`~mongoengine.ListField` or
+:class:`~mongoengine.DictField` we automatically dereference any
+:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the
+number the queries to mongo.
+
+There are times when that efficiency is not enough, documents that have
+:class:`~mongoengine.ReferenceField` objects or
+:class:`~mongoengine.GenericReferenceField` objects at the top level are
+expensive as the number of queries to MongoDB can quickly rise.
+
+To limit the number of queries use
+:func:`~mongoengine.queryset.QuerySet.select_related` which converts the
+QuerySet to a list and dereferences as efficiently as possible.  By default
+:func:`~mongoengine.queryset.QuerySet.select_related` only dereferences any
+references to the depth of 1 level.  If you have more complicated documents and
+want to dereference more of the object at once then increasing the :attr:`max_depth`
+will dereference more levels of the document.
+
 Advanced queries
 ================
 Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword

docs/guide/signals.rst

@@ -5,11 +5,13 @@ Signals
 
 .. versionadded:: 0.5
 
-Signal support is provided by the excellent `blinker`_ library and
-will gracefully fall back if it is not available.
+.. note::
+
+  Signal support is provided by the excellent `blinker`_ library and
+  will gracefully fall back if it is not available.
 
 
-The following document signals exist in MongoEngine and are pretty self explaintary:
+The following document signals exist in MongoEngine and are pretty self-explanatory:
 
   * `mongoengine.signals.pre_init`
   * `mongoengine.signals.post_init`
@@ -17,6 +19,8 @@ The following document signals exist in MongoEngine and are pretty self explaint
   * `mongoengine.signals.post_save`
   * `mongoengine.signals.pre_delete`
   * `mongoengine.signals.post_delete`
+  * `mongoengine.signals.pre_bulk_insert`
+  * `mongoengine.signals.post_bulk_insert`
 
 Example usage::
 
@@ -42,8 +46,8 @@ Example usage::
                 else:
                     logging.debug("Updated")
 
-        signals.pre_save.connect(Author.pre_save, sender=Author)
-        signals.post_save.connect(Author.post_save, sender=Author)
+    signals.pre_save.connect(Author.pre_save, sender=Author)
+    signals.post_save.connect(Author.post_save, sender=Author)
 
 
 .. _blinker: http://pypi.python.org/pypi/blinker

docs/index.rst

@@ -18,6 +18,9 @@ MongoDB. To install it, simply run
 :doc:`apireference`
   The complete API documentation.
 
+:doc:`upgrade`
+  How to upgrade MongoEngine.
+
 :doc:`django`
   Using MongoEngine and Django
 
@@ -42,7 +45,8 @@ Also, you can join the developers' `mailing list
 
 Changes
 -------
-See the :doc:`changelog` for a full list of changes to MongoEngine.
+See the :doc:`changelog` for a full list of changes to MongoEngine and
+:doc:`upgrade` for upgrade information.
 
 .. toctree::
    :hidden:

docs/tutorial.rst

@@ -167,6 +167,11 @@ To delete all the posts if a user is deleted set the rule::
 
 See :class:`~mongoengine.ReferenceField` for more information.
 
+..note::
+    MapFields and DictFields currently don't support automatic handling of
+    deleted references
+
+
 Adding data to our Tumblelog
 ============================
 Now that we've defined how our documents will be structured, let's start adding

docs/upgrade.rst

@@ -2,6 +2,85 @@
 Upgrading
 =========
 
+0.6 to 0.7
+==========
+
+Cascade saves
+-------------
+
+Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set
+to True.  This is because in 0.8 it will default to False.  If you require
+cascading saves then either set it in the `meta` or pass
+via `save` eg ::
+
+    # At the class level:
+    class Person(Document):
+        meta = {'cascade': True}
+
+    # Or in code:
+    my_document.save(cascade=True)
+
+.. note ::
+    Remember: cascading saves **do not** cascade through lists.
+
+ReferenceFields
+---------------
+
+ReferenceFields now can store references as ObjectId strings instead of DBRefs.
+This will become the default in 0.8 and if `dbref` is not set a `FutureWarning`
+will be raised.
+
+
+To explicitly continue to use DBRefs change the `dbref` flag
+to True ::
+
+   class Person(Document):
+       groups = ListField(ReferenceField(Group, dbref=True))
+
+To migrate to using strings instead of DBRefs you will have to manually
+migrate ::
+
+        # Step 1 - Migrate the model definition
+        class Group(Document):
+            author = ReferenceField(User, dbref=False)
+            members = ListField(ReferenceField(User, dbref=False))
+
+        # Step 2 - Migrate the data
+        for g in Group.objects():
+            g.author = g.author
+            g.members = g.members
+            g.save()
+
+
+item_frequencies
+----------------
+
+In the 0.6 series we added support for null / zero / false values in
+item_frequencies.  A side effect was to return keys in the value they are
+stored in rather than as string representations.  Your code may need to be
+updated to handle native types rather than strings keys for the results of
+item frequency queries.
+
+0.5 to 0.6
+==========
+
+Embedded Documents - if you had a `pk` field you will have to rename it from
+`_id` to `pk` as pk is no longer a property of Embedded Documents.
+
+Reverse Delete Rules in Embedded Documents, MapFields and DictFields now throw
+an InvalidDocument error as they aren't currently supported.
+
+Document._get_subclasses - Is no longer used and the class method has been
+removed.
+
+Document.objects.with_id - now raises an InvalidQueryError if used with a
+filter.
+
+FutureWarning - A future warning has been added to all inherited classes that
+don't define `allow_inheritance` in their meta.
+
+You may need to update pyMongo to 2.0 for use with Sharding.
+
 0.4 to 0.5
 ===========
 
@@ -9,7 +88,7 @@ There have been the following backwards incompatibilities from 0.4 to 0.5.  The
 main areas of changed are: choices in fields, map_reduce and collection names.
 
 Choice options:
---------------
+---------------
 
 Are now expected to be an iterable of tuples, with  the first element in each
 tuple being the actual value to be stored. The second element is the
@@ -19,11 +98,11 @@ human-readable name for the option.
 PyMongo / MongoDB
 -----------------
 
-map reduce now requires pymongo 1.11+- The pymongo merge_output and reduce_output
-parameters, have been depreciated.
+map reduce now requires pymongo 1.11+- The pymongo `merge_output` and
+`reduce_output` parameters, have been depreciated.
 
-More methods now use map_reduce as db.eval is not supported for sharding as such
-the following have been changed:
+More methods now use map_reduce as db.eval is not supported for sharding as
+such the following have been changed:
 
     * :meth:`~mongoengine.queryset.QuerySet.sum`
     * :meth:`~mongoengine.queryset.QuerySet.average`
@@ -33,8 +112,8 @@ the following have been changed:
 Default collection naming
 -------------------------
 
-Previously it was just lowercase, its now much more pythonic and readable as its
-lowercase and underscores, previously ::
+Previously it was just lowercase, its now much more pythonic and readable as
+its lowercase and underscores, previously ::
 
     class MyAceDocument(Document):
         pass
@@ -58,7 +137,7 @@ To upgrade use a Mixin class to set meta like so ::
     class MyAceDocument(Document, BaseMixin):
         pass
 
-    MyAceDocument._get_collection_name() == myacedocument
+    MyAceDocument._get_collection_name() == "myacedocument"
 
 Alternatively, you can rename your collections eg ::
 
@@ -70,7 +149,8 @@ Alternatively, you can rename your collections eg ::
 
         failure = False
 
-        collection_names = [d._get_collection_name() for d in _document_registry.values()]
+        collection_names = [d._get_collection_name()
+                            for d in _document_registry.values()]
 
         for new_style_name in collection_names:
             if not new_style_name:  # embedded documents don't have collections
@@ -88,7 +168,8 @@ Alternatively, you can rename your collections eg ::
                         old_style_name, new_style_name)
                 else:
                     db[old_style_name].rename(new_style_name)
-                    print "Renamed:  %s to %s" % (old_style_name, new_style_name)
+                    print "Renamed:  %s to %s" % (old_style_name,
+                                                  new_style_name)
 
         if failure:
             print "Upgrading  collection names failed"

mongoengine/__init__.py

@@ -12,15 +12,12 @@ from signals import *
 __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
            queryset.__all__ + signals.__all__)
 
-__author__ = 'Harry Marr'
-
-VERSION = (0, 5, 2)
+VERSION = (0, 7, 2)
 
 
 def get_version():
-    version = '%s.%s' % (VERSION[0], VERSION[1])
-    if VERSION[2]:
-        version = '%s.%s' % (version, VERSION[2])
-    return version
+    if isinstance(VERSION[-1], basestring):
+        return '.'.join(map(str, VERSION[:-1])) + VERSION[-1]
+    return '.'.join(map(str, VERSION))
 
 __version__ = get_version()

mongoengine/connection.py

@@ -1,82 +1,166 @@
-from pymongo import Connection
-import multiprocessing
-import threading
-
-__all__ = ['ConnectionError', 'connect']
+import pymongo
+from pymongo import Connection, ReplicaSetConnection, uri_parser
 
 
-_connection_defaults = {
-    'host': 'localhost',
-    'port': 27017,
-}
-_connection = {}
-_connection_settings = _connection_defaults.copy()
+__all__ = ['ConnectionError', 'connect', 'register_connection',
+           'DEFAULT_CONNECTION_NAME']
 
-_db_name = None
-_db_username = None
-_db_password = None
-_db = {}
+
+DEFAULT_CONNECTION_NAME = 'default'
 
 
 class ConnectionError(Exception):
     pass
 
 
-def _get_connection(reconnect=False):
-    """Handles the connection to the database
+_connection_settings = {}
+_connections = {}
+_dbs = {}
+
+
+def register_connection(alias, name, host='localhost', port=27017,
+                        is_slave=False, read_preference=False, slaves=None,
+                        username=None, password=None, **kwargs):
+    """Add a connection.
+
+    :param alias: the name that will be used to refer to this connection
+        throughout MongoEngine
+    :param name: the name of the specific database to use
+    :param host: the host name of the :program:`mongod` instance to connect to
+    :param port: the port that the :program:`mongod` instance is running on
+    :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+
+    :param read_preference: The read preference for the collection ** Added pymongo 2.1
+    :param slaves: a list of aliases of slave connections; each of these must
+        be a registered connection that has :attr:`is_slave` set to ``True``
+    :param username: username to authenticate with
+    :param password: password to authenticate with
+    :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver
+
     """
-    global _connection
-    identity = get_identity()
+    global _connection_settings
+
+    conn_settings = {
+        'name': name,
+        'host': host,
+        'port': port,
+        'is_slave': is_slave,
+        'slaves': slaves or [],
+        'username': username,
+        'password': password,
+        'read_preference': read_preference
+    }
+
+    # Handle uri style connections
+    if "://" in host:
+        uri_dict = uri_parser.parse_uri(host)
+        if uri_dict.get('database') is None:
+            raise ConnectionError("If using URI style connection include "\
+                                  "database name in string")
+        conn_settings.update({
+            'host': host,
+            'name': uri_dict.get('database'),
+            'username': uri_dict.get('username'),
+            'password': uri_dict.get('password'),
+            'read_preference': read_preference,
+        })
+        if "replicaSet" in host:
+            conn_settings['replicaSet'] = True
+
+    conn_settings.update(kwargs)
+    _connection_settings[alias] = conn_settings
+
+
+def disconnect(alias=DEFAULT_CONNECTION_NAME):
+    global _connections
+    global _dbs
+
+    if alias in _connections:
+        get_connection(alias=alias).disconnect()
+        del _connections[alias]
+    if alias in _dbs:
+        del _dbs[alias]
+
+
+def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
+    global _connections
     # Connect to the database if not already connected
-    if _connection.get(identity) is None or reconnect:
+    if reconnect:
+        disconnect(alias)
+
+    if alias not in _connections:
+        if alias not in _connection_settings:
+            msg = 'Connection with alias "%s" has not been defined' % alias
+            if alias == DEFAULT_CONNECTION_NAME:
+                msg = 'You have not defined a default connection'
+            raise ConnectionError(msg)
+        conn_settings = _connection_settings[alias].copy()
+
+        if hasattr(pymongo, 'version_tuple'):  # Support for 2.1+
+            conn_settings.pop('name', None)
+            conn_settings.pop('slaves', None)
+            conn_settings.pop('is_slave', None)
+            conn_settings.pop('username', None)
+            conn_settings.pop('password', None)
+        else:
+            # Get all the slave connections
+            if 'slaves' in conn_settings:
+                slaves = []
+                for slave_alias in conn_settings['slaves']:
+                    slaves.append(get_connection(slave_alias))
+                conn_settings['slaves'] = slaves
+                conn_settings.pop('read_preference', None)
+
+        connection_class = Connection
+        if 'replicaSet' in conn_settings:
+            conn_settings['hosts_or_uri'] = conn_settings.pop('host', None)
+            # Discard port since it can't be used on ReplicaSetConnection
+            conn_settings.pop('port', None)
+            # Discard replicaSet if not base string
+            if not isinstance(conn_settings['replicaSet'], basestring):
+                conn_settings.pop('replicaSet', None)
+            connection_class = ReplicaSetConnection
+
         try:
-            _connection[identity] = Connection(**_connection_settings)
+            _connections[alias] = connection_class(**conn_settings)
         except Exception, e:
-            raise ConnectionError("Cannot connect to the database:\n%s" % e)
-    return _connection[identity]
+            raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e))
+    return _connections[alias]
 
-def _get_db(reconnect=False):
-    """Handles database connections and authentication based on the current
-    identity
+
+def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
+    global _dbs
+    if reconnect:
+        disconnect(alias)
+
+    if alias not in _dbs:
+        conn = get_connection(alias)
+        conn_settings = _connection_settings[alias]
+        _dbs[alias] = conn[conn_settings['name']]
+        # Authenticate if necessary
+        if conn_settings['username'] and conn_settings['password']:
+            _dbs[alias].authenticate(conn_settings['username'],
+                                     conn_settings['password'])
+    return _dbs[alias]
+
+
+def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs):
+    """Connect to the database specified by the 'db' argument.
+
+    Connection settings may be provided here as well if the database is not
+    running on the default port on localhost. If authentication is needed,
+    provide username and password arguments as well.
+
+    Multiple databases are supported by using aliases.  Provide a separate
+    `alias` to connect to a different instance of :program:`mongod`.
+
+    .. versionchanged:: 0.6 - added multiple database support.
     """
-    global _db, _connection
-    identity = get_identity()
-    # Connect if not already connected
-    if _connection.get(identity) is None or reconnect:
-        _connection[identity] = _get_connection(reconnect=reconnect)
+    global _connections
+    if alias not in _connections:
+        register_connection(alias, db, **kwargs)
 
-    if _db.get(identity) is None or reconnect:
-        # _db_name will be None if the user hasn't called connect()
-        if _db_name is None:
-            raise ConnectionError('Not connected to the database')
-
-        # Get DB from current connection and authenticate if necessary
-        _db[identity] = _connection[identity][_db_name]
-        if _db_username and _db_password:
-            _db[identity].authenticate(_db_username, _db_password)
-
-    return _db[identity]
-
-def get_identity():
-    """Creates an identity key based on the current process and thread
-    identity.
-    """
-    identity = multiprocessing.current_process()._identity
-    identity = 0 if not identity else identity[0]
-
-    identity = (identity, threading.current_thread().ident)
-    return identity
-
-def connect(db, username=None, password=None, **kwargs):
-    """Connect to the database specified by the 'db' argument. Connection
-    settings may be provided here as well if the database is not running on
-    the default port on localhost. If authentication is needed, provide
-    username and password arguments as well.
-    """
-    global _connection_settings, _db_name, _db_username, _db_password, _db
-    _connection_settings = dict(_connection_defaults, **kwargs)
-    _db_name = db
-    _db_username = username
-    _db_password = password
-    return _get_db(reconnect=True)
+    return get_connection(alias)
 
+# Support old naming convention
+_get_connection = get_connection
+_get_db = get_db

mongoengine/dereference.py

@@ -1,17 +1,15 @@
-import operator
+from bson import DBRef, SON
 
-import pymongo
-
-from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass
-from fields import ReferenceField
-from connection import _get_db
+from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document)
+from fields import (ReferenceField, ListField, DictField, MapField)
+from connection import get_db
 from queryset import QuerySet
 from document import Document
 
 
 class DeReference(object):
 
-    def __call__(self, items, max_depth=1, instance=None, name=None, get=False):
+    def __call__(self, items, max_depth=1, instance=None, name=None):
         """
         Cheaply dereferences the items to a set depth.
         Also handles the convertion of complex data types.
@@ -36,16 +34,35 @@ class DeReference(object):
 
         doc_type = None
         if instance and instance._fields:
-            doc_type = instance._fields[name].field
+            doc_type = instance._fields[name]
+            if hasattr(doc_type, 'field'):
+                doc_type = doc_type.field
 
             if isinstance(doc_type, ReferenceField):
+                field = doc_type
                 doc_type = doc_type.document_type
-                if all([i.__class__ == doc_type for i in items]):
+                is_list = not hasattr(items, 'items')
+
+                if is_list and all([i.__class__ == doc_type for i in items]):
                     return items
+                elif not is_list and all([i.__class__ == doc_type
+                                         for i in items.values()]):
+                    return items
+                elif not field.dbref:
+                    if not hasattr(items, 'items'):
+                        items = [field.to_python(v)
+                             if not isinstance(v, (DBRef, Document)) else v
+                             for v in items]
+                    else:
+                        items = dict([
+                            (k, field.to_python(v))
+                            if not isinstance(v, (DBRef, Document)) else (k, v)
+                            for k, v in items.iteritems()]
+                        )
 
         self.reference_map = self._find_references(items)
         self.object_map = self._fetch_objects(doc_type=doc_type)
-        return self._attach_objects(items, 0, instance, name, get)
+        return self._attach_objects(items, 0, instance, name)
 
     def _find_references(self, items, depth=0):
         """
@@ -55,7 +72,7 @@ class DeReference(object):
         :param depth: The current depth of recursion
         """
         reference_map = {}
-        if not items:
+        if not items or depth >= self.max_depth:
             return reference_map
 
         # Determine the iterator to use
@@ -65,13 +82,14 @@ class DeReference(object):
             iterator = items.iteritems()
 
         # Recursively find dbreferences
+        depth += 1
         for k, item in iterator:
             if hasattr(item, '_fields'):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, (pymongo.dbref.DBRef)):
+                    if isinstance(v, (DBRef)):
                         reference_map.setdefault(field.document_type, []).append(v.id)
-                    elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v:
+                    elif isinstance(v, (dict, SON)) and '_ref' in v:
                         reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id)
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                         field_cls = getattr(getattr(field, 'field', None), 'document_type', None)
@@ -80,15 +98,15 @@ class DeReference(object):
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
                             reference_map.setdefault(key, []).extend(refs)
-            elif isinstance(item, (pymongo.dbref.DBRef)):
+            elif isinstance(item, (DBRef)):
                 reference_map.setdefault(item.collection, []).append(item.id)
-            elif isinstance(item, (dict, pymongo.son.SON)) and '_ref' in item:
+            elif isinstance(item, (dict, SON)) and '_ref' in item:
                 reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id)
-            elif isinstance(item, (dict, list, tuple)) and depth <= self.max_depth:
-                references = self._find_references(item, depth)
+            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:
+                references = self._find_references(item, depth - 1)
                 for key, refs in references.iteritems():
                     reference_map.setdefault(key, []).extend(refs)
-        depth += 1
+
         return reference_map
 
     def _fetch_objects(self, doc_type=None):
@@ -103,16 +121,26 @@ class DeReference(object):
                 for key, doc in references.iteritems():
                     object_map[key] = doc
             else:  # Generic reference: use the refs data to convert to document
-                references = _get_db()[col].find({'_id': {'$in': refs}})
-                for ref in references:
-                    if '_cls' in ref:
-                        doc = get_document(ref['_cls'])._from_son(ref)
-                    else:
+                if doc_type and not isinstance(doc_type, (ListField, DictField, MapField,) ):
+                    references = doc_type._get_db()[col].find({'_id': {'$in': refs}})
+                    for ref in references:
                         doc = doc_type._from_son(ref)
-                    object_map[doc.id] = doc
+                        object_map[doc.id] = doc
+                else:
+                    references = get_db()[col].find({'_id': {'$in': refs}})
+                    for ref in references:
+                        if '_cls' in ref:
+                            doc = get_document(ref["_cls"])._from_son(ref)
+                        elif doc_type is None:
+                            doc = get_document(
+                                ''.join(x.capitalize()
+                                        for x in col.split('_')))._from_son(ref)
+                        else:
+                            doc = doc_type._from_son(ref)
+                        object_map[doc.id] = doc
         return object_map
 
-    def _attach_objects(self, items, depth=0, instance=None, name=None, get=False):
+    def _attach_objects(self, items, depth=0, instance=None, name=None):
         """
         Recursively finds all db references to be dereferenced
 
@@ -122,7 +150,6 @@ class DeReference(object):
             :class:`~mongoengine.base.ComplexBaseField`
         :param name: The name of the field, used for tracking changes by
             :class:`~mongoengine.base.ComplexBaseField`
-        :param get: A boolean determining if being called by __get__
         """
         if not items:
             if isinstance(items, (BaseDict, BaseList)):
@@ -130,17 +157,16 @@ class DeReference(object):
 
             if instance:
                 if isinstance(items, dict):
-                    return BaseDict(items, instance=instance, name=name)
+                    return BaseDict(items, instance, name)
                 else:
-                    return BaseList(items, instance=instance, name=name)
+                    return BaseList(items, instance, name)
 
-        if isinstance(items, (dict, pymongo.son.SON)):
+        if isinstance(items, (dict, SON)):
             if '_ref' in items:
                 return self.object_map.get(items['_ref'].id, items)
             elif '_types' in items and '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
-                if not get:
-                    doc._data = self._attach_objects(doc._data, depth, doc, name, get)
+                doc._data = self._attach_objects(doc._data, depth, doc, name)
                 return doc
 
         if not hasattr(items, 'items'):
@@ -152,35 +178,34 @@ class DeReference(object):
             iterator = items.iteritems()
             data = {}
 
+        depth += 1
         for k, v in iterator:
             if is_list:
                 data.append(v)
             else:
                 data[k] = v
 
-            if k in self.object_map:
+            if k in self.object_map and not is_list:
                 data[k] = self.object_map[k]
             elif hasattr(v, '_fields'):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
-                    if isinstance(v, (pymongo.dbref.DBRef)):
+                    if isinstance(v, (DBRef)):
                         data[k]._data[field_name] = self.object_map.get(v.id, v)
-                    elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v:
+                    elif isinstance(v, (dict, SON)) and '_ref' in v:
                         data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
-                    elif isinstance(v, dict) and depth < self.max_depth:
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
-                    elif isinstance(v, (list, tuple)):
-                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
-            elif isinstance(v, (dict, list, tuple)) and depth < self.max_depth:
-                data[k] = self._attach_objects(v, depth, instance=instance, name=name, get=get)
+                    elif isinstance(v, dict) and depth <= self.max_depth:
+                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+                    elif isinstance(v, (list, tuple)) and depth <= self.max_depth:
+                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
+            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
+                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name)
             elif hasattr(v, 'id'):
                 data[k] = self.object_map.get(v.id, v)
 
         if instance and name:
             if is_list:
-                return BaseList(data, instance=instance, name=name)
-            return BaseDict(data, instance=instance, name=name)
+                return BaseList(data, instance, name)
+            return BaseDict(data, instance, name)
         depth += 1
         return data
-
-dereference = DeReference()

mongoengine/django/auth.py

@@ -1,23 +1,39 @@
+import datetime
+
 from mongoengine import *
 
-from django.utils.hashcompat import md5_constructor, sha_constructor
 from django.utils.encoding import smart_str
 from django.contrib.auth.models import AnonymousUser
 from django.utils.translation import ugettext_lazy as _
 
-import datetime
+try:
+    from django.contrib.auth.hashers import check_password, make_password
+except ImportError:
+    """Handle older versions of Django"""
+    from django.utils.hashcompat import md5_constructor, sha_constructor
+
+    def get_hexdigest(algorithm, salt, raw_password):
+        raw_password, salt = smart_str(raw_password), smart_str(salt)
+        if algorithm == 'md5':
+            return md5_constructor(salt + raw_password).hexdigest()
+        elif algorithm == 'sha1':
+            return sha_constructor(salt + raw_password).hexdigest()
+        raise ValueError('Got unknown password algorithm type in password')
+
+    def check_password(raw_password, password):
+        algo, salt, hash = password.split('$')
+        return hash == get_hexdigest(algo, salt, raw_password)
+
+    def make_password(raw_password):
+        from random import random
+        algo = 'sha1'
+        salt = get_hexdigest(algo, str(random()), str(random()))[:5]
+        hash = get_hexdigest(algo, salt, raw_password)
+        return '%s$%s$%s' % (algo, salt, hash)
+
 
 REDIRECT_FIELD_NAME = 'next'
 
-def get_hexdigest(algorithm, salt, raw_password):
-    raw_password, salt = smart_str(raw_password), smart_str(salt)
-    if algorithm == 'md5':
-        return md5_constructor(salt + raw_password).hexdigest()
-    elif algorithm == 'sha1':
-        return sha_constructor(salt + raw_password).hexdigest()
-    raise ValueError('Got unknown password algorithm type in password')
-
-
 class User(Document):
     """A User document that aims to mirror most of the API specified by Django
     at http://docs.djangoproject.com/en/dev/topics/auth/#users
@@ -34,7 +50,7 @@ class User(Document):
     email = EmailField(verbose_name=_('e-mail address'))
     password = StringField(max_length=128,
                            verbose_name=_('password'),
-                           help_text=_("Use '[algo]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."))
+                           help_text=_("Use '[algo]$[iterations]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."))
     is_staff = BooleanField(default=False,
                             verbose_name=_('staff status'),
                             help_text=_("Designates whether the user can log into this admin site."))
@@ -50,6 +66,7 @@ class User(Document):
                                 verbose_name=_('date joined'))
 
     meta = {
+        'allow_inheritance': True,
         'indexes': [
             {'fields': ['username'], 'unique': True}
         ]
@@ -75,11 +92,7 @@ class User(Document):
         assigning to :attr:`~mongoengine.django.auth.User.password` as the
         password is hashed before storage.
         """
-        from random import random
-        algo = 'sha1'
-        salt = get_hexdigest(algo, str(random()), str(random()))[:5]
-        hash = get_hexdigest(algo, salt, raw_password)
-        self.password = '%s$%s$%s' % (algo, salt, hash)
+        self.password = make_password(raw_password)
         self.save()
         return self
 
@@ -89,8 +102,7 @@ class User(Document):
         :attr:`~mongoengine.django.auth.User.password` as the password is
         hashed before storage.
         """
-        algo, salt, hash = self.password.split('$')
-        return hash == get_hexdigest(algo, salt, raw_password)
+        return check_password(raw_password, self.password)
 
     @classmethod
     def create_user(cls, username, password, email=None):

mongoengine/django/sessions.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+from django.conf import settings
 from django.contrib.sessions.backends.base import SessionBase, CreateError
 from django.core.exceptions import SuspiciousOperation
 from django.utils.encoding import force_unicode
@@ -5,16 +8,22 @@ from django.utils.encoding import force_unicode
 from mongoengine.document import Document
 from mongoengine import fields
 from mongoengine.queryset import OperationError
+from mongoengine.connection import DEFAULT_CONNECTION_NAME
 
-from datetime import datetime
+
+MONGOENGINE_SESSION_DB_ALIAS = getattr(
+    settings, 'MONGOENGINE_SESSION_DB_ALIAS',
+    DEFAULT_CONNECTION_NAME)
 
 
 class MongoSession(Document):
     session_key = fields.StringField(primary_key=True, max_length=40)
     session_data = fields.StringField()
     expire_date = fields.DateTimeField()
-    
-    meta = {'collection': 'django_session', 'allow_inheritance': False}
+
+    meta = {'collection': 'django_session',
+            'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
+            'allow_inheritance': False}
 
 
 class SessionStore(SessionBase):
@@ -35,7 +44,7 @@ class SessionStore(SessionBase):
 
     def create(self):
         while True:
-            self.session_key = self._get_new_session_key()
+            self._session_key = self._get_new_session_key()
             try:
                 self.save(must_create=True)
             except CreateError:
@@ -45,6 +54,8 @@ class SessionStore(SessionBase):
             return
 
     def save(self, must_create=False):
+        if self.session_key is None:
+            self._session_key = self._get_new_session_key()
         s = MongoSession(session_key=self.session_key)
         s.session_data = self.encode(self._get_session(no_load=must_create))
         s.expire_date = self.get_expiry_date()

mongoengine/django/shortcuts.py

@@ -1,4 +1,3 @@
-from django.http import Http404
 from mongoengine.queryset import QuerySet
 from mongoengine.base import BaseDocument
 from mongoengine.base import ValidationError
@@ -27,6 +26,7 @@ def get_document_or_404(cls, *args, **kwargs):
     try:
         return queryset.get(*args, **kwargs)
     except (queryset._document.DoesNotExist, ValidationError):
+        from django.http import Http404
         raise Http404('No %s matches the given query.' % queryset._document._class_name)
 
 def get_list_or_404(cls, *args, **kwargs):
@@ -42,5 +42,6 @@ def get_list_or_404(cls, *args, **kwargs):
     queryset = _get_queryset(cls)
     obj_list = list(queryset.filter(*args, **kwargs))
     if not obj_list:
+        from django.http import Http404
         raise Http404('No %s matches the given query.' % queryset._document._class_name)
     return obj_list

mongoengine/django/tests.py

@@ -1,16 +1,34 @@
 #coding: utf-8
-from django.test import TestCase
-from django.conf import settings
+from nose.plugins.skip import SkipTest
 
+from mongoengine.python_support import PY3
 from mongoengine import connect
 
+try:
+    from django.test import TestCase
+    from django.conf import settings
+except Exception as err:
+    if PY3:
+        from unittest import TestCase
+        # Dummy value so no error
+        class settings:
+            MONGO_DATABASE_NAME = 'dummy'
+    else:
+        raise err
+
+
 class MongoTestCase(TestCase):
+
+    def setUp(self):
+        if PY3:
+            raise SkipTest('django does not have Python 3 support')
+
     """
     TestCase class that clear the collection between the tests
     """
     db_name = 'test_%s' % settings.MONGO_DATABASE_NAME
     def __init__(self, methodName='runtest'):
-        self.db = connect(self.db_name)
+        self.db = connect(self.db_name).get_db()
         super(MongoTestCase, self).__init__(methodName)
 
     def _post_teardown(self):

mongoengine/document.py

@@ -1,13 +1,19 @@
-from mongoengine import signals
-from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument,
-                  ValidationError, BaseDict, BaseList)
-from queryset import OperationError
-from connection import _get_db
+import warnings
 
 import pymongo
+import re
 
-__all__ = ['Document', 'EmbeddedDocument', 'ValidationError',
-           'OperationError', 'InvalidCollectionError']
+from bson.dbref import DBRef
+from mongoengine import signals, queryset
+
+from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument,
+                  BaseDict, BaseList)
+from queryset import OperationError, NotUniqueError
+from connection import get_db, DEFAULT_CONNECTION_NAME
+
+__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument',
+           'DynamicEmbeddedDocument', 'OperationError',
+           'InvalidCollectionError', 'NotUniqueError']
 
 
 class InvalidCollectionError(Exception):
@@ -21,8 +27,15 @@ class EmbeddedDocument(BaseDocument):
     :class:`~mongoengine.EmbeddedDocumentField` field type.
     """
 
+    # The __metaclass__ attribute is removed by 2to3 when running with Python3
+    # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
+    my_metaclass  = DocumentMetaclass
     __metaclass__ = DocumentMetaclass
 
+    def __init__(self, *args, **kwargs):
+        super(EmbeddedDocument, self).__init__(*args, **kwargs)
+        self._changed_fields = []
+
     def __delattr__(self, *args, **kwargs):
         """Handle deletions of fields"""
         field_name = args[0]
@@ -34,6 +47,10 @@ class EmbeddedDocument(BaseDocument):
         else:
             super(EmbeddedDocument, self).__delattr__(*args, **kwargs)
 
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return self._data == other._data
+        return False
 
 
 class Document(BaseDocument):
@@ -70,49 +87,76 @@ class Document(BaseDocument):
     names. Index direction may be specified by prefixing the field names with
     a **+** or **-** sign.
 
+    Automatic index creation can be disabled by specifying
+    attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to
+    False then indexes will not be created by MongoEngine.  This is useful in
+    production systems where index creation is performed as part of a deployment
+    system.
+
     By default, _types will be added to the start of every index (that
-    doesn't contain a list) if allow_inheritence is True. This can be
+    doesn't contain a list) if allow_inheritance is True. This can be
     disabled by either setting types to False on the specific index or
     by setting index_types to False on the meta dictionary for the document.
     """
+
+    # The __metaclass__ attribute is removed by 2to3 when running with Python3
+    # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
+    my_metaclass  = TopLevelDocumentMetaclass
     __metaclass__ = TopLevelDocumentMetaclass
 
-    @classmethod
-    def _get_collection(self):
-        """Returns the collection for the document."""
-        db = _get_db()
-        collection_name = self._get_collection_name()
+    def pk():
+        """Primary key alias
+        """
+        def fget(self):
+            return getattr(self, self._meta['id_field'])
+        def fset(self, value):
+            return setattr(self, self._meta['id_field'], value)
+        return property(fget, fset)
+    pk = pk()
 
-        if not hasattr(self, '_collection') or self._collection is None:
+    @classmethod
+    def _get_db(cls):
+        """Some Model using other db_alias"""
+        return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME ))
+
+    @classmethod
+    def _get_collection(cls):
+        """Returns the collection for the document."""
+        if not hasattr(cls, '_collection') or cls._collection is None:
+            db = cls._get_db()
+            collection_name = cls._get_collection_name()
             # Create collection as a capped collection if specified
-            if self._meta['max_size'] or self._meta['max_documents']:
+            if cls._meta['max_size'] or cls._meta['max_documents']:
                 # Get max document limit and max byte size from meta
-                max_size = self._meta['max_size'] or 10000000  # 10MB default
-                max_documents = self._meta['max_documents']
+                max_size = cls._meta['max_size'] or 10000000  # 10MB default
+                max_documents = cls._meta['max_documents']
 
                 if collection_name in db.collection_names():
-                    self._collection = db[collection_name]
+                    cls._collection = db[collection_name]
                     # The collection already exists, check if its capped
                     # options match the specified capped options
-                    options = self._collection.options()
+                    options = cls._collection.options()
                     if options.get('max') != max_documents or \
                        options.get('size') != max_size:
-                        msg = ('Cannot create collection "%s" as a capped '
-                               'collection as it already exists') % self._collection
+                        msg = (('Cannot create collection "%s" as a capped '
+                               'collection as it already exists')
+                                % cls._collection)
                         raise InvalidCollectionError(msg)
                 else:
                     # Create the collection as a capped collection
                     opts = {'capped': True, 'size': max_size}
                     if max_documents:
                         opts['max'] = max_documents
-                    self._collection = db.create_collection(
+                    cls._collection = db.create_collection(
                         collection_name, **opts
                     )
             else:
-                self._collection = db[collection_name]
-        return self._collection
+                cls._collection = db[collection_name]
+        return cls._collection
 
-    def save(self, safe=True, force_insert=False, validate=True, write_options=None, _refs=None):
+    def save(self, safe=True, force_insert=False, validate=True,
+             write_options=None,  cascade=None, cascade_kwargs=None,
+             _refs=None):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -125,19 +169,31 @@ class Document(BaseDocument):
             updates of existing documents
         :param validate: validates the document; set to ``False`` to skip.
         :param write_options: Extra keyword arguments are passed down to
-                :meth:`~pymongo.collection.Collection.save` OR
-                :meth:`~pymongo.collection.Collection.insert`
-                which will be used as options for the resultant ``getLastError`` command.
-                For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers
-                have recorded the write and will force an fsync on each server being written to.
+            :meth:`~pymongo.collection.Collection.save` OR
+            :meth:`~pymongo.collection.Collection.insert`
+            which will be used as options for the resultant
+            ``getLastError`` command.  For example,
+            ``save(..., write_options={w: 2, fsync: True}, ...)`` will
+            wait until at least two servers have recorded the write and
+            will force an fsync on the primary server.
+        :param cascade: Sets the flag for cascading saves.  You can set a
+            default by setting "cascade" in the document __meta__
+        :param cascade_kwargs: optional kwargs dictionary to be passed throw
+            to cascading saves
+        :param _refs: A list of processed references used in cascading saves
 
         .. versionchanged:: 0.5
-            In existing documents it only saves changed fields using set / unset
-            Saves are cascaded and any :class:`~pymongo.dbref.DBRef` objects
-            that have changes are saved as well.
+            In existing documents it only saves changed fields using
+            set / unset.  Saves are cascaded and any
+            :class:`~bson.dbref.DBRef` objects that have changes are
+            saved as well.
+        .. versionchanged:: 0.6
+            Cascade saves are optional = defaults to True, if you want
+            fine grain control then you can turn off using document
+            meta['cascade'] = False  Also you can pass different kwargs to
+            the cascade save using cascade_kwargs which overwrites the
+            existing kwargs with custom values
         """
-        from fields import ReferenceField, GenericReferenceField
-
         signals.pre_save.send(self.__class__, document=self)
 
         if validate:
@@ -148,57 +204,106 @@ class Document(BaseDocument):
 
         doc = self.to_mongo()
 
-        created = '_id' in doc
-        creation_mode = force_insert or not created
+        created = force_insert or '_id' not in doc
+
         try:
             collection = self.__class__.objects._collection
-            if creation_mode:
+            if created:
                 if force_insert:
-                    object_id = collection.insert(doc, safe=safe, **write_options)
+                    object_id = collection.insert(doc, safe=safe,
+                                                  **write_options)
                 else:
-                    object_id = collection.save(doc, safe=safe, **write_options)
+                    object_id = collection.save(doc, safe=safe,
+                                                **write_options)
             else:
                 object_id = doc['_id']
                 updates, removals = self._delta()
-                if updates:
-                    collection.update({'_id': object_id}, {"$set": updates}, upsert=True, safe=safe, **write_options)
-                if removals:
-                    collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options)
+                # Need to add shard key to query, or you get an error
+                select_dict = {'_id': object_id}
+                shard_key = self.__class__._meta.get('shard_key', tuple())
+                for k in shard_key:
+                    actual_key = self._db_field_map.get(k, k)
+                    select_dict[actual_key] = doc[actual_key]
 
-            # Save any references / generic references
-            _refs = _refs or []
-            for name, cls in self._fields.items():
-                if isinstance(cls, (ReferenceField, GenericReferenceField)):
-                    ref = getattr(self, name)
-                    if ref and str(ref) not in _refs:
-                        _refs.append(str(ref))
-                        ref.save(safe=safe, force_insert=force_insert,
-                                 validate=validate, write_options=write_options,
-                                 _refs=_refs)
+                upsert = self._created
+                if updates:
+                    collection.update(select_dict, {"$set": updates},
+                        upsert=upsert, safe=safe, **write_options)
+                if removals:
+                    collection.update(select_dict, {"$unset": removals},
+                        upsert=upsert, safe=safe, **write_options)
+
+            warn_cascade = not cascade and 'cascade' not in self._meta
+            cascade = (self._meta.get('cascade', True)
+                       if cascade is None else cascade)
+            if cascade:
+                kwargs = {
+                    "safe": safe,
+                    "force_insert": force_insert,
+                    "validate": validate,
+                    "write_options": write_options,
+                    "cascade": cascade
+                }
+                if cascade_kwargs:  # Allow granular control over cascades
+                    kwargs.update(cascade_kwargs)
+                kwargs['_refs'] = _refs
+                self.cascade_save(warn_cascade=warn_cascade, **kwargs)
 
         except pymongo.errors.OperationFailure, err:
             message = 'Could not save document (%s)'
-            if u'duplicate key' in unicode(err):
+            if re.match('^E1100[01] duplicate key', unicode(err)):
+                # E11000 - duplicate key error index
+                # E11001 - duplicate key on update
                 message = u'Tried to save duplicate unique keys (%s)'
+                raise NotUniqueError(message % unicode(err))
             raise OperationError(message % unicode(err))
         id_field = self._meta['id_field']
-        self[id_field] = self._fields[id_field].to_python(object_id)
+        if id_field not in self._meta.get('shard_key', []):
+            self[id_field] = self._fields[id_field].to_python(object_id)
 
-        def reset_changed_fields(doc, inspected_docs=None):
-            """Loop through and reset changed fields lists"""
+        self._changed_fields = []
+        self._created = False
+        signals.post_save.send(self.__class__, document=self, created=created)
+        return self
 
-            inspected_docs = inspected_docs or []
-            inspected_docs.append(doc)
-            if hasattr(doc, '_changed_fields'):
-                doc._changed_fields = []
+    def cascade_save(self, warn_cascade=None, *args, **kwargs):
+        """Recursively saves any references /
+           generic references on an objects"""
+        import fields
+        _refs = kwargs.get('_refs', []) or []
 
-            for field_name in doc._fields:
-                field = getattr(doc, field_name)
-                if field not in inspected_docs and hasattr(field, '_changed_fields'):
-                    reset_changed_fields(field, inspected_docs)
+        for name, cls in self._fields.items():
+            if not isinstance(cls, (fields.ReferenceField,
+                                    fields.GenericReferenceField)):
+                continue
 
-        reset_changed_fields(self)
-        signals.post_save.send(self.__class__, document=self, created=creation_mode)
+            ref = getattr(self, name)
+            if not ref or isinstance(ref, DBRef):
+                continue
+
+            if not getattr(ref, '_changed_fields', True):
+                continue
+
+            ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data))
+            if ref and ref_id not in _refs:
+                if warn_cascade:
+                    msg = ("Cascading saves will default to off in 0.8, "
+                          "please  explicitly set `.save(cascade=True)`")
+                    warnings.warn(msg, FutureWarning)
+                _refs.append(ref_id)
+                kwargs["_refs"] = _refs
+                ref.save(**kwargs)
+                ref._changed_fields = []
+
+    @property
+    def _object_key(self):
+        """Dict to identify object in collection
+        """
+        select_dict = {'pk': self.pk}
+        shard_key = self.__class__._meta.get('shard_key', tuple())
+        for k in shard_key:
+            select_dict[k] = getattr(self, k)
+        return select_dict
 
     def update(self, **kwargs):
         """Performs an update on the :class:`~mongoengine.Document`
@@ -210,7 +315,8 @@ class Document(BaseDocument):
         if not self.pk:
             raise OperationError('attempt to update a document not yet saved')
 
-        return self.__class__.objects(pk=self.pk).update_one(**kwargs)
+        # Need to add shard key to query, or you get an error
+        return self.__class__.objects(**self._object_key).update_one(**kwargs)
 
     def delete(self, safe=False):
         """Delete the :class:`~mongoengine.Document` from the database. This
@@ -220,10 +326,8 @@ class Document(BaseDocument):
         """
         signals.pre_delete.send(self.__class__, document=self)
 
-        id_field = self._meta['id_field']
-        object_id = self._fields[id_field].to_mongo(self[id_field])
         try:
-            self.__class__.objects(**{id_field: object_id}).delete(safe=safe)
+            self.__class__.objects(**self._object_key).delete(safe=safe)
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
@@ -231,62 +335,123 @@ class Document(BaseDocument):
         signals.post_delete.send(self.__class__, document=self)
 
     def select_related(self, max_depth=1):
-        """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to
+        """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to
         a maximum depth in order to cut down the number queries to mongodb.
 
         .. versionadded:: 0.5
         """
-        from dereference import dereference
-        self._data = dereference(self._data, max_depth)
+        import dereference
+        self._data = dereference.DeReference()(self._data, max_depth)
         return self
 
-    def reload(self):
+    def reload(self, max_depth=1):
         """Reloads all attributes from the database.
 
         .. versionadded:: 0.1.2
+        .. versionchanged:: 0.6  Now chainable
         """
         id_field = self._meta['id_field']
-        obj = self.__class__.objects(**{id_field: self[id_field]}).first()
+        obj = self.__class__.objects(
+                **{id_field: self[id_field]}
+              ).first().select_related(max_depth=max_depth)
         for field in self._fields:
             setattr(self, field, self._reload(field, obj[field]))
-        self._changed_fields = []
+        if self._dynamic:
+            for name in self._dynamic_fields.keys():
+                setattr(self, name, self._reload(name, obj._data[name]))
+        self._changed_fields = obj._changed_fields
+        return obj
 
     def _reload(self, key, value):
         """Used by :meth:`~mongoengine.Document.reload` to ensure the
         correct instance is linked to self.
         """
         if isinstance(value, BaseDict):
-            value = [(k, self._reload(k,v)) for k,v in value.items()]
-            value = BaseDict(value, instance=self, name=key)
+            value = [(k, self._reload(k, v)) for k, v in value.items()]
+            value = BaseDict(value, self, key)
         elif isinstance(value, BaseList):
             value = [self._reload(key, v) for v in value]
-            value = BaseList(value, instance=self, name=key)
-        elif isinstance(value, EmbeddedDocument):
+            value = BaseList(value, self, key)
+        elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)):
             value._changed_fields = []
         return value
 
     def to_dbref(self):
-        """Returns an instance of :class:`~pymongo.dbref.DBRef` useful in
+        """Returns an instance of :class:`~bson.dbref.DBRef` useful in
         `__raw__` queries."""
         if not self.pk:
             msg = "Only saved documents can have a valid dbref"
             raise OperationError(msg)
-        return pymongo.dbref.DBRef(self.__class__._get_collection_name(), self.pk)
+        return DBRef(self.__class__._get_collection_name(), self.pk)
 
     @classmethod
     def register_delete_rule(cls, document_cls, field_name, rule):
         """This method registers the delete rules to apply when removing this
         object.
         """
-        cls._meta['delete_rules'][(document_cls, field_name)] = rule
+        delete_rules = cls._meta.get('delete_rules') or {}
+        delete_rules[(document_cls, field_name)] = rule
+        cls._meta['delete_rules'] = delete_rules
 
     @classmethod
     def drop_collection(cls):
         """Drops the entire collection associated with this
         :class:`~mongoengine.Document` type from the database.
         """
-        db = _get_db()
+        db = cls._get_db()
         db.drop_collection(cls._get_collection_name())
+        queryset.QuerySet._reset_already_indexed(cls)
+
+
+class DynamicDocument(Document):
+    """A Dynamic Document class allowing flexible, expandable and uncontrolled
+    schemas.  As a :class:`~mongoengine.Document` subclass, acts in the same
+    way as an ordinary document but has expando style properties.  Any data
+    passed or set against the :class:`~mongoengine.DynamicDocument` that is
+    not a field is automatically converted into a
+    :class:`~mongoengine.DynamicField` and data can be attributed to that
+    field.
+
+    .. note::
+
+        There is one caveat on Dynamic Documents: fields cannot start with `_`
+    """
+
+    # The __metaclass__ attribute is removed by 2to3 when running with Python3
+    # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
+    my_metaclass  = TopLevelDocumentMetaclass
+    __metaclass__ = TopLevelDocumentMetaclass
+
+    _dynamic = True
+
+    def __delattr__(self, *args, **kwargs):
+        """Deletes the attribute by setting to None and allowing _delta to unset
+        it"""
+        field_name = args[0]
+        if field_name in self._dynamic_fields:
+            setattr(self, field_name, None)
+        else:
+            super(DynamicDocument, self).__delattr__(*args, **kwargs)
+
+
+class DynamicEmbeddedDocument(EmbeddedDocument):
+    """A Dynamic Embedded Document class allowing flexible, expandable and
+    uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more
+    information about dynamic documents.
+    """
+
+    # The __metaclass__ attribute is removed by 2to3 when running with Python3
+    # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
+    my_metaclass  = DocumentMetaclass
+    __metaclass__ = DocumentMetaclass
+
+    _dynamic = True
+
+    def __delattr__(self, *args, **kwargs):
+        """Deletes the attribute by setting to None and allowing _delta to unset
+        it"""
+        field_name = args[0]
+        setattr(self, field_name, None)
 
 
 class MapReduceDocument(object):
@@ -294,7 +459,7 @@ class MapReduceDocument(object):
 
     :param collection: An instance of :class:`~pymongo.Collection`
     :param key: Document/result key, often an instance of
-                :class:`~pymongo.objectid.ObjectId`. If supplied as
+                :class:`~bson.objectid.ObjectId`. If supplied as
                 an ``ObjectId`` found in the given ``collection``,
                 the object can be accessed via the ``object`` property.
     :param value: The result(s) for this key.

mongoengine/python_support.py

@@ -0,0 +1,61 @@
+"""Helper functions and types to aid with Python 2.5 - 3 support."""
+
+import sys
+
+PY3 = sys.version_info[0] == 3
+PY25 = sys.version_info[:2] == (2, 5)
+UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264
+
+if PY3:
+    import codecs
+    from io import BytesIO as StringIO
+    # return s converted to binary.  b('test') should be equivalent to b'test'
+    def b(s):
+        return codecs.latin_1_encode(s)[0]
+
+    bin_type = bytes
+    txt_type   = str
+else:
+    try:
+        from cStringIO import StringIO
+    except ImportError:
+        from StringIO import StringIO
+
+    # Conversion to binary only necessary in Python 3
+    def b(s):
+        return s
+
+    bin_type = str
+    txt_type = unicode
+
+str_types = (bin_type, txt_type)
+
+if PY25:
+    def product(*args, **kwds):
+        pools = map(tuple, args) * kwds.get('repeat', 1)
+        result = [[]]
+        for pool in pools:
+            result = [x + [y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+    reduce = reduce
+else:
+    from itertools import product
+    from functools import reduce
+
+
+# For use with Python 2.5
+# converts all keys from unicode to str for d and all nested dictionaries
+def to_str_keys_recursive(d):
+    if isinstance(d, list):
+        for val in d:
+            if isinstance(val, (dict, list)):
+                to_str_keys_recursive(val)
+    elif isinstance(d, dict):
+        for key, val in d.items():
+            if isinstance(val, (dict, list)):
+                to_str_keys_recursive(val)
+            if isinstance(key, unicode):
+                d[str(key)] = d.pop(key)
+    else:
+        raise ValueError("non list/dict parameter not allowed")

mongoengine/signals.py

@@ -42,3 +42,5 @@ pre_save = _signals.signal('pre_save')
 post_save = _signals.signal('post_save')
 pre_delete = _signals.signal('pre_delete')
 post_delete = _signals.signal('post_delete')
+pre_bulk_insert = _signals.signal('pre_bulk_insert')
+post_bulk_insert = _signals.signal('post_bulk_insert')

mongoengine/tests.py

@@ -1,4 +1,4 @@
-from mongoengine.connection import _get_db
+from mongoengine.connection import get_db
 
 
 class query_counter(object):
@@ -7,7 +7,7 @@ class query_counter(object):
     def __init__(self):
         """ Construct the query_counter. """
         self.counter = 0
-        self.db = _get_db()
+        self.db = get_db()
 
     def __enter__(self):
         """ On every with block we need to drop the profile collection. """

python-mongoengine.spec

@@ -0,0 +1,54 @@
+# sitelib for noarch packages, sitearch for others (remove the unneeded one)
+%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+
+%define srcname mongoengine
+
+Name:           python-%{srcname}
+Version:        0.7.2
+Release:        1%{?dist}
+Summary:        A Python Document-Object Mapper for working with MongoDB
+
+Group:          Development/Libraries
+License:        MIT
+URL:            https://github.com/MongoEngine/mongoengine
+Source0:        %{srcname}-%{version}.tar.bz2
+
+BuildRequires:  python-devel
+BuildRequires:  python-setuptools
+
+Requires:       mongodb
+Requires:       pymongo
+Requires:       python-blinker
+Requires:       python-imaging
+
+
+%description
+MongoEngine is an ORM-like layer on top of PyMongo.
+
+%prep
+%setup -q -n %{srcname}-%{version}
+
+
+%build
+# Remove CFLAGS=... for noarch packages (unneeded)
+CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root,-)
+%doc docs AUTHORS LICENSE README.rst
+# For noarch packages: sitelib
+ %{python_sitelib}/*
+# For arch-specific packages: sitearch
+# %{python_sitearch}/*
+
+%changelog
+* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html

requirements.txt

@@ -0,0 +1 @@
+pymongo

setup.cfg

@@ -0,0 +1,11 @@
+[nosetests]
+verbosity = 3
+detailed-errors = 1
+#with-coverage = 1
+#cover-erase = 1
+#cover-html = 1
+#cover-html-dir = ../htmlcov
+#cover-package = mongoengine
+py3where = build
+where = tests
+#tests =  test_bugfix.py

setup.py

@@ -1,27 +1,35 @@
-from setuptools import setup, find_packages
 import os
+import sys
+from setuptools import setup, find_packages
 
-DESCRIPTION = "A Python Document-Object Mapper for working with MongoDB"
+# Hack to silence atexit traceback in newer python versions
+try:
+    import multiprocessing
+except ImportError:
+    pass
 
+DESCRIPTION = """MongoEngine is a Python Object-Document
+Mapper for working with MongoDB."""
 LONG_DESCRIPTION = None
 try:
     LONG_DESCRIPTION = open('README.rst').read()
 except:
     pass
 
+
 def get_version(version_tuple):
-    version = '%s.%s' % (version_tuple[0], version_tuple[1])
-    if version_tuple[2]:
-        version = '%s.%s' % (version, version_tuple[2])
-    return version
+    if not isinstance(version_tuple[-1], int):
+        return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1]
+    return '.'.join(map(str, version_tuple))
 
 # Dirty hack to get version number from monogengine/__init__.py - we can't
 # import it as it depends on PyMongo and PyMongo isn't installed until this
 # file is read
 init = os.path.join(os.path.dirname(__file__), 'mongoengine', '__init__.py')
-version_line = filter(lambda l: l.startswith('VERSION'), open(init))[0]
+version_line = list(filter(lambda l: l.startswith('VERSION'), open(init)))[0]
+
 VERSION = get_version(eval(version_line.split('=')[-1]))
-print VERSION
+print(VERSION)
 
 CLASSIFIERS = [
     'Development Status :: 4 - Beta',
@@ -29,16 +37,38 @@ CLASSIFIERS = [
     'License :: OSI Approved :: MIT License',
     'Operating System :: OS Independent',
     'Programming Language :: Python',
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: Implementation :: CPython",
     'Topic :: Database',
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
 
+extra_opts = {}
+if sys.version_info[0] == 3:
+    extra_opts['use_2to3'] = True
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker']
+    extra_opts['packages'] = find_packages(exclude=('tests',))
+    if "test" in sys.argv or "nosetests" in sys.argv:
+        extra_opts['packages'].append("tests")
+        extra_opts['package_data'] = {"tests": ["mongoengine.png"]}
+else:
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL']
+    extra_opts['packages'] = find_packages(exclude=('tests',))
+
 setup(name='mongoengine',
       version=VERSION,
-      packages=find_packages(),
       author='Harry Marr',
       author_email='harry.marr@{nospam}gmail.com',
-      url='http://hmarr.com/mongoengine/',
+      maintainer="Ross Lawley",
+      maintainer_email="ross.lawley@{nospam}gmail.com",
+      url='http://mongoengine.org/',
+      download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
       license='MIT',
       include_package_data=True,
       description=DESCRIPTION,
@@ -46,6 +76,6 @@ setup(name='mongoengine',
       platforms=['any'],
       classifiers=CLASSIFIERS,
       install_requires=['pymongo'],
-      test_suite='tests',
-      tests_require=['blinker', 'django==1.3']
+      test_suite='nose.collector',
+      **extra_opts
 )

tests/fixtures.py

@@ -1,9 +1,6 @@
 from datetime import datetime
-import pymongo
 
 from mongoengine import *
-from mongoengine.base import BaseField
-from mongoengine.connection import _get_db
 
 
 class PickleEmbedded(EmbeddedDocument):
@@ -15,6 +12,7 @@ class PickleTest(Document):
     string = StringField(choices=(('One', '1'), ('Two', '2')))
     embedded = EmbeddedDocumentField(PickleEmbedded)
     lists = ListField(StringField())
+    photo = FileField()
 
 
 class Mixin(object):
@@ -22,4 +20,4 @@ class Mixin(object):
 
 
 class Base(Document):
-    pass
+    meta = {'allow_inheritance': True}

tests/test_all_warnings.py

@@ -0,0 +1,96 @@
+import unittest
+import warnings
+
+from mongoengine import *
+from mongoengine.tests import query_counter
+
+
+class TestWarnings(unittest.TestCase):
+
+    def setUp(self):
+        conn = connect(db='mongoenginetest')
+        self.warning_list = []
+        self.showwarning_default = warnings.showwarning
+        warnings.showwarning = self.append_to_warning_list
+
+    def append_to_warning_list(self, message, category, *args):
+        self.warning_list.append({"message": message,
+                                  "category": category})
+
+    def tearDown(self):
+        # restore default handling of warnings
+        warnings.showwarning = self.showwarning_default
+
+    def test_allow_inheritance_future_warning(self):
+        """Add FutureWarning for future allow_inhertiance default change.
+        """
+
+        class SimpleBase(Document):
+            a = IntField()
+
+        class InheritedClass(SimpleBase):
+            b = IntField()
+
+        InheritedClass()
+        self.assertEqual(len(self.warning_list), 1)
+        warning = self.warning_list[0]
+        self.assertEqual(FutureWarning, warning["category"])
+        self.assertTrue("InheritedClass" in str(warning["message"]))
+
+    def test_dbref_reference_field_future_warning(self):
+
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self')
+
+        Person.drop_collection()
+
+        p1 = Person()
+        p1.parent = None
+        p1.save()
+
+        p2 = Person(name="Wilson Jr")
+        p2.parent = p1
+        p2.save(cascade=False)
+
+        self.assertEqual(len(self.warning_list), 1)
+        warning = self.warning_list[0]
+        self.assertEqual(FutureWarning, warning["category"])
+        self.assertTrue("ReferenceFields will default to using ObjectId"
+                        in str(warning["message"]))
+
+    def test_document_save_cascade_future_warning(self):
+
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self')
+
+        Person.drop_collection()
+
+        p1 = Person(name="Wilson Snr")
+        p1.parent = None
+        p1.save()
+
+        p2 = Person(name="Wilson Jr")
+        p2.parent = p1
+        p2.parent.name = "Poppa Wilson"
+        p2.save()
+
+        self.assertEqual(len(self.warning_list), 1)
+        warning = self.warning_list[0]
+        self.assertEqual(FutureWarning, warning["category"])
+        self.assertTrue("Cascading saves will default to off in 0.8"
+                        in str(warning["message"]))
+
+    def test_document_collection_syntax_warning(self):
+
+        class NonAbstractBase(Document):
+            pass
+
+        class InheritedDocumentFailTest(NonAbstractBase):
+            meta = {'collection': 'fail'}
+
+        warning = self.warning_list[0]
+        self.assertEqual(SyntaxWarning, warning["category"])
+        self.assertEqual('non_abstract_base',
+                         InheritedDocumentFailTest._get_collection_name())

tests/test_connection.py

@@ -0,0 +1,98 @@
+import datetime
+import pymongo
+import unittest
+
+import mongoengine.connection
+
+from bson.tz_util import utc
+
+from mongoengine import *
+from mongoengine.connection import get_db, get_connection, ConnectionError
+
+
+class ConnectionTest(unittest.TestCase):
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    def test_connect(self):
+        """Ensure that the connect() method works properly.
+        """
+        connect('mongoenginetest')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+        connect('mongoenginetest2', alias='testdb')
+        conn = get_connection('testdb')
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+    def test_connect_uri(self):
+        """Ensure that the connect() method works properly with uri's
+        """
+        c = connect(db='mongoenginetest', alias='admin')
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
+        c.admin.add_user("admin", "password")
+        c.admin.authenticate("admin", "password")
+        c.mongoenginetest.add_user("username", "password")
+
+        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
+
+        connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+    def test_register_connection(self):
+        """Ensure that connections with different aliases may be registered.
+        """
+        register_connection('testdb', 'mongoenginetest2')
+
+        self.assertRaises(ConnectionError, get_connection)
+        conn = get_connection('testdb')
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = get_db('testdb')
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest2')
+
+    def test_connection_kwargs(self):
+        """Ensure that connection kwargs get passed to pymongo.
+        """
+        connect('mongoenginetest', alias='t1', tz_aware=True)
+        conn = get_connection('t1')
+
+        self.assertTrue(conn.tz_aware)
+
+        connect('mongoenginetest2', alias='t2')
+        conn = get_connection('t2')
+        self.assertFalse(conn.tz_aware)
+
+    def test_datetime(self):
+        connect('mongoenginetest', tz_aware=True)
+        d = datetime.datetime(2010, 5, 5, tzinfo=utc)
+
+        class DateDoc(Document):
+            the_date = DateTimeField(required=True)
+
+        DateDoc.drop_collection()
+        DateDoc(the_date=d).save()
+
+        date_doc = DateDoc.objects.first()
+        self.assertEqual(d, date_doc.the_date)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_dereference.py

@@ -1,7 +1,10 @@
+from __future__ import with_statement
 import unittest
 
+from bson import DBRef
+
 from mongoengine import *
-from mongoengine.connection import _get_db
+from mongoengine.connection import get_db
 from mongoengine.tests import query_counter
 
 
@@ -9,7 +12,7 @@ class FieldTest(unittest.TestCase):
 
     def setUp(self):
         connect(db='mongoenginetest')
-        self.db = _get_db()
+        self.db = get_db()
 
     def test_list_item_dereference(self):
         """Ensure that DBRef items in ListFields are dereferenced.
@@ -63,6 +66,130 @@ class FieldTest(unittest.TestCase):
         User.drop_collection()
         Group.drop_collection()
 
+    def test_list_item_dereference_dref_false(self):
+        """Ensure that DBRef items in ListFields are dereferenced.
+        """
+        class User(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = ListField(ReferenceField(User, dbref=False))
+
+        User.drop_collection()
+        Group.drop_collection()
+
+        for i in xrange(1, 51):
+            user = User(name='user %s' % i)
+            user.save()
+
+        group = Group(members=User.objects)
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 2)
+
+        # Document select_related
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first().select_related()
+
+            self.assertEqual(q, 2)
+            [m for m in group_obj.members]
+            self.assertEqual(q, 2)
+
+        # Queryset select_related
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            group_objs = Group.objects.select_related()
+            self.assertEqual(q, 2)
+            for group_obj in group_objs:
+                [m for m in group_obj.members]
+                self.assertEqual(q, 2)
+
+        User.drop_collection()
+        Group.drop_collection()
+
+    def test_handle_old_style_references(self):
+        """Ensure that DBRef items in ListFields are dereferenced.
+        """
+        class User(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = ListField(ReferenceField(User, dbref=True))
+
+        User.drop_collection()
+        Group.drop_collection()
+
+        for i in xrange(1, 26):
+            user = User(name='user %s' % i)
+            user.save()
+
+        group = Group(members=User.objects)
+        group.save()
+
+        group = Group._get_collection().find_one()
+
+        # Update the model to change the reference
+        class Group(Document):
+            members = ListField(ReferenceField(User, dbref=False))
+
+        group = Group.objects.first()
+        group.members.append(User(name="String!").save())
+        group.save()
+
+        group = Group.objects.first()
+        self.assertEqual(group.members[0].name, 'user 1')
+        self.assertEqual(group.members[-1].name, 'String!')
+
+    def test_migrate_references(self):
+        """Example of migrating ReferenceField storage
+        """
+
+        # Create some sample data
+        class User(Document):
+            name = StringField()
+
+        class Group(Document):
+            author = ReferenceField(User, dbref=True)
+            members = ListField(ReferenceField(User, dbref=True))
+
+        User.drop_collection()
+        Group.drop_collection()
+
+        user = User(name="Ross").save()
+        group = Group(author=user, members=[user]).save()
+
+        raw_data = Group._get_collection().find_one()
+        self.assertTrue(isinstance(raw_data['author'], DBRef))
+        self.assertTrue(isinstance(raw_data['members'][0], DBRef))
+
+        # Migrate the model definition
+        class Group(Document):
+            author = ReferenceField(User, dbref=False)
+            members = ListField(ReferenceField(User, dbref=False))
+
+        # Migrate the data
+        for g in Group.objects():
+            g.author = g.author
+            g.members = g.members
+            g.save()
+
+        group = Group.objects.first()
+        self.assertEqual(group.author, user)
+        self.assertEqual(group.members, [user])
+
+        raw_data = Group._get_collection().find_one()
+        self.assertTrue(isinstance(raw_data['author'], basestring))
+        self.assertTrue(isinstance(raw_data['members'][0], basestring))
+
     def test_recursive_reference(self):
         """Ensure that ReferenceFields can reference their own documents.
         """
@@ -109,10 +236,10 @@ class FieldTest(unittest.TestCase):
             peter = Employee.objects.with_id(peter.id).select_related()
             self.assertEqual(q, 2)
 
-            self.assertEquals(peter.boss, bill)
+            self.assertEqual(peter.boss, bill)
             self.assertEqual(q, 2)
 
-            self.assertEquals(peter.friends, friends)
+            self.assertEqual(peter.friends, friends)
             self.assertEqual(q, 2)
 
         # Queryset select_related
@@ -123,10 +250,10 @@ class FieldTest(unittest.TestCase):
             self.assertEqual(q, 2)
 
             for employee in employees:
-                self.assertEquals(employee.boss, bill)
+                self.assertEqual(employee.boss, bill)
                 self.assertEqual(q, 2)
 
-                self.assertEquals(employee.friends, friends)
+                self.assertEqual(employee.friends, friends)
                 self.assertEqual(q, 2)
 
     def test_circular_reference(self):
@@ -160,7 +287,7 @@ class FieldTest(unittest.TestCase):
         daughter.relations.append(self_rel)
         daughter.save()
 
-        self.assertEquals("[<Person: Mother>, <Person: Daughter>]", "%s" % Person.objects())
+        self.assertEqual("[<Person: Mother>, <Person: Daughter>]", "%s" % Person.objects())
 
     def test_circular_reference_on_self(self):
         """Ensure you can handle circular references
@@ -186,7 +313,7 @@ class FieldTest(unittest.TestCase):
         daughter.relations.append(daughter)
         daughter.save()
 
-        self.assertEquals("[<Person: Mother>, <Person: Daughter>]", "%s" % Person.objects())
+        self.assertEqual("[<Person: Mother>, <Person: Daughter>]", "%s" % Person.objects())
 
     def test_circular_tree_reference(self):
         """Ensure you can handle circular references with more than one level
@@ -228,7 +355,7 @@ class FieldTest(unittest.TestCase):
         anna.other.name = "Anna's friends"
         anna.save()
 
-        self.assertEquals(
+        self.assertEqual(
             "[<Person: Paul>, <Person: Maria>, <Person: Julia>, <Person: Anna>]",
             "%s" % Person.objects()
         )
@@ -760,3 +887,106 @@ class FieldTest(unittest.TestCase):
         UserB.drop_collection()
         UserC.drop_collection()
         Group.drop_collection()
+
+    def test_multidirectional_lists(self):
+
+        class Asset(Document):
+            name = StringField(max_length=250, required=True)
+            parent = GenericReferenceField(default=None)
+            parents = ListField(GenericReferenceField())
+            children = ListField(GenericReferenceField())
+
+        Asset.drop_collection()
+
+        root = Asset(name='', path="/", title="Site Root")
+        root.save()
+
+        company = Asset(name='company', title='Company', parent=root, parents=[root])
+        company.save()
+
+        root.children = [company]
+        root.save()
+
+        root = root.reload()
+        self.assertEqual(root.children, [company])
+        self.assertEqual(company.parents, [root])
+
+    def test_dict_in_dbref_instance(self):
+
+        class Person(Document):
+            name = StringField(max_length=250, required=True)
+
+        class Room(Document):
+            number = StringField(max_length=250, required=True)
+            staffs_with_position = ListField(DictField())
+
+        Person.drop_collection()
+        Room.drop_collection()
+
+        bob = Person.objects.create(name='Bob')
+        bob.save()
+        sarah = Person.objects.create(name='Sarah')
+        sarah.save()
+
+        room_101 = Room.objects.create(number="101")
+        room_101.staffs_with_position = [
+            {'position_key': 'window', 'staff': sarah},
+            {'position_key': 'door', 'staff': bob.to_dbref()}]
+        room_101.save()
+
+        room = Room.objects.first().select_related()
+        self.assertEqual(room.staffs_with_position[0]['staff'], sarah)
+        self.assertEqual(room.staffs_with_position[1]['staff'], bob)
+
+    def test_document_reload_no_inheritance(self):
+        class Foo(Document):
+            meta = {'allow_inheritance': False}
+            bar = ReferenceField('Bar')
+            baz = ReferenceField('Baz')
+
+        class Bar(Document):
+            meta = {'allow_inheritance': False}
+            msg = StringField(required=True, default='Blammo!')
+
+        class Baz(Document):
+            meta = {'allow_inheritance': False}
+            msg = StringField(required=True, default='Kaboom!')
+
+        Foo.drop_collection()
+        Bar.drop_collection()
+        Baz.drop_collection()
+
+        bar = Bar()
+        bar.save()
+        baz = Baz()
+        baz.save()
+        foo = Foo()
+        foo.bar = bar
+        foo.baz = baz
+        foo.save()
+        foo.reload()
+
+        self.assertEqual(type(foo.bar), Bar)
+        self.assertEqual(type(foo.baz), Baz)
+
+    def test_list_lookup_not_checked_in_map(self):
+        """Ensure we dereference list data correctly
+        """
+        class Comment(Document):
+            id = IntField(primary_key=True)
+            text = StringField()
+
+        class Message(Document):
+            id = IntField(primary_key=True)
+            comments = ListField(ReferenceField(Comment))
+
+        Comment.drop_collection()
+        Message.drop_collection()
+
+        c1 = Comment(id=0, text='zero').save()
+        c2 = Comment(id=1, text='one').save()
+        Message(id=1, comments=[c1, c2]).save()
+
+        msg = Message.objects.get(id=1)
+        self.assertEqual(0, msg.comments[0].id)
+        self.assertEqual(1, msg.comments[1].id)

tests/test_django.py

@@ -1,18 +1,34 @@
-# -*- coding: utf-8 -*-
-
+from __future__ import with_statement
 import unittest
-
+from nose.plugins.skip import SkipTest
+from mongoengine.python_support import PY3
 from mongoengine import *
-from mongoengine.django.shortcuts import get_document_or_404
 
-from django.http import Http404
-from django.template import Context, Template
-from django.conf import settings
-settings.configure()
+try:
+    from mongoengine.django.shortcuts import get_document_or_404
+
+    from django.http import Http404
+    from django.template import Context, Template
+    from django.conf import settings
+    from django.core.paginator import Paginator
+
+    settings.configure()
+
+    from django.contrib.sessions.tests import SessionTestsMixin
+    from mongoengine.django.sessions import SessionStore, MongoSession
+except Exception, err:
+    if PY3:
+        SessionTestsMixin = type  # dummy value so no error
+        SessionStore = None  # dummy value so no error
+    else:
+        raise err
+
 
 class QuerySetTest(unittest.TestCase):
 
     def setUp(self):
+        if PY3:
+            raise SkipTest('django does not have Python 3 support')
         connect(db='mongoenginetest')
 
         class Person(Document):
@@ -67,3 +83,40 @@ class QuerySetTest(unittest.TestCase):
         self.assertRaises(Http404, get_document_or_404, self.Person, pk='1234')
         self.assertEqual(p, get_document_or_404(self.Person, pk=p.pk))
 
+    def test_pagination(self):
+        """Ensure that Pagination works as expected
+        """
+        class Page(Document):
+            name = StringField()
+
+        Page.drop_collection()
+
+        for i in xrange(1, 11):
+            Page(name=str(i)).save()
+
+        paginator = Paginator(Page.objects.all(), 2)
+
+        t = Template("{% for i in page.object_list  %}{{ i.name }}:{% endfor %}")
+        for p in paginator.page_range:
+            d = {"page": paginator.page(p)}
+            end = p * 2
+            start = end - 1
+            self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end))
+
+
+
+class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase):
+    backend = SessionStore
+
+    def setUp(self):
+        if PY3:
+            raise SkipTest('django does not have Python 3 support')
+        connect(db='mongoenginetest')
+        MongoSession.drop_collection()
+        super(MongoDBSessionTest, self).setUp()
+
+    def test_first_save(self):
+        session = SessionStore()
+        session['test'] = True
+        session.save()
+        self.assertTrue('test' in session)

tests/test_dynamic_document.py

@@ -0,0 +1,533 @@
+import unittest
+
+from mongoengine import *
+from mongoengine.connection import get_db
+
+
+class DynamicDocTest(unittest.TestCase):
+
+    def setUp(self):
+        connect(db='mongoenginetest')
+        self.db = get_db()
+
+        class Person(DynamicDocument):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        Person.drop_collection()
+
+        self.Person = Person
+
+    def test_simple_dynamic_document(self):
+        """Ensures simple dynamic documents are saved correctly"""
+
+        p = self.Person()
+        p.name = "James"
+        p.age = 34
+
+        self.assertEqual(p.to_mongo(),
+            {"_types": ["Person"], "_cls": "Person",
+             "name": "James", "age": 34}
+        )
+
+        p.save()
+
+        self.assertEqual(self.Person.objects.first().age, 34)
+
+        # Confirm no changes to self.Person
+        self.assertFalse(hasattr(self.Person, 'age'))
+
+    def test_dynamic_document_delta(self):
+        """Ensures simple dynamic documents can delta correctly"""
+        p = self.Person(name="James", age=34)
+        self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {}))
+
+        p.doc = 123
+        del(p.doc)
+        self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1}))
+
+    def test_change_scope_of_variable(self):
+        """Test changing the scope of a dynamic field has no adverse effects"""
+        p = self.Person()
+        p.name = "Dean"
+        p.misc = 22
+        p.save()
+
+        p = self.Person.objects.get()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertEqual(p.misc, {'hello': 'world'})
+
+    def test_delete_dynamic_field(self):
+        """Test deleting a dynamic field works"""
+        self.Person.drop_collection()
+        p = self.Person()
+        p.name = "Dean"
+        p.misc = 22
+        p.save()
+
+        p = self.Person.objects.get()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertEqual(p.misc, {'hello': 'world'})
+        collection = self.db[self.Person._get_collection_name()]
+        obj = collection.find_one()
+        self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name'])
+
+        del(p.misc)
+        p.save()
+
+        p = self.Person.objects.get()
+        self.assertFalse(hasattr(p, 'misc'))
+
+        obj = collection.find_one()
+        self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'name'])
+
+    def test_dynamic_document_queries(self):
+        """Ensure we can query dynamic fields"""
+        p = self.Person()
+        p.name = "Dean"
+        p.age = 22
+        p.save()
+
+        self.assertEqual(1, self.Person.objects(age=22).count())
+        p = self.Person.objects(age=22)
+        p = p.get()
+        self.assertEqual(22, p.age)
+
+    def test_complex_dynamic_document_queries(self):
+        class Person(DynamicDocument):
+            name = StringField()
+
+        Person.drop_collection()
+
+        p = Person(name="test")
+        p.age = "ten"
+        p.save()
+
+        p1 = Person(name="test1")
+        p1.age = "less then ten and a half"
+        p1.save()
+
+        p2 = Person(name="test2")
+        p2.age = 10
+        p2.save()
+
+        self.assertEqual(Person.objects(age__icontains='ten').count(), 2)
+        self.assertEqual(Person.objects(age__gte=10).count(), 1)
+
+    def test_complex_data_lookups(self):
+        """Ensure you can query dynamic document dynamic fields"""
+        p = self.Person()
+        p.misc = {'hello': 'world'}
+        p.save()
+
+        self.assertEqual(1, self.Person.objects(misc__hello='world').count())
+
+    def test_inheritance(self):
+        """Ensure that dynamic document plays nice with inheritance"""
+        class Employee(self.Person):
+            salary = IntField()
+
+        Employee.drop_collection()
+
+        self.assertTrue('name' in Employee._fields)
+        self.assertTrue('salary' in Employee._fields)
+        self.assertEqual(Employee._get_collection_name(),
+                         self.Person._get_collection_name())
+
+        joe_bloggs = Employee()
+        joe_bloggs.name = "Joe Bloggs"
+        joe_bloggs.salary = 10
+        joe_bloggs.age = 20
+        joe_bloggs.save()
+
+        self.assertEqual(1, self.Person.objects(age=20).count())
+        self.assertEqual(1, Employee.objects(age=20).count())
+
+        joe_bloggs = self.Person.objects.first()
+        self.assertTrue(isinstance(joe_bloggs, Employee))
+
+    def test_embedded_dynamic_document(self):
+        """Test dynamic embedded documents"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+        embedded_1.list_field = ['1', 2, {'hello': 'world'}]
+        doc.embedded_field = embedded_1
+
+        self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc",
+            "embedded_field": {
+                "_types": ['Embedded'], "_cls": "Embedded",
+                "string_field": "hello",
+                "int_field": 1,
+                "dict_field": {"hello": "world"},
+                "list_field": ['1', 2, {'hello': 'world'}]
+            }
+        })
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEqual(doc.embedded_field.__class__, Embedded)
+        self.assertEqual(doc.embedded_field.string_field, "hello")
+        self.assertEqual(doc.embedded_field.int_field, 1)
+        self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'})
+        self.assertEqual(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}])
+
+    def test_complex_embedded_documents(self):
+        """Test complex dynamic embedded documents setups"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+
+        embedded_2 = Embedded()
+        embedded_2.string_field = 'hello'
+        embedded_2.int_field = 1
+        embedded_2.dict_field = {'hello': 'world'}
+        embedded_2.list_field = ['1', 2, {'hello': 'world'}]
+
+        embedded_1.list_field = ['1', 2, embedded_2]
+        doc.embedded_field = embedded_1
+
+        self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc",
+            "embedded_field": {
+                "_types": ['Embedded'], "_cls": "Embedded",
+                "string_field": "hello",
+                "int_field": 1,
+                "dict_field": {"hello": "world"},
+                "list_field": ['1', 2,
+                    {"_types": ['Embedded'], "_cls": "Embedded",
+                    "string_field": "hello",
+                    "int_field": 1,
+                    "dict_field": {"hello": "world"},
+                    "list_field": ['1', 2, {'hello': 'world'}]}
+                ]
+            }
+        })
+        doc.save()
+        doc = Doc.objects.first()
+        self.assertEqual(doc.embedded_field.__class__, Embedded)
+        self.assertEqual(doc.embedded_field.string_field, "hello")
+        self.assertEqual(doc.embedded_field.int_field, 1)
+        self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'})
+        self.assertEqual(doc.embedded_field.list_field[0], '1')
+        self.assertEqual(doc.embedded_field.list_field[1], 2)
+
+        embedded_field = doc.embedded_field.list_field[2]
+
+        self.assertEqual(embedded_field.__class__, Embedded)
+        self.assertEqual(embedded_field.string_field, "hello")
+        self.assertEqual(embedded_field.int_field, 1)
+        self.assertEqual(embedded_field.dict_field, {'hello': 'world'})
+        self.assertEqual(embedded_field.list_field, ['1', 2, {'hello': 'world'}])
+
+    def test_delta_for_dynamic_documents(self):
+        p = self.Person()
+        p.name = "Dean"
+        p.age = 22
+        p.save()
+
+        p.age = 24
+        self.assertEqual(p.age, 24)
+        self.assertEqual(p._get_changed_fields(), ['age'])
+        self.assertEqual(p._delta(), ({'age': 24}, {}))
+
+        p = self.Person.objects(age=22).get()
+        p.age = 24
+        self.assertEqual(p.age, 24)
+        self.assertEqual(p._get_changed_fields(), ['age'])
+        self.assertEqual(p._delta(), ({'age': 24}, {}))
+
+        p.save()
+        self.assertEqual(1, self.Person.objects(age=24).count())
+
+    def test_delta(self):
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEqual(doc._get_changed_fields(), [])
+        self.assertEqual(doc._delta(), ({}, {}))
+
+        doc.string_field = 'hello'
+        self.assertEqual(doc._get_changed_fields(), ['string_field'])
+        self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {}))
+
+        doc._changed_fields = []
+        doc.int_field = 1
+        self.assertEqual(doc._get_changed_fields(), ['int_field'])
+        self.assertEqual(doc._delta(), ({'int_field': 1}, {}))
+
+        doc._changed_fields = []
+        dict_value = {'hello': 'world', 'ping': 'pong'}
+        doc.dict_field = dict_value
+        self.assertEqual(doc._get_changed_fields(), ['dict_field'])
+        self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {}))
+
+        doc._changed_fields = []
+        list_value = ['1', 2, {'hello': 'world'}]
+        doc.list_field = list_value
+        self.assertEqual(doc._get_changed_fields(), ['list_field'])
+        self.assertEqual(doc._delta(), ({'list_field': list_value}, {}))
+
+        # Test unsetting
+        doc._changed_fields = []
+        doc.dict_field = {}
+        self.assertEqual(doc._get_changed_fields(), ['dict_field'])
+        self.assertEqual(doc._delta(), ({}, {'dict_field': 1}))
+
+        doc._changed_fields = []
+        doc.list_field = []
+        self.assertEqual(doc._get_changed_fields(), ['list_field'])
+        self.assertEqual(doc._delta(), ({}, {'list_field': 1}))
+
+    def test_delta_recursive(self):
+        """Testing deltaing works with dynamic documents"""
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+        doc.save()
+
+        doc = Doc.objects.first()
+        self.assertEqual(doc._get_changed_fields(), [])
+        self.assertEqual(doc._delta(), ({}, {}))
+
+        embedded_1 = Embedded()
+        embedded_1.string_field = 'hello'
+        embedded_1.int_field = 1
+        embedded_1.dict_field = {'hello': 'world'}
+        embedded_1.list_field = ['1', 2, {'hello': 'world'}]
+        doc.embedded_field = embedded_1
+
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field'])
+
+        embedded_delta = {
+            'string_field': 'hello',
+            'int_field': 1,
+            'dict_field': {'hello': 'world'},
+            'list_field': ['1', 2, {'hello': 'world'}]
+        }
+        self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {}))
+        embedded_delta.update({
+            '_types': ['Embedded'],
+            '_cls': 'Embedded',
+        })
+        self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {}))
+
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.dict_field = {}
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field'])
+        self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1}))
+
+        self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1}))
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.list_field = []
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1}))
+        self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1}))
+        doc.save()
+        doc.reload()
+
+        embedded_2 = Embedded()
+        embedded_2.string_field = 'hello'
+        embedded_2.int_field = 1
+        embedded_2.dict_field = {'hello': 'world'}
+        embedded_2.list_field = ['1', 2, {'hello': 'world'}]
+
+        doc.embedded_field.list_field = ['1', 2, embedded_2]
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEqual(doc.embedded_field._delta(), ({
+            'list_field': ['1', 2, {
+                '_cls': 'Embedded',
+                '_types': ['Embedded'],
+                'string_field': 'hello',
+                'dict_field': {'hello': 'world'},
+                'int_field': 1,
+                'list_field': ['1', 2, {'hello': 'world'}],
+            }]
+        }, {}))
+
+        self.assertEqual(doc._delta(), ({
+            'embedded_field.list_field': ['1', 2, {
+                '_cls': 'Embedded',
+                 '_types': ['Embedded'],
+                 'string_field': 'hello',
+                 'dict_field': {'hello': 'world'},
+                 'int_field': 1,
+                 'list_field': ['1', 2, {'hello': 'world'}],
+            }]
+        }, {}))
+        doc.save()
+        doc.reload()
+
+        self.assertEqual(doc.embedded_field.list_field[2]._changed_fields, [])
+        self.assertEqual(doc.embedded_field.list_field[0], '1')
+        self.assertEqual(doc.embedded_field.list_field[1], 2)
+        for k in doc.embedded_field.list_field[2]._fields:
+            self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k])
+
+        doc.embedded_field.list_field[2].string_field = 'world'
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field'])
+        self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {}))
+        self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world')
+
+        # Test multiple assignments
+        doc.embedded_field.list_field[2].string_field = 'hello world'
+        doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2]
+        self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field'])
+        self.assertEqual(doc.embedded_field._delta(), ({
+            'list_field': ['1', 2, {
+            '_types': ['Embedded'],
+            '_cls': 'Embedded',
+            'string_field': 'hello world',
+            'int_field': 1,
+            'list_field': ['1', 2, {'hello': 'world'}],
+            'dict_field': {'hello': 'world'}}]}, {}))
+        self.assertEqual(doc._delta(), ({
+            'embedded_field.list_field': ['1', 2, {
+                '_types': ['Embedded'],
+                '_cls': 'Embedded',
+                'string_field': 'hello world',
+                'int_field': 1,
+                'list_field': ['1', 2, {'hello': 'world'}],
+                'dict_field': {'hello': 'world'}}
+            ]}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world')
+
+        # Test list native methods
+        doc.embedded_field.list_field[2].list_field.pop(0)
+        self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {}))
+        doc.save()
+        doc.reload()
+
+        doc.embedded_field.list_field[2].list_field.append(1)
+        self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {}))
+        doc.save()
+        doc.reload()
+        self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1])
+
+        doc.embedded_field.list_field[2].list_field.sort(key=str)# use str as a key to allow comparing uncomperable types
+        doc.save()
+        doc.reload()
+        self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}])
+
+        del(doc.embedded_field.list_field[2].list_field[2]['hello'])
+        self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {}))
+        doc.save()
+        doc.reload()
+
+        del(doc.embedded_field.list_field[2].list_field)
+        self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1}))
+
+        doc.save()
+        doc.reload()
+
+        doc.dict_field = {'embedded': embedded_1}
+        doc.save()
+        doc.reload()
+
+        doc.dict_field['embedded'].string_field = 'Hello World'
+        self.assertEqual(doc._get_changed_fields(), ['dict_field.embedded.string_field'])
+        self.assertEqual(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {}))
+
+    def test_indexes(self):
+        """Ensure that indexes are used when meta[indexes] is specified.
+        """
+        class BlogPost(DynamicDocument):
+            meta = {
+                'indexes': [
+                    '-date',
+                    ('category', '-date')
+                ],
+            }
+
+        BlogPost.drop_collection()
+
+        info = BlogPost.objects._collection.index_information()
+        # _id, '-date', ('cat', 'date')
+        # NB: there is no index on _types by itself, since
+        # the indices on -date and tags will both contain
+        # _types as first element in the key
+        self.assertEqual(len(info), 3)
+
+        # Indexes are lazy so use list() to perform query
+        list(BlogPost.objects)
+        info = BlogPost.objects._collection.index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('_types', 1), ('category', 1), ('date', -1)]
+                        in info)
+        self.assertTrue([('_types', 1), ('date', -1)] in info)
+
+    def test_dynamic_and_embedded(self):
+        """Ensure embedded documents play nicely"""
+
+        class Address(EmbeddedDocument):
+            city = StringField()
+
+        class Person(DynamicDocument):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        Person.drop_collection()
+
+        Person(name="Ross", address=Address(city="London")).save()
+
+        person = Person.objects.first()
+        person.address.city = "Lundenne"
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Lundenne")
+
+        person = Person.objects.first()
+        person.address = Address(city="Londinium")
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Londinium")
+
+        person = Person.objects.first()
+        person.age = 35
+        person.save()
+        self.assertEqual(Person.objects.first().age, 35)

tests/test_replicaset_connection.py

@@ -0,0 +1,33 @@
+import unittest
+
+import pymongo
+from pymongo import ReadPreference, ReplicaSetConnection
+
+import mongoengine
+from mongoengine import *
+from mongoengine.connection import get_db, get_connection, ConnectionError
+
+
+class ConnectionTest(unittest.TestCase):
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+
+    def test_replicaset_uri_passes_read_preference(self):
+        """Requires a replica set called "rs" on port 27017
+        """
+
+        try:
+            conn = connect(db='mongoenginetest', host="mongodb://localhost/mongoenginetest?replicaSet=rs", read_preference=ReadPreference.SECONDARY_ONLY)
+        except ConnectionError, e:
+            return
+
+        if not isinstance(conn, ReplicaSetConnection):
+            return
+
+        self.assertEqual(conn.read_preference, ReadPreference.SECONDARY_ONLY)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_signals.py

@@ -56,6 +56,18 @@ class SignalTests(unittest.TestCase):
             @classmethod
             def post_delete(cls, sender, document, **kwargs):
                 signal_output.append('post_delete signal, %s' % document)
+
+            @classmethod
+            def pre_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('pre_bulk_insert signal, %s' % documents)
+
+            @classmethod
+            def post_bulk_insert(cls, sender, documents, **kwargs):
+                signal_output.append('post_bulk_insert signal, %s' % documents)
+                if kwargs.get('loaded', False):
+                    signal_output.append('Is loaded')
+                else:
+                    signal_output.append('Not loaded')
         self.Author = Author
 
 
@@ -104,7 +116,9 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_save.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
-            len(signals.post_delete.receivers)
+            len(signals.post_delete.receivers),
+            len(signals.pre_bulk_insert.receivers),
+            len(signals.post_bulk_insert.receivers),
         )
 
         signals.pre_init.connect(Author.pre_init, sender=Author)
@@ -113,6 +127,8 @@ class SignalTests(unittest.TestCase):
         signals.post_save.connect(Author.post_save, sender=Author)
         signals.pre_delete.connect(Author.pre_delete, sender=Author)
         signals.post_delete.connect(Author.post_delete, sender=Author)
+        signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author)
+        signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author)
 
         signals.pre_init.connect(Another.pre_init, sender=Another)
         signals.post_init.connect(Another.post_init, sender=Another)
@@ -128,6 +144,8 @@ class SignalTests(unittest.TestCase):
         signals.pre_delete.disconnect(self.Author.pre_delete)
         signals.post_save.disconnect(self.Author.post_save)
         signals.pre_save.disconnect(self.Author.pre_save)
+        signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert)
+        signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert)
 
         signals.pre_init.disconnect(self.Another.pre_init)
         signals.post_init.disconnect(self.Another.post_init)
@@ -143,7 +161,9 @@ class SignalTests(unittest.TestCase):
             len(signals.pre_save.receivers),
             len(signals.post_save.receivers),
             len(signals.pre_delete.receivers),
-            len(signals.post_delete.receivers)
+            len(signals.post_delete.receivers),
+            len(signals.pre_bulk_insert.receivers),
+            len(signals.post_bulk_insert.receivers),
         )
 
         self.assertEqual(self.pre_signals, post_signals)
@@ -154,6 +174,14 @@ class SignalTests(unittest.TestCase):
         def create_author():
             a1 = self.Author(name='Bill Shakespeare')
 
+        def bulk_create_author_with_load():
+            a1 = self.Author(name='Bill Shakespeare')
+            self.Author.objects.insert([a1], load_bulk=True)
+
+        def bulk_create_author_without_load():
+            a1 = self.Author(name='Bill Shakespeare')
+            self.Author.objects.insert([a1], load_bulk=False)
+
         self.assertEqual(self.get_signal_output(create_author), [
             "pre_init signal, Author",
             "{'name': 'Bill Shakespeare'}",
@@ -178,4 +206,25 @@ class SignalTests(unittest.TestCase):
         self.assertEqual(self.get_signal_output(a1.delete), [
             'pre_delete signal, William Shakespeare',
             'post_delete signal, William Shakespeare',
-        ])
+        ])
+
+        signal_output = self.get_signal_output(bulk_create_author_with_load)
+
+        # The output of this signal is not entirely deterministic. The reloaded
+        # object will have an object ID. Hence, we only check part of the output
+        self.assertEqual(signal_output[3],
+            "pre_bulk_insert signal, [<Author: Bill Shakespeare>]")
+        self.assertEqual(signal_output[-2:],
+            ["post_bulk_insert signal, [<Author: Bill Shakespeare>]",
+             "Is loaded",])
+
+        self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [
+            "pre_init signal, Author",
+            "{'name': 'Bill Shakespeare'}",
+            "post_init signal, Bill Shakespeare",
+            "pre_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            "post_bulk_insert signal, [<Author: Bill Shakespeare>]",
+            "Not loaded",
+        ])
+
+        self.Author.objects.delete()