Changed an invalid classifier to a valid one.

Updated travis build

.travis.yml

@@ -1,33 +1,72 @@
-# http://travis-ci.org/#!/MongoEngine/mongoengine
 language: python
-services: mongodb
 python:
-    - "2.6"
-    - "2.7"
-    - "3.2"
-    - "3.3"
+- '2.6'
+- '2.7'
+- '3.2'
+- '3.3'
+- '3.4'
+- pypy
+- pypy3
 env:
-  - PYMONGO=dev DJANGO=1.6
-  - PYMONGO=dev DJANGO=1.5.5
-  - PYMONGO=dev DJANGO=1.4.10
-  - PYMONGO=2.5 DJANGO=1.6
-  - PYMONGO=2.5 DJANGO=1.5.5
-  - PYMONGO=2.5 DJANGO=1.4.10
-  - PYMONGO=3.2 DJANGO=1.6
-  - PYMONGO=3.2 DJANGO=1.5.5
-  - PYMONGO=3.3 DJANGO=1.6
-  - PYMONGO=3.3 DJANGO=1.5.5
+- PYMONGO=2.7.2 DJANGO=dev
+- PYMONGO=2.7.2 DJANGO=1.7.1
+- PYMONGO=2.7.2 DJANGO=1.6.8
+- PYMONGO=2.7.2 DJANGO=1.5.11
+- PYMONGO=2.8 DJANGO=dev
+- PYMONGO=2.8 DJANGO=1.7.1
+- PYMONGO=2.8 DJANGO=1.6.8
+- PYMONGO=2.8 DJANGO=1.5.11
+matrix:
+  exclude:
+  - python: '2.6'
+    env: PYMONGO=2.7.2 DJANGO=dev
+  - python: '2.6'
+    env: PYMONGO=2.8 DJANGO=dev
+  - python: '2.6'
+    env: PYMONGO=2.7.2 DJANGO=1.7.1
+  - python: '2.6'
+    env: PYMONGO=2.8 DJANGO=1.7.1
+  allow_failures:
+  - python: pypy3
+  fast_finish: true
+before_install:
+- travis_retry sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
+- echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' |
+  sudo tee /etc/apt/sources.list.d/mongodb.list
+- travis_retry sudo apt-get update
+- travis_retry sudo apt-get install mongodb-org-server
 install:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
-    - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
-    - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
-    - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b
-    - python setup.py install
+- sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
+  libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev
+  python-tk
+- if [[ $PYMONGO == 'dev' ]]; then travis_retry pip install https://github.com/mongodb/mongo-python-driver/tarball/master;
+  true; fi
+- if [[ $PYMONGO != 'dev' ]]; then travis_retry pip install pymongo==$PYMONGO; true;
+  fi
+- if [[ $DJANGO == 'dev' ]]; then travis_retry pip install git+https://github.com/django/django.git;
+  fi
+- if [[ $DJANGO != 'dev' ]]; then travis_retry pip install Django==$DJANGO; fi
+- travis_retry pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b
+- travis_retry pip install coveralls
+- travis_retry python setup.py install
 script:
-    - python setup.py test
+- travis_retry python setup.py test
+- if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then 2to3 . -w; fi;
+- coverage run --source=mongoengine setup.py test
+- coverage report -m
+- python benchmark.py
+after_script: coveralls --verbose
 notifications:
-  irc: "irc.freenode.org#mongoengine"
+  irc: irc.freenode.org#mongoengine
 branches:
   only:
-    - master
+  - master
+  - /^v.*$/
+deploy:
+  provider: pypi
+  user: the_drow
+  password:
+    secure: QMyatmWBnC6ZN3XLW2+fTBDU4LQcp1m/LjR2/0uamyeUzWKdlOoh/Wx5elOgLwt/8N9ppdPeG83ose1jOz69l5G0MUMjv8n/RIcMFSpCT59tGYqn3kh55b0cIZXFT9ar+5cxlif6a5rS72IHm5li7QQyxexJIII6Uxp0kpvUmek=
+  on:
+    tags: true
+    repo: MongoEngine/mongoengine

AUTHORS

@@ -142,7 +142,7 @@ that much better:
  * Pete Campton
  * Martyn Smith
  * Marcelo Anton
- * Aleksey Porfirov
+ * Aleksey Porfirov (https://github.com/lexqt)
  * Nicolas Trippar
  * Manuel Hermann
  * Gustavo Gawryszewski
@@ -171,7 +171,7 @@ that much better:
  * Michael Bartnett (https://github.com/michaelbartnett)
  * Alon Horev (https://github.com/alonho)
  * Kelvin Hammond (https://github.com/kelvinhammond)
- * Jatin- (https://github.com/jatin-)
+ * Jatin Chopra (https://github.com/jatin)
  * Paul Uithol (https://github.com/PaulUithol)
  * Thom Knowles (https://github.com/fleat)
  * Paul (https://github.com/squamous)
@@ -189,3 +189,32 @@ that much better:
  * Tom (https://github.com/tomprimozic)
  * j0hnsmith (https://github.com/j0hnsmith)
  * Damien Churchill (https://github.com/damoxc)
+ * Jonathan Simon Prates (https://github.com/jonathansp)
+ * Thiago Papageorgiou (https://github.com/tmpapageorgiou)
+ * Omer Katz (https://github.com/thedrow)
+ * Falcon Dai (https://github.com/falcondai)
+ * Polyrabbit (https://github.com/polyrabbit)
+ * Sagiv Malihi (https://github.com/sagivmalihi)
+ * Dmitry Konishchev (https://github.com/KonishchevDmitry)
+ * Martyn Smith (https://github.com/martynsmith)
+ * Andrei Zbikowski (https://github.com/b1naryth1ef)
+ * Ronald van Rij (https://github.com/ronaldvanrij)
+ * François Schmidts (https://github.com/jaesivsm)
+ * Eric Plumb (https://github.com/professorplumb)
+ * Damien Churchill (https://github.com/damoxc)
+ * Aleksandr Sorokoumov (https://github.com/Gerrrr)
+ * Clay McClure (https://github.com/claymation)
+ * Bruno Rocha (https://github.com/rochacbruno)
+ * Norberto Leite (https://github.com/nleite)
+ * Bob Cribbs (https://github.com/bocribbz)
+ * Jay Shirley (https://github.com/jshirley)
+ * David Bordeynik (https://github.com/DavidBord)
+ * Axel Haustant (https://github.com/noirbizarre)
+ * David Czarnecki (https://github.com/czarneckid)
+ * Vyacheslav Murashkin (https://github.com/a4tunado)
+ * André Ericson https://github.com/aericson)
+ * Mikhail Moshnogorsky (https://github.com/mikhailmoshnogorsky)
+ * Diego Berrocal (https://github.com/cestdiego)
+ * Matthew Ellison (https://github.com/seglberg)
+ * Jimmy Shen (https://github.com/jimmyshen)
+ * J. Fernando Sánchez (https://github.com/balkian)

README.rst

@@ -8,6 +8,13 @@ MongoEngine
 
 .. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master
   :target: http://travis-ci.org/MongoEngine/mongoengine
+  
+.. image:: https://coveralls.io/repos/MongoEngine/mongoengine/badge.png?branch=master 
+  :target: https://coveralls.io/r/MongoEngine/mongoengine?branch=master
+  
+.. image:: https://landscape.io/github/MongoEngine/mongoengine/master/landscape.png
+   :target: https://landscape.io/github/MongoEngine/mongoengine/master
+   :alt: Code Health
 
 About
 =====
@@ -26,9 +33,18 @@ setup.py install``.
 
 Dependencies
 ============
-- pymongo 2.5+
+- pymongo>=2.7.1
 - sphinx (optional - for documentation generation)
 
+Optional Dependencies
+---------------------
+- **Django Integration:** Django>=1.4.0 for Python 2.x or PyPy and Django>=1.5.0 for Python 3.x
+- **Image Fields**: Pillow>=2.0.0 or PIL (not recommended since MongoEngine is tested with Pillow)
+- dateutil>=2.1.0
+
+.. note
+   MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: Django 1.6.5
+
 Examples
 ========
 Some simple examples of what MongoEngine code looks like::

benchmark.py

@@ -15,7 +15,7 @@ def cprofile_main():
     class Noddy(Document):
         fields = DictField()
 
-    for i in xrange(1):
+    for i in range(1):
         noddy = Noddy()
         for j in range(20):
             noddy.fields["key" + str(j)] = "value " + str(j)
@@ -113,6 +113,7 @@ def main():
     4.68946313858
     ----------------------------------------------------------------------------------------------------
     """
+    print("Benchmarking...")
 
     setup = """
 from pymongo import MongoClient
@@ -127,7 +128,7 @@ connection = MongoClient()
 db = connection.timeit_test
 noddy = db.noddy
 
-for i in xrange(10000):
+for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
         example['fields']["key"+str(j)] = "value "+str(j)
@@ -138,10 +139,10 @@ myNoddys = noddy.find()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - Pymongo"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - Pymongo""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
 from pymongo import MongoClient
@@ -150,7 +151,7 @@ connection = MongoClient()
 db = connection.timeit_test
 noddy = db.noddy
 
-for i in xrange(10000):
+for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
         example['fields']["key"+str(j)] = "value "+str(j)
@@ -161,10 +162,10 @@ myNoddys = noddy.find()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     setup = """
 from pymongo import MongoClient
@@ -180,7 +181,7 @@ class Noddy(Document):
 """
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -190,13 +191,13 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     fields = {}
     for j in range(20):
@@ -208,13 +209,13 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries without continual assign - MongoEngine"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries without continual assign - MongoEngine""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -224,13 +225,13 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -240,13 +241,13 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -256,13 +257,13 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
     stmt = """
-for i in xrange(10000):
+for i in range(10000):
     noddy = Noddy()
     for j in range(20):
         noddy.fields["key"+str(j)] = "value "+str(j)
@@ -272,11 +273,11 @@ myNoddys = Noddy.objects()
 [n for n in myNoddys] # iterate
 """
 
-    print "-" * 100
-    print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False"""
+    print("-" * 100)
+    print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""")
     t = timeit.Timer(stmt=stmt, setup=setup)
-    print t.timeit(1)
+    print(t.timeit(1))
 
 
 if __name__ == "__main__":
-    main()
+    main()

docs/apireference.rst

@@ -79,11 +79,13 @@ Fields
 .. autoclass:: mongoengine.fields.GenericEmbeddedDocumentField
 .. autoclass:: mongoengine.fields.DynamicField
 .. autoclass:: mongoengine.fields.ListField
+.. autoclass:: mongoengine.fields.EmbeddedDocumentListField
 .. autoclass:: mongoengine.fields.SortedListField
 .. autoclass:: mongoengine.fields.DictField
 .. autoclass:: mongoengine.fields.MapField
 .. autoclass:: mongoengine.fields.ReferenceField
 .. autoclass:: mongoengine.fields.GenericReferenceField
+.. autoclass:: mongoengine.fields.CachedReferenceField
 .. autoclass:: mongoengine.fields.BinaryField
 .. autoclass:: mongoengine.fields.FileField
 .. autoclass:: mongoengine.fields.ImageField
@@ -94,11 +96,29 @@ Fields
 .. autoclass:: mongoengine.fields.PointField
 .. autoclass:: mongoengine.fields.LineStringField
 .. autoclass:: mongoengine.fields.PolygonField
+.. autoclass:: mongoengine.fields.MultiPointField
+.. autoclass:: mongoengine.fields.MultiLineStringField
+.. autoclass:: mongoengine.fields.MultiPolygonField
 .. autoclass:: mongoengine.fields.GridFSError
 .. autoclass:: mongoengine.fields.GridFSProxy
 .. autoclass:: mongoengine.fields.ImageGridFsProxy
 .. autoclass:: mongoengine.fields.ImproperlyConfigured
 
+Embedded Document Querying
+==========================
+
+.. versionadded:: 0.9
+
+Additional queries for Embedded Documents are available when using the
+:class:`~mongoengine.EmbeddedDocumentListField` to store a list of embedded
+documents.
+
+A list of embedded documents is returned as a special list with the
+following methods:
+
+.. autoclass:: mongoengine.base.datastructures.EmbeddedDocumentList
+    :members:
+
 Misc
 ====
 

docs/changelog.rst

@@ -2,9 +2,86 @@
 Changelog
 =========
 
+
+Changes in 0.9.X - DEV
+======================
+- Update FileField when creating a new file #714
+- Added `EmbeddedDocumentListField` for Lists of Embedded Documents. #826
+- ComplexDateTimeField should fall back to None when null=True #864
+- Request Support for $min, $max Field update operators #863
+- `BaseDict` does not follow `setdefault` #866
+- Add support for $type operator # 766
+- Fix tests for pymongo 2.8+ #877
+- No module named 'django.utils.importlib' (Django dev) #872
+- Field Choices Now Accept Subclasses of Documents
+- Ensure Indexes before Each Save #812
+- Generate Unique Indices for Lists of EmbeddedDocuments #358
+- Sparse fields #515
+- write_concern not in params of Collection#remove #801
+- Better BaseDocument equality check when not saved #798
+- OperationError: Shard Keys are immutable. Tried to update id even though the document is not yet saved #771
+- with_limit_and_skip for count should default like in pymongo #759
+- Fix storing value of precision attribute in DecimalField #787
+- Set attribute to None does not work (at least for fields with default values) #734
+- Querying by a field defined in a subclass raises InvalidQueryError #744
+- Add Support For MongoDB 2.6.X's maxTimeMS #778
+- abstract shouldn't be inherited in EmbeddedDocument # 789
+- Allow specifying the '_cls' as a field for indexes #397
+- Stop ensure_indexes running on a secondaries unless connection is through mongos #746
+- Not overriding default values when loading a subset of fields #399
+- Saving document doesn't create new fields in existing collection #620
+- Added `Queryset.aggregate` wrapper to aggregation framework #703
+- Added support to show original model fields on to_json calls instead of db_field #697
+- Added Queryset.search_text to Text indexes searchs #700
+- Fixed tests for Django 1.7 #696
+- Follow ReferenceFields in EmbeddedDocuments with select_related #690
+- Added preliminary support for text indexes #680
+- Added `elemMatch` operator as well - `match` is too obscure #653
+- Added support for progressive JPEG #486 #548
+- Allow strings to be used in index creation #675
+- Fixed EmbeddedDoc weakref proxy issue #592
+- Fixed nested reference field distinct error #583
+- Fixed change tracking on nested MapFields #539
+- Dynamic fields in embedded documents now visible to queryset.only() / qs.exclude() #425 #507
+- Add authentication_source option to register_connection #178 #464 #573 #580 #590
+- Implemented equality between Documents and DBRefs #597
+- Fixed ReferenceField inside nested ListFields dereferencing problem #368
+- Added the ability to reload specific document fields #100
+- Added db_alias support and fixes for custom map/reduce output #586
+- post_save signal now has access to delta information about field changes #594 #589
+- Don't query with $orderby for qs.get() #600
+- Fix id shard key save issue #636
+- Fixes issue with recursive embedded document errors #557
+- Fix clear_changed_fields() clearing unsaved documents bug #602
+- Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x.
+- Removing support for Python < 2.6.6
+- Fixed $maxDistance location for geoJSON $near queries with MongoDB 2.6+ #664
+- QuerySet.modify() and Document.modify() methods to provide find_and_modify() like behaviour #677 #773
+- Added support for the using() method on a queryset #676
+- PYPY support #673
+- Connection pooling #674
+- Avoid to open all documents from cursors in an if stmt #655
+- Ability to clear the ordering #657
+- Raise NotUniqueError in Document.update() on pymongo.errors.DuplicateKeyError #626
+- Slots - memory improvements #625
+- Fixed incorrectly split a query key when it ends with "_" #619
+- Geo docs updates #613
+- Workaround a dateutil bug #608
+- Conditional save for atomic-style operations #511
+- Allow dynamic dictionary-style field access #559
+- Increase email field length to accommodate new TLDs #726
+- index_cls is ignored when deciding to set _cls as index prefix #733
+- Make 'db' argument to connection optional #737
+- Allow atomic update for the entire `DictField` #742
+- Added MultiPointField, MultiLineField, MultiPolygonField
+- Fix multiple connections aliases being rewritten #748
+- Fixed a few instances where reverse_delete_rule was written as reverse_delete_rules. #791
+- Make `in_bulk()` respect `no_dereference()` #775
+- Handle None from model __str__; Fixes #753 #754
+
 Changes in 0.8.7
 ================
-- Calling reload on deleted / nonexistant documents raises DoesNotExist (#538)
+- Calling reload on deleted / nonexistent documents raises DoesNotExist (#538)
 - Stop ensure_indexes running on a secondaries (#555)
 - Fix circular import issue with django auth (#531) (#545)
 
@@ -17,7 +94,7 @@ Changes in 0.8.5
 - Fix multi level nested fields getting marked as changed (#523)
 - Django 1.6 login fix (#522) (#527)
 - Django 1.6 session fix (#509)
-- EmbeddedDocument._instance is now set when settng the attribute (#506)
+- EmbeddedDocument._instance is now set when setting the attribute (#506)
 - Fixed EmbeddedDocument with ReferenceField equality issue (#502)
 - Fixed GenericReferenceField serialization order (#499)
 - Fixed count and none bug (#498)
@@ -107,7 +184,7 @@ Changes in 0.8.0
 - Added `get_next_value` preview for SequenceFields (#319)
 - Added no_sub_classes context manager and queryset helper (#312)
 - Querysets now utilises a local cache
-- Changed __len__ behavour in the queryset (#247, #311)
+- Changed __len__ behaviour in the queryset (#247, #311)
 - Fixed querying string versions of ObjectIds issue with ReferenceField (#307)
 - Added $setOnInsert support for upserts (#308)
 - Upserts now possible with just query parameters (#309)
@@ -158,7 +235,7 @@ Changes in 0.8.0
 - Uses getlasterror to test created on updated saves (#163)
 - Fixed inheritance and unique index creation (#140)
 - Fixed reverse delete rule with inheritance (#197)
-- Fixed validation for GenericReferences which havent been dereferenced
+- Fixed validation for GenericReferences which haven't been dereferenced
 - Added switch_db context manager (#106)
 - Added switch_db method to document instances (#106)
 - Added no_dereference context manager (#82) (#61)
@@ -240,11 +317,11 @@ Changes in 0.7.2
 - Update index spec generation so its not destructive (#113)
 
 Changes in 0.7.1
-=================
+================
 - Fixed index spec inheritance (#111)
 
 Changes in 0.7.0
-=================
+================
 - Updated queryset.delete so you can use with skip / limit (#107)
 - Updated index creation allows kwargs to be passed through refs (#104)
 - Fixed Q object merge edge case (#109)
@@ -325,7 +402,7 @@ Changes in 0.6.12
 - Fixes error with _delta handling DBRefs
 
 Changes in 0.6.11
-==================
+=================
 - Fixed inconsistency handling None values field attrs
 - Fixed map_field embedded db_field issue
 - Fixed .save() _delta issue with DbRefs
@@ -405,7 +482,7 @@ Changes in 0.6.1
 - Fix for replicaSet connections
 
 Changes in 0.6
-================
+==============
 
 - Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7
 - Added support for covered indexes when inheritance is off
@@ -493,8 +570,8 @@ Changes in v0.5
 - Updated default collection naming convention
 - Added Document Mixin support
 - Fixed queryet __repr__ mid iteration
-- Added hint() support, so cantell Mongo the proper index to use for the query
-- Fixed issue with inconsitent setting of _cls breaking inherited referencing
+- Added hint() support, so can tell Mongo the proper index to use for the query
+- Fixed issue with inconsistent setting of _cls breaking inherited referencing
 - Added help_text and verbose_name to fields to help with some form libs
 - Updated item_frequencies to handle embedded document lookups
 - Added delta tracking now only sets / unsets explicitly changed fields

docs/guide/connecting.rst

@@ -23,21 +23,32 @@ arguments should be provided::
 
     connect('project1', username='webapp', password='pwd123')
 
-Uri style connections are also supported - just supply the uri as
+URI style connections are also supported -- just supply the URI as
 the :attr:`host` to
 :func:`~mongoengine.connect`::
 
     connect('project1', host='mongodb://localhost/database_name')
 
-Note that database name from uri has priority over name
-in ::func:`~mongoengine.connect`
+.. note:: Database, username and password from URI string overrides
+    corresponding parameters in :func:`~mongoengine.connect`: ::
+
+        connect(
+            name='test',
+            username='user',
+            password='12345',
+            host='mongodb://admin:qwerty@localhost/production'
+        )
+
+    will establish connection to ``production`` database using
+    ``admin`` username and ``qwerty`` password.
 
 ReplicaSets
 ===========
 
-MongoEngine supports :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`
-to use them please use a URI style connection and provide the `replicaSet` name in the
-connection kwargs.
+MongoEngine supports
+:class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`. To use them,
+please use an URI style connection and provide the ``replicaSet`` name
+in the connection kwargs.
 
 Read preferences are supported through the connection or via individual
 queries by passing the read_preference ::
@@ -77,36 +88,38 @@ to point across databases and collections.  Below is an example schema, using
             meta = {"db_alias": "users-books-db"}
 
 
-Switch Database Context Manager
-===============================
-
-Sometimes you may want to switch the database to query against for a class
-for example, archiving older data into a separate database for performance
-reasons.
+Context Managers
+================
+Sometimes you may want to switch the database or collection to query against
+for a class.
+For example, archiving older data into a separate database for performance
+reasons or writing functions that dynamically choose collections to write
+document to.
 
+Switch Database
+---------------
 The :class:`~mongoengine.context_managers.switch_db` context manager allows
 you to change the database alias for a given class allowing quick and easy
-access to the same User document across databases::
+access the same User document across databases::
 
-        from mongoengine.context_managers import switch_db
+    from mongoengine.context_managers import switch_db
 
-        class User(Document):
-            name = StringField()
+    class User(Document):
+        name = StringField()
 
-            meta = {"db_alias": "user-db"}
+        meta = {"db_alias": "user-db"}
 
-        with switch_db(User, 'archive-user-db') as User:
-            User(name="Ross").save()  # Saves the 'archive-user-db'
+    with switch_db(User, 'archive-user-db') as User:
+        User(name="Ross").save()  # Saves the 'archive-user-db'
 
-.. note:: Make sure any aliases have been registered with
-    :func:`~mongoengine.register_connection` before using the context manager.
 
-There is also a switch collection context manager as well.  The
-:class:`~mongoengine.context_managers.switch_collection` context manager allows
-you to change the collection for a given class allowing quick and easy
-access to the same Group document across collection::
+Switch Collection
+-----------------
+The :class:`~mongoengine.context_managers.switch_collection` context manager
+allows you to change the collection for a given class allowing quick and easy
+access the same Group document across collection::
 
-        from mongoengine.context_managers import switch_db
+        from mongoengine.context_managers import switch_collection
 
         class Group(Document):
             name = StringField()
@@ -115,3 +128,9 @@ access to the same Group document across collection::
 
         with switch_collection(Group, 'group2000') as Group:
             Group(name="hello Group 2000 collection!").save()  # Saves in group2000 collection
+
+
+
+.. note:: Make sure any aliases have been registered with
+    :func:`~mongoengine.register_connection` or :func:`~mongoengine.connect`
+    before using the context manager.

docs/guide/defining-documents.rst

@@ -4,7 +4,7 @@ Defining documents
 In MongoDB, a **document** is roughly equivalent to a **row** in an RDBMS. When
 working with relational databases, rows are stored in **tables**, which have a
 strict **schema** that the rows follow. MongoDB stores documents in
-**collections** rather than tables - the principle difference is that no schema
+**collections** rather than tables --- the principal difference is that no schema
 is enforced at a database level.
 
 Defining a document's schema
@@ -91,6 +91,12 @@ are as follows:
 * :class:`~mongoengine.fields.StringField`
 * :class:`~mongoengine.fields.URLField`
 * :class:`~mongoengine.fields.UUIDField`
+* :class:`~mongoengine.fields.PointField`
+* :class:`~mongoengine.fields.LineStringField`
+* :class:`~mongoengine.fields.PolygonField`
+* :class:`~mongoengine.fields.MultiPointField`
+* :class:`~mongoengine.fields.MultiLineStringField`
+* :class:`~mongoengine.fields.MultiPolygonField`
 
 Field arguments
 ---------------
@@ -165,15 +171,15 @@ arguments can be set on all fields:
             size = StringField(max_length=3, choices=SIZE)
 
 :attr:`help_text` (Default: None)
-    Optional help text to output with the field - used by form libraries
+    Optional help text to output with the field -- used by form libraries
 
 :attr:`verbose_name` (Default: None)
-    Optional human-readable name for the field - used by form libraries
+    Optional human-readable name for the field -- used by form libraries
 
 
 List fields
 -----------
-MongoDB allows the storage of lists of items. To add a list of items to a
+MongoDB allows storing lists of items. To add a list of items to a
 :class:`~mongoengine.Document`, use the :class:`~mongoengine.fields.ListField` field
 type. :class:`~mongoengine.fields.ListField` takes another field object as its first
 argument, which specifies which type elements may be stored within the list::
@@ -328,7 +334,7 @@ Its value can take any of the following constants:
   Any object's fields still referring to the object being deleted are removed
   (using MongoDB's "unset" operation), effectively nullifying the relationship.
 :const:`mongoengine.CASCADE`
-  Any object containing fields that are refererring to the object being deleted
+  Any object containing fields that are referring to the object being deleted
   are deleted first.
 :const:`mongoengine.PULL`
   Removes the reference to the object (using MongoDB's "pull" operation)
@@ -422,7 +428,7 @@ Document collections
 ====================
 Document classes that inherit **directly** from :class:`~mongoengine.Document`
 will have their own **collection** in the database. The name of the collection
-is by default the name of the class, coverted to lowercase (so in the example
+is by default the name of the class, converted to lowercase (so in the example
 above, the collection would be called `page`). If you need to change the name
 of the collection (e.g. to use MongoEngine with an existing database), then
 create a class dictionary attribute called :attr:`meta` on your document, and
@@ -459,13 +465,22 @@ by creating a list of index specifications called :attr:`indexes` in the
 either be a single field name, a tuple containing multiple field names, or a
 dictionary containing a full index definition. A direction may be specified on
 fields by prefixing the field name with a **+** (for ascending) or a **-** sign
-(for descending). Note that direction only matters on multi-field indexes. ::
+(for descending). Note that direction only matters on multi-field indexes.
+Text indexes may be specified by prefixing the field name with a **$**. ::
 
     class Page(Document):
         title = StringField()
         rating = StringField()
+        created = DateTimeField()
         meta = {
-            'indexes': ['title', ('title', '-rating')]
+            'indexes': [
+                'title',
+                ('title', '-rating'),
+                {
+                    'fields': ['created'],
+                    'expireAfterSeconds': 3600
+                }
+            ]
         }
 
 If a dictionary is passed then the following options are available:
@@ -531,6 +546,8 @@ field name to the index definition.
 Sometimes its more efficient to index parts of Embedded / dictionary fields,
 in this case use 'dot' notation to identify the value to index eg: `rank.title`
 
+.. _geospatial-indexes:
+
 Geospatial indexes
 ------------------
 
@@ -541,6 +558,9 @@ The following fields will explicitly add a "2dsphere" index:
     - :class:`~mongoengine.fields.PointField`
     - :class:`~mongoengine.fields.LineStringField`
     - :class:`~mongoengine.fields.PolygonField`
+    - :class:`~mongoengine.fields.MultiPointField`
+    - :class:`~mongoengine.fields.MultiLineStringField`
+    - :class:`~mongoengine.fields.MultiPolygonField`
 
 As "2dsphere" indexes can be part of a compound index, you may not want the
 automatic index but would prefer a compound index.  In this example we turn off
@@ -652,11 +672,11 @@ Shard keys
 ==========
 
 If your collection is sharded, then you need to specify the shard key as a tuple,
-using the :attr:`shard_key` attribute of :attr:`-mongoengine.Document.meta`.
+using the :attr:`shard_key` attribute of :attr:`~mongoengine.Document.meta`.
 This ensures that the shard key is sent with the query when calling the
 :meth:`~mongoengine.document.Document.save` or
 :meth:`~mongoengine.document.Document.update` method on an existing
-:class:`-mongoengine.Document` instance::
+:class:`~mongoengine.Document` instance::
 
     class LogEntry(Document):
         machine = StringField()
@@ -678,7 +698,7 @@ defined, you may subclass it and add any extra fields or methods you may need.
 As this is new class is not a direct subclass of
 :class:`~mongoengine.Document`, it will not be stored in its own collection; it
 will use the same collection as its superclass uses. This allows for more
-convenient and efficient retrieval of related documents - all you need do is
+convenient and efficient retrieval of related documents -- all you need do is
 set :attr:`allow_inheritance` to True in the :attr:`meta` data for a
 document.::
 
@@ -692,12 +712,12 @@ document.::
     class DatedPage(Page):
         date = DateTimeField()
 
-.. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults
+.. note:: From 0.8 onwards :attr:`allow_inheritance` defaults
           to False, meaning you must set it to True to use inheritance.
 
 Working with existing data
 --------------------------
-As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and
+As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and
 easily get working with existing data.  Just define the document to match
 the expected schema in your database ::
 
@@ -720,7 +740,7 @@ Abstract classes
 
 If you want to add some extra functionality to a group of Document classes but
 you don't need or want the overhead of inheritance you can use the
-:attr:`abstract` attribute of :attr:`-mongoengine.Document.meta`.
+:attr:`abstract` attribute of :attr:`~mongoengine.Document.meta`.
 This won't turn on :ref:`document-inheritance` but will allow you to keep your
 code DRY::
 

docs/guide/document-instances.rst

@@ -2,7 +2,7 @@
 Documents instances
 ===================
 To create a new document object, create an instance of the relevant document
-class, providing values for its fields as its constructor keyword arguments.
+class, providing values for its fields as constructor keyword arguments.
 You may provide values for any of the fields on the document::
 
     >>> page = Page(title="Test Page")
@@ -32,11 +32,11 @@ already exist, then any changes will be updated atomically.  For example::
 
     Changes to documents are tracked and on the whole perform ``set`` operations.
 
-    * ``list_field.push(0)`` - *sets* the resulting list
-    * ``del(list_field)``   - *unsets* whole list
+    * ``list_field.push(0)`` --- *sets* the resulting list
+    * ``del(list_field)``   --- *unsets* whole list
 
     With lists its preferable to use ``Doc.update(push__list_field=0)`` as
-    this stops the whole list being updated - stopping any race conditions.
+    this stops the whole list being updated --- stopping any race conditions.
 
 .. seealso::
     :ref:`guide-atomic-updates`
@@ -74,7 +74,7 @@ Cascading Saves
 If your document contains :class:`~mongoengine.fields.ReferenceField` or
 :class:`~mongoengine.fields.GenericReferenceField` objects, then by default the
 :meth:`~mongoengine.Document.save` method will not save any changes to
-those objects.  If you want all references to also be saved also, noting each
+those objects.  If you want all references to be saved also, noting each
 save is a separate query, then passing :attr:`cascade` as True
 to the save method will cascade any saves.
 
@@ -113,12 +113,13 @@ you may still use :attr:`id` to access the primary key if you want::
     >>> bob.id == bob.email == 'bob@example.com'
     True
 
-You can also access the document's "primary key" using the :attr:`pk` field; in
-is an alias to :attr:`id`::
+You can also access the document's "primary key" using the :attr:`pk` field,
+it's an alias to :attr:`id`::
 
     >>> page = Page(title="Another Test Page")
     >>> page.save()
     >>> page.id == page.pk
+    True
 
 .. note::
 

docs/guide/gridfs.rst

@@ -46,7 +46,7 @@ slightly different manner.  First, a new file must be created by calling the
     marmot.photo.write('some_more_image_data')
     marmot.photo.close()
 
-    marmot.photo.save()
+    marmot.save()
 
 Deletion
 --------

docs/guide/index.rst

@@ -12,3 +12,4 @@ User Guide
    querying
    gridfs
    signals
+   text-indexes

docs/guide/querying.rst

@@ -17,7 +17,7 @@ fetch documents from the database::
 
     As of MongoEngine 0.8 the querysets utilise a local cache.  So iterating
     it multiple times will only cause a single query.  If this is not the
-    desired behavour you can call :class:`~mongoengine.QuerySet.no_cache`
+    desired behaviour you can call :class:`~mongoengine.QuerySet.no_cache`
     (version **0.8.3+**) to return a non-caching queryset.
 
 Filtering queries
@@ -42,7 +42,7 @@ syntax::
 
 Query operators
 ===============
-Operators other than equality may also be used in queries; just attach the
+Operators other than equality may also be used in queries --- just attach the
 operator name to a key with a double-underscore::
 
     # Only find users whose age is 18 or less
@@ -84,19 +84,20 @@ expressions:
 Geo queries
 -----------
 
-There are a few special operators for performing geographical queries. The following
-were added in 0.8 for:  :class:`~mongoengine.fields.PointField`,
+There are a few special operators for performing geographical queries.
+The following were added in MongoEngine 0.8 for
+:class:`~mongoengine.fields.PointField`,
 :class:`~mongoengine.fields.LineStringField` and
 :class:`~mongoengine.fields.PolygonField`:
 
-* ``geo_within`` -- Check if a geometry is within a polygon.  For ease of use
-    it accepts either a geojson geometry or just the polygon coordinates eg::
+* ``geo_within`` -- check if a geometry is within a polygon. For ease of use
+  it accepts either a geojson geometry or just the polygon coordinates eg::
 
         loc.objects(point__geo_within=[[[40, 5], [40, 6], [41, 6], [40, 5]]])
         loc.objects(point__geo_within={"type": "Polygon",
                                  "coordinates": [[[40, 5], [40, 6], [41, 6], [40, 5]]]})
 
-* ``geo_within_box`` - simplified geo_within searching with a box eg::
+* ``geo_within_box`` -- simplified geo_within searching with a box eg::
 
         loc.objects(point__geo_within_box=[(-125.0, 35.0), (-100.0, 40.0)])
         loc.objects(point__geo_within_box=[<bottom left coordinates>, <upper right coordinates>])
@@ -132,23 +133,21 @@ were added in 0.8 for:  :class:`~mongoengine.fields.PointField`,
         loc.objects(poly__geo_intersects={"type": "Polygon",
                                           "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]})
 
-* ``near`` -- Find all the locations near a given point::
+* ``near`` -- find all the locations near a given point::
 
         loc.objects(point__near=[40, 5])
         loc.objects(point__near={"type": "Point", "coordinates": [40, 5]})
 
-
-    You can also set the maximum distance in meters as well::
+  You can also set the maximum distance in meters as well::
 
         loc.objects(point__near=[40, 5], point__max_distance=1000)
 
-
 The older 2D indexes are still supported with the
 :class:`~mongoengine.fields.GeoPointField`:
 
 * ``within_distance`` -- provide a list containing a point and a maximum
   distance (e.g. [(41.342, -87.653), 5])
-* ``within_spherical_distance`` -- Same as above but using the spherical geo model
+* ``within_spherical_distance`` -- same as above but using the spherical geo model
   (e.g. [(41.342, -87.653), 5/earth_radius])
 * ``near`` -- order the documents by how close they are to a given point
 * ``near_sphere`` -- Same as above but using the spherical geo model
@@ -198,12 +197,14 @@ However, this doesn't map well to the syntax so you can also use a capital S ins
 
     Post.objects(comments__by="joe").update(inc__comments__S__votes=1)
 
-    .. note:: Due to Mongo currently the $ operator only applies to the first matched item in the query.
+.. note::
+    Due to :program:`Mongo`, currently the $ operator only applies to the
+    first matched item in the query.
 
 
 Raw queries
 -----------
-It is possible to provide a raw PyMongo query as a query parameter, which will
+It is possible to provide a raw :mod:`PyMongo` query as a query parameter, which will
 be integrated directly into the query. This is done using the ``__raw__``
 keyword argument::
 
@@ -213,12 +214,12 @@ keyword argument::
 
 Limiting and skipping results
 =============================
-Just as with traditional ORMs, you may limit the number of results returned, or
+Just as with traditional ORMs, you may limit the number of results returned or
 skip a number or results in you query.
 :meth:`~mongoengine.queryset.QuerySet.limit` and
 :meth:`~mongoengine.queryset.QuerySet.skip` and methods are available on
-:class:`~mongoengine.queryset.QuerySet` objects, but the prefered syntax for
-achieving this is using array-slicing syntax::
+:class:`~mongoengine.queryset.QuerySet` objects, but the `array-slicing` syntax
+is preferred for achieving this::
 
     # Only the first 5 people
     users = User.objects[:5]
@@ -252,10 +253,10 @@ To retrieve a result that should be unique in the collection, use
 no document matches the query, and
 :class:`~mongoengine.queryset.MultipleObjectsReturned`
 if more than one document matched the query.  These exceptions are merged into
-your document defintions eg: `MyDoc.DoesNotExist`
+your document definitions eg: `MyDoc.DoesNotExist`
 
 A variation of this method exists,
-:meth:`~mongoengine.queryset.Queryset.get_or_create`, that will create a new
+:meth:`~mongoengine.queryset.QuerySet.get_or_create`, that will create a new
 document with the query arguments if no documents match the query. An
 additional keyword argument, :attr:`defaults` may be provided, which will be
 used as default values for the new document, in the case that it should need
@@ -266,9 +267,13 @@ to be created::
     >>> a.name == b.name and a.age == b.age
     True
 
+.. warning::
+    :meth:`~mongoengine.queryset.QuerySet.get_or_create` method is deprecated
+    since :mod:`mongoengine` 0.8.
+
 Default Document queries
 ========================
-By default, the objects :attr:`~mongoengine.Document.objects` attribute on a
+By default, the objects :attr:`~Document.objects` attribute on a
 document returns a :class:`~mongoengine.queryset.QuerySet` that doesn't filter
 the collection -- it returns all objects. This may be changed by defining a
 method on a document that modifies a queryset. The method should accept two
@@ -311,7 +316,7 @@ Should you want to add custom methods for interacting with or filtering
 documents, extending the :class:`~mongoengine.queryset.QuerySet` class may be
 the way to go. To use a custom :class:`~mongoengine.queryset.QuerySet` class on
 a document, set ``queryset_class`` to the custom class in a
-:class:`~mongoengine.Document`\ s ``meta`` dictionary::
+:class:`~mongoengine.Document`'s ``meta`` dictionary::
 
     class AwesomerQuerySet(QuerySet):
 
@@ -488,22 +493,28 @@ calling it with keyword arguments::
 Atomic updates
 ==============
 Documents may be updated atomically by using the
-:meth:`~mongoengine.queryset.QuerySet.update_one` and
-:meth:`~mongoengine.queryset.QuerySet.update` methods on a
-:meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers"
-that you may use with these methods:
+:meth:`~mongoengine.queryset.QuerySet.update_one`,
+:meth:`~mongoengine.queryset.QuerySet.update` and
+:meth:`~mongoengine.queryset.QuerySet.modify` methods on a
+:class:`~mongoengine.queryset.QuerySet` or
+:meth:`~mongoengine.Document.modify` and
+:meth:`~mongoengine.Document.save` (with :attr:`save_condition` argument) on a
+:class:`~mongoengine.Document`.
+There are several different "modifiers" that you may use with these methods:
 
 * ``set`` -- set a particular value
-* ``unset`` -- delete a particular value (since MongoDB v1.3+)
+* ``unset`` -- delete a particular value (since MongoDB v1.3)
 * ``inc`` -- increment a value by a given amount
 * ``dec`` -- decrement a value by a given amount
 * ``push`` -- append a value to a list
 * ``push_all`` -- append several values to a list
-* ``pop`` -- remove the first or last element of a list
+* ``pop`` -- remove the first or last element of a list `depending on the value`_
 * ``pull`` -- remove a value from a list
 * ``pull_all`` -- remove several values from a list
 * ``add_to_set`` -- add value to a list only if its not in the list already
 
+.. _depending on the value: http://docs.mongodb.org/manual/reference/operator/update/pop/
+
 The syntax for atomic updates is similar to the querying syntax, but the
 modifier comes before the field, not after it::
 
@@ -522,6 +533,13 @@ modifier comes before the field, not after it::
     >>> post.tags
     ['database', 'nosql']
 
+.. note::
+
+    If no modifier operator is specified the default will be ``$set``. So the following sentences are identical::
+
+        >>> BlogPost.objects(id=post.id).update(title='Example Post')
+        >>> BlogPost.objects(id=post.id).update(set__title='Example Post')
+
 .. note::
 
     In version 0.5 the :meth:`~mongoengine.Document.save` runs atomic updates
@@ -645,3 +663,4 @@ following example shows how the substitutions are made::
         return comments;
     }
     """)
+

docs/guide/signals.rst

@@ -35,25 +35,25 @@ Available signals include:
   :class:`~mongoengine.EmbeddedDocument` instance has been completed.
 
 `pre_save`
-  Called within :meth:`~mongoengine.document.Document.save` prior to performing
+  Called within :meth:`~mongoengine.Document.save` prior to performing
   any actions.
 
 `pre_save_post_validation`
-  Called within :meth:`~mongoengine.document.Document.save` after validation
+  Called within :meth:`~mongoengine.Document.save` after validation
   has taken place but before saving.
 
 `post_save`
-  Called within :meth:`~mongoengine.document.Document.save` after all actions
+  Called within :meth:`~mongoengine.Document.save` after all actions
   (validation, insert/update, cascades, clearing dirty flags) have completed
   successfully.  Passed the additional boolean keyword argument `created` to
   indicate if the save was an insert or an update.
 
 `pre_delete`
-  Called within :meth:`~mongoengine.document.Document.delete` prior to
+  Called within :meth:`~mongoengine.Document.delete` prior to
   attempting the delete operation.
 
 `post_delete`
-  Called within :meth:`~mongoengine.document.Document.delete` upon successful
+  Called within :meth:`~mongoengine.Document.delete` upon successful
   deletion of the record.
 
 `pre_bulk_insert`
@@ -145,7 +145,7 @@ cleaner looking while still allowing manual execution of the callback::
 ReferenceFields and Signals
 ---------------------------
 
-Currently `reverse_delete_rules` do not trigger signals on the other part of
+Currently `reverse_delete_rule` does not trigger signals on the other part of
 the relationship.  If this is required you must manually handle the
 reverse deletion.
 

docs/guide/text-indexes.rst

@@ -0,0 +1,51 @@
+===========
+Text Search
+===========
+
+After MongoDB 2.4 version, supports search documents by text indexes.
+
+
+Defining a Document with text index
+===================================
+Use the *$* prefix to set a text index, Look the declaration::
+  
+  class News(Document):
+      title = StringField()
+      content = StringField()
+      is_active = BooleanField()
+
+      meta = {'indexes': [
+          {'fields': ['$title', "$content"],
+           'default_language': 'english',
+           'weight': {'title': 10, 'content': 2}
+          }
+      ]}
+
+
+
+Querying
+========
+
+Saving a document::
+
+  News(title="Using mongodb text search",
+       content="Testing text search").save()
+
+  News(title="MongoEngine 0.9 released",
+       content="Various improvements").save()
+
+Next, start a text search using :attr:`QuerySet.search_text` method::
+  
+  document = News.objects.search_text('testing').first()
+  document.title # may be: "Using mongodb text search"
+  
+  document = News.objects.search_text('released').first()
+  document.title # may be: "MongoEngine 0.9 released"
+
+
+Ordering by text score
+======================
+
+::
+
+  objects = News.objects.search('mongo').order_by('$text_score')

docs/index.rst

@@ -14,7 +14,7 @@ MongoDB. To install it, simply run
   MongoEngine.
 
 :doc:`guide/index`
-  The Full guide to MongoEngine - from modeling documents to storing files,
+  The Full guide to MongoEngine --- from modeling documents to storing files,
   from querying for data to firing signals and *everything* between.
 
 :doc:`apireference`

docs/tutorial.rst

@@ -65,7 +65,7 @@ which fields a :class:`User` may have, and what types of data they might store::
         first_name = StringField(max_length=50)
         last_name = StringField(max_length=50)
 
-This looks similar to how a the structure of a table would be defined in a
+This looks similar to how the structure of a table would be defined in a
 regular ORM. The key difference is that this schema will never be passed on to
 MongoDB --- this will only be enforced at the application level, making future
 changes easy to manage. Also, the User documents will be stored in a

docs/upgrade.rst

@@ -5,7 +5,7 @@ Upgrading
 0.8.7
 *****
 
-Calling reload on deleted / nonexistant documents now raises a DoesNotExist
+Calling reload on deleted / nonexistent documents now raises a DoesNotExist
 exception.
 
 
@@ -263,7 +263,7 @@ update your code like so: ::
     [m for m in mammals]                                               # This will return all carnivores
 
 Len iterates the queryset
---------------------------
+-------------------------
 
 If you ever did `len(queryset)` it previously did a `count()` under the covers,
 this caused some unusual issues.  As `len(queryset)` is most often used by

mongoengine/__init__.py

@@ -15,7 +15,7 @@ import django
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
 
-VERSION = (0, 8, 7)
+VERSION = (0, 9, 0)
 
 
 def get_version():

mongoengine/base/datastructures.py

@@ -1,12 +1,14 @@
 import weakref
+import functools
+import itertools
 from mongoengine.common import _import_class
+from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
 
-__all__ = ("BaseDict", "BaseList")
+__all__ = ("BaseDict", "BaseList", "EmbeddedDocumentList")
 
 
 class BaseDict(dict):
-    """A special dict so we can watch any changes
-    """
+    """A special dict so we can watch any changes"""
 
     _dereferenced = False
     _instance = None
@@ -21,29 +23,37 @@ class BaseDict(dict):
         self._name = name
         return super(BaseDict, self).__init__(dict_items)
 
-    def __getitem__(self, *args, **kwargs):
-        value = super(BaseDict, self).__getitem__(*args, **kwargs)
+    def __getitem__(self, key, *args, **kwargs):
+        value = super(BaseDict, self).__getitem__(key)
 
         EmbeddedDocument = _import_class('EmbeddedDocument')
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
+        elif not isinstance(value, BaseDict) and isinstance(value, dict):
+            value = BaseDict(value, None, '%s.%s' % (self._name, key))
+            super(BaseDict, self).__setitem__(key, value)
+            value._instance = self._instance
+        elif not isinstance(value, BaseList) and isinstance(value, list):
+            value = BaseList(value, None, '%s.%s' % (self._name, key))
+            super(BaseDict, self).__setitem__(key, value)
+            value._instance = self._instance
         return value
 
-    def __setitem__(self, *args, **kwargs):
-        self._mark_as_changed()
-        return super(BaseDict, self).__setitem__(*args, **kwargs)
+    def __setitem__(self, key, value, *args, **kwargs):
+        self._mark_as_changed(key)
+        return super(BaseDict, self).__setitem__(key, value)
 
     def __delete__(self, *args, **kwargs):
         self._mark_as_changed()
         return super(BaseDict, self).__delete__(*args, **kwargs)
 
-    def __delitem__(self, *args, **kwargs):
-        self._mark_as_changed()
-        return super(BaseDict, self).__delitem__(*args, **kwargs)
+    def __delitem__(self, key, *args, **kwargs):
+        self._mark_as_changed(key)
+        return super(BaseDict, self).__delitem__(key)
 
-    def __delattr__(self, *args, **kwargs):
-        self._mark_as_changed()
-        return super(BaseDict, self).__delattr__(*args, **kwargs)
+    def __delattr__(self, key, *args, **kwargs):
+        self._mark_as_changed(key)
+        return super(BaseDict, self).__delattr__(key)
 
     def __getstate__(self):
         self.instance = None
@@ -66,13 +76,20 @@ class BaseDict(dict):
         self._mark_as_changed()
         return super(BaseDict, self).popitem(*args, **kwargs)
 
+    def setdefault(self, *args, **kwargs):
+        self._mark_as_changed()
+        return super(BaseDict, self).setdefault(*args, **kwargs)
+
     def update(self, *args, **kwargs):
         self._mark_as_changed()
         return super(BaseDict, self).update(*args, **kwargs)
 
-    def _mark_as_changed(self):
+    def _mark_as_changed(self, key=None):
         if hasattr(self._instance, '_mark_as_changed'):
-            self._instance._mark_as_changed(self._name)
+            if key:
+                self._instance._mark_as_changed('%s.%s' % (self._name, key))
+            else:
+                self._instance._mark_as_changed(self._name)
 
 
 class BaseList(list):
@@ -90,23 +107,37 @@ class BaseList(list):
         if isinstance(instance, (Document, EmbeddedDocument)):
             self._instance = weakref.proxy(instance)
         self._name = name
-        return super(BaseList, self).__init__(list_items)
+        super(BaseList, self).__init__(list_items)
 
-    def __getitem__(self, *args, **kwargs):
-        value = super(BaseList, self).__getitem__(*args, **kwargs)
+    def __getitem__(self, key, *args, **kwargs):
+        value = super(BaseList, self).__getitem__(key)
 
         EmbeddedDocument = _import_class('EmbeddedDocument')
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
+        elif not isinstance(value, BaseDict) and isinstance(value, dict):
+            value = BaseDict(value, None, '%s.%s' % (self._name, key))
+            super(BaseList, self).__setitem__(key, value)
+            value._instance = self._instance
+        elif not isinstance(value, BaseList) and isinstance(value, list):
+            value = BaseList(value, None, '%s.%s' % (self._name, key))
+            super(BaseList, self).__setitem__(key, value)
+            value._instance = self._instance
         return value
 
-    def __setitem__(self, *args, **kwargs):
-        self._mark_as_changed()
-        return super(BaseList, self).__setitem__(*args, **kwargs)
+    def __setitem__(self, key, value, *args, **kwargs):
+        if isinstance(key, slice):
+            self._mark_as_changed()
+        else:
+            self._mark_as_changed(key)
+        return super(BaseList, self).__setitem__(key, value)
 
-    def __delitem__(self, *args, **kwargs):
-        self._mark_as_changed()
-        return super(BaseList, self).__delitem__(*args, **kwargs)
+    def __delitem__(self, key, *args, **kwargs):
+        if isinstance(key, slice):
+            self._mark_as_changed()
+        else:
+            self._mark_as_changed(key)
+        return super(BaseList, self).__delitem__(key)
 
     def __setslice__(self, *args, **kwargs):
         self._mark_as_changed()
@@ -153,6 +184,266 @@ class BaseList(list):
         self._mark_as_changed()
         return super(BaseList, self).sort(*args, **kwargs)
 
-    def _mark_as_changed(self):
+    def _mark_as_changed(self, key=None):
         if hasattr(self._instance, '_mark_as_changed'):
-            self._instance._mark_as_changed(self._name)
+            if key:
+                self._instance._mark_as_changed('%s.%s' % (self._name, key))
+            else:
+                self._instance._mark_as_changed(self._name)
+
+
+class EmbeddedDocumentList(BaseList):
+
+    @classmethod
+    def __match_all(cls, i, kwargs):
+        items = kwargs.items()
+        return all([
+            getattr(i, k) == v or str(getattr(i, k)) == v for k, v in items
+        ])
+
+    @classmethod
+    def __only_matches(cls, obj, kwargs):
+        if not kwargs:
+            return obj
+        return filter(lambda i: cls.__match_all(i, kwargs), obj)
+
+    def __init__(self, list_items, instance, name):
+        super(EmbeddedDocumentList, self).__init__(list_items, instance, name)
+        self._instance = instance
+
+    def filter(self, **kwargs):
+        """
+        Filters the list by only including embedded documents with the
+        given keyword arguments.
+
+        :param kwargs: The keyword arguments corresponding to the fields to
+         filter on. *Multiple arguments are treated as if they are ANDed
+         together.*
+        :return: A new ``EmbeddedDocumentList`` containing the matching
+         embedded documents.
+
+        Raises ``AttributeError`` if a given keyword is not a valid field for
+        the embedded document class.
+        """
+        values = self.__only_matches(self, kwargs)
+        return EmbeddedDocumentList(values, self._instance, self._name)
+
+    def exclude(self, **kwargs):
+        """
+        Filters the list by excluding embedded documents with the given
+        keyword arguments.
+
+        :param kwargs: The keyword arguments corresponding to the fields to
+         exclude on. *Multiple arguments are treated as if they are ANDed
+         together.*
+        :return: A new ``EmbeddedDocumentList`` containing the non-matching
+         embedded documents.
+
+        Raises ``AttributeError`` if a given keyword is not a valid field for
+        the embedded document class.
+        """
+        exclude = self.__only_matches(self, kwargs)
+        values = [item for item in self if item not in exclude]
+        return EmbeddedDocumentList(values, self._instance, self._name)
+
+    def count(self):
+        """
+        The number of embedded documents in the list.
+
+        :return: The length of the list, equivalent to the result of ``len()``.
+        """
+        return len(self)
+
+    def get(self, **kwargs):
+        """
+        Retrieves an embedded document determined by the given keyword
+        arguments.
+
+        :param kwargs: The keyword arguments corresponding to the fields to
+         search on. *Multiple arguments are treated as if they are ANDed
+         together.*
+        :return: The embedded document matched by the given keyword arguments.
+
+        Raises ``DoesNotExist`` if the arguments used to query an embedded
+        document returns no results. ``MultipleObjectsReturned`` if more
+        than one result is returned.
+        """
+        values = self.__only_matches(self, kwargs)
+        if len(values) == 0:
+            raise DoesNotExist(
+                "%s matching query does not exist." % self._name
+            )
+        elif len(values) > 1:
+            raise MultipleObjectsReturned(
+                "%d items returned, instead of 1" % len(values)
+            )
+
+        return values[0]
+
+    def first(self):
+        """
+        Returns the first embedded document in the list, or ``None`` if empty.
+        """
+        if len(self) > 0:
+            return self[0]
+
+    def create(self, **values):
+        """
+        Creates a new embedded document and saves it to the database.
+
+        .. note::
+            The embedded document changes are not automatically saved
+            to the database after calling this method.
+
+        :param values: A dictionary of values for the embedded document.
+        :return: The new embedded document instance.
+        """
+        name = self._name
+        EmbeddedClass = self._instance._fields[name].field.document_type_obj
+        self._instance[self._name].append(EmbeddedClass(**values))
+
+        return self._instance[self._name][-1]
+
+    def save(self, *args, **kwargs):
+        """
+        Saves the ancestor document.
+
+        :param args: Arguments passed up to the ancestor Document's save
+         method.
+        :param kwargs: Keyword arguments passed up to the ancestor Document's
+         save method.
+        """
+        self._instance.save(*args, **kwargs)
+
+    def delete(self):
+        """
+        Deletes the embedded documents from the database.
+
+        .. note::
+            The embedded document changes are not automatically saved
+            to the database after calling this method.
+
+        :return: The number of entries deleted.
+        """
+        values = list(self)
+        for item in values:
+            self._instance[self._name].remove(item)
+
+        return len(values)
+
+    def update(self, **update):
+        """
+        Updates the embedded documents with the given update values.
+
+        .. note::
+            The embedded document changes are not automatically saved
+            to the database after calling this method.
+
+        :param update: A dictionary of update values to apply to each
+         embedded document.
+        :return: The number of entries updated.
+        """
+        if len(update) == 0:
+            return 0
+        values = list(self)
+        for item in values:
+            for k, v in update.items():
+                setattr(item, k, v)
+
+        return len(values)
+
+
+class StrictDict(object):
+    __slots__ = ()
+    _special_fields = set(['get', 'pop', 'iteritems', 'items', 'keys', 'create'])
+    _classes = {}
+    def __init__(self, **kwargs):
+        for k,v in kwargs.iteritems():
+            setattr(self, k, v)
+    def __getitem__(self, key):
+        key = '_reserved_' + key if key in self._special_fields else key
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key)
+    def __setitem__(self, key, value):
+        key = '_reserved_' + key if key in self._special_fields else key
+        return setattr(self, key, value)
+    def __contains__(self, key):
+        return hasattr(self, key)
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+    def pop(self, key, default=None):
+        v = self.get(key, default)
+        try:
+            delattr(self, key)
+        except AttributeError:
+            pass
+        return v
+    def iteritems(self):
+        for key in self:
+            yield key, self[key]
+    def items(self):
+        return [(k, self[k]) for k in iter(self)]
+    def keys(self):
+        return list(iter(self))
+    def __iter__(self):
+        return (key for key in self.__slots__ if hasattr(self, key))
+    def __len__(self):
+        return len(list(self.iteritems()))
+    def __eq__(self, other):
+        return self.items() == other.items()
+    def __neq__(self, other):
+        return self.items() != other.items()
+
+    @classmethod
+    def create(cls, allowed_keys):
+        allowed_keys_tuple = tuple(('_reserved_' + k if k in cls._special_fields else k) for k in allowed_keys)
+        allowed_keys = frozenset(allowed_keys_tuple)
+        if allowed_keys not in cls._classes:
+            class SpecificStrictDict(cls):
+                __slots__ = allowed_keys_tuple
+                def __repr__(self):
+                    return "{%s}" % ', '.join('"{0!s}": {0!r}'.format(k,v) for (k,v) in self.iteritems())
+            cls._classes[allowed_keys] = SpecificStrictDict
+        return cls._classes[allowed_keys]
+
+
+class SemiStrictDict(StrictDict):
+    __slots__ = ('_extras')
+    _classes = {}
+    def __getattr__(self, attr):
+        try:
+            super(SemiStrictDict, self).__getattr__(attr)
+        except AttributeError:
+            try:
+                return self.__getattribute__('_extras')[attr]
+            except KeyError as e:
+                raise AttributeError(e)
+    def __setattr__(self, attr, value):
+        try:
+            super(SemiStrictDict, self).__setattr__(attr, value)
+        except AttributeError:
+            try:
+                self._extras[attr] = value
+            except AttributeError:
+                self._extras = {attr: value}
+
+    def __delattr__(self, attr):
+        try:
+            super(SemiStrictDict, self).__delattr__(attr)
+        except AttributeError:
+            try:
+                del self._extras[attr]
+            except KeyError as e:
+                raise AttributeError(e)
+
+    def __iter__(self):
+        try:
+            extras_iter = iter(self.__getattribute__('_extras'))
+        except AttributeError:
+            extras_iter = ()
+        return itertools.chain(super(SemiStrictDict, self).__iter__(), extras_iter)

mongoengine/base/document.py

@@ -12,12 +12,17 @@ from bson.son import SON
 from mongoengine import signals
 from mongoengine.common import _import_class
 from mongoengine.errors import (ValidationError, InvalidDocumentError,
-                                LookUpError)
-from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type,
-                                        to_str_keys_recursive)
+                                LookUpError, FieldDoesNotExist)
+from mongoengine.python_support import PY3, txt_type
 
 from mongoengine.base.common import get_document, ALLOW_INHERITANCE
-from mongoengine.base.datastructures import BaseDict, BaseList
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    EmbeddedDocumentList,
+    StrictDict,
+    SemiStrictDict
+)
 from mongoengine.base.fields import ComplexBaseField
 
 __all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
@@ -26,11 +31,12 @@ NON_FIELD_ERRORS = '__all__'
 
 
 class BaseDocument(object):
+    __slots__ = ('_changed_fields', '_initialised', '_created', '_data',
+                 '_dynamic_fields', '_auto_id_field', '_db_field_map', '__weakref__')
 
     _dynamic = False
-    _created = True
     _dynamic_lock = True
-    _initialised = False
+    STRICT = False
 
     def __init__(self, *args, **values):
         """
@@ -39,6 +45,8 @@ class BaseDocument(object):
         :param __auto_convert: Try and will cast python objects to Object types
         :param values: A dictionary of values for the document
         """
+        self._initialised = False
+        self._created = True
         if args:
             # Combine positional arguments with named arguments.
             # We only want named arguments.
@@ -49,21 +57,48 @@ class BaseDocument(object):
             for value in args:
                 name = next(field)
                 if name in values:
-                    raise TypeError("Multiple values for keyword argument '" + name + "'")
+                    raise TypeError(
+                        "Multiple values for keyword argument '" + name + "'")
                 values[name] = value
+
         __auto_convert = values.pop("__auto_convert", True)
+
+        # 399: set default values only to fields loaded from DB
+        __only_fields = set(values.pop("__only_fields", values))
+
+        _created = values.pop("_created", True)
+
         signals.pre_init.send(self.__class__, document=self, values=values)
 
+        # Check if there are undefined fields supplied, if so raise an
+        # Exception.
+        if not self._dynamic:
+            for var in values.keys():
+                if var not in self._fields.keys() + ['id', 'pk', '_cls', '_text_score']:
+                    msg = (
+                        "The field '{0}' does not exist on the document '{1}'"
+                    ).format(var, self._class_name)
+                    raise FieldDoesNotExist(msg)
+
+        if self.STRICT and not self._dynamic:
+            self._data = StrictDict.create(allowed_keys=self._fields_ordered)()
+        else:
+            self._data = SemiStrictDict.create(
+                allowed_keys=self._fields_ordered)()
+
         self._data = {}
         self._dynamic_fields = SON()
 
         # Assign default values to instance
         for key, field in self._fields.iteritems():
-            if self._db_field_map.get(key, key) in values:
+            if self._db_field_map.get(key, key) in __only_fields:
                 continue
             value = getattr(self, key, None)
             setattr(self, key, value)
 
+        if "_cls" not in values:
+            self._cls = self._class_name
+
         # Set passed values after initialisation
         if self._dynamic:
             dynamic_data = {}
@@ -97,6 +132,7 @@ class BaseDocument(object):
 
         # Flag initialised
         self._initialised = True
+        self._created = _created
         signals.post_init.send(self.__class__, document=self)
 
     def __delattr__(self, *args, **kwargs):
@@ -130,18 +166,26 @@ class BaseDocument(object):
                 self._data[name] = value
                 if hasattr(self, '_changed_fields'):
                     self._mark_as_changed(name)
+        try:
+            self__created = self._created
+        except AttributeError:
+            self__created = True
 
-        if (self._is_document and not self._created and
-           name in self._meta.get('shard_key', tuple()) and
-           self._data.get(name) != value):
+        if (self._is_document and not self__created and
+                name in self._meta.get('shard_key', tuple()) and
+                self._data.get(name) != value):
             OperationError = _import_class('OperationError')
             msg = "Shard Keys are immutable. Tried to update %s" % name
             raise OperationError(msg)
 
+        try:
+            self__initialised = self._initialised
+        except AttributeError:
+            self__initialised = False
         # Check if the user has created a new instance of a class
-        if (self._is_document and self._initialised
-           and self._created and name == self._meta['id_field']):
-                super(BaseDocument, self).__setattr__('_created', False)
+        if (self._is_document and self__initialised
+                and self__created and name == self._meta['id_field']):
+            super(BaseDocument, self).__setattr__('_created', False)
 
         super(BaseDocument, self).__setattr__(name, value)
 
@@ -158,9 +202,11 @@ class BaseDocument(object):
         if isinstance(data["_data"], SON):
             data["_data"] = self.__class__._from_son(data["_data"])._data
         for k in ('_changed_fields', '_initialised', '_created', '_data',
-                  '_fields_ordered', '_dynamic_fields'):
+                  '_dynamic_fields'):
             if k in data:
                 setattr(self, k, data[k])
+        if '_fields_ordered' in data:
+            setattr(type(self), '_fields_ordered', data['_fields_ordered'])
         dynamic_fields = data.get('_dynamic_fields') or SON()
         for k in dynamic_fields.keys():
             setattr(self, k, data["_data"].get(k))
@@ -182,7 +228,7 @@ class BaseDocument(object):
         """Dictionary-style field access, set a field's value.
         """
         # Ensure that the field exists before settings its value
-        if name not in self._fields:
+        if not self._dynamic and name not in self._fields:
             raise KeyError(name)
         return setattr(self, name, value)
 
@@ -201,7 +247,7 @@ class BaseDocument(object):
             u = self.__str__()
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = '[Bad Unicode data]'
-        repr_type = type(u)
+        repr_type = str if u is None else type(u)
         return repr_type('<%s: %s>' % (self.__class__.__name__, u))
 
     def __str__(self):
@@ -213,9 +259,12 @@ class BaseDocument(object):
         return txt_type('%s object' % self.__class__.__name__)
 
     def __eq__(self, other):
-        if isinstance(other, self.__class__) and hasattr(other, 'id'):
-            if self.id == other.id:
-                return True
+        if isinstance(other, self.__class__) and hasattr(other, 'id') and other.id is not None:
+            return self.id == other.id
+        if isinstance(other, DBRef):
+            return self._get_collection_name() == other.collection and self.id == other.id
+        if self.id is None:
+            return self is other
         return False
 
     def __ne__(self, other):
@@ -238,21 +287,56 @@ class BaseDocument(object):
         """
         pass
 
-    def to_mongo(self):
-        """Return as SON data ready for use with MongoDB.
+    def get_text_score(self):
         """
+        Get text score from text query
+        """
+
+        if '_text_score' not in self._data:
+            raise InvalidDocumentError('This document is not originally built from a text query')
+
+        return self._data['_text_score']
+
+    def to_mongo(self, use_db_field=True, fields=None):
+        """
+        Return as SON data ready for use with MongoDB.
+        """
+        if not fields:
+            fields = []
+        
         data = SON()
         data["_id"] = None
         data['_cls'] = self._class_name
+        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+        # only root fields ['test1.a', 'test2'] => ['test1', 'test2']
+        root_fields = set([f.split('.')[0] for f in fields])
 
         for field_name in self:
+            if root_fields and field_name not in root_fields:
+                continue
+
             value = self._data.get(field_name, None)
             field = self._fields.get(field_name)
+
             if field is None and self._dynamic:
                 field = self._dynamic_fields.get(field_name)
 
             if value is not None:
-                value = field.to_mongo(value)
+
+                if isinstance(field, (EmbeddedDocumentField)):
+                    if fields:
+                        key = '%s.' % field_name
+                        embedded_fields = [
+                            i.replace(key, '') for i in fields
+                            if i.startswith(key)]
+
+                    else:
+                        embedded_fields = []
+
+                    value = field.to_mongo(value, use_db_field=use_db_field,
+                                           fields=embedded_fields)
+                else:
+                    value = field.to_mongo(value)
 
             # Handle self generating fields
             if value is None and field._auto_gen:
@@ -260,7 +344,10 @@ class BaseDocument(object):
                 self._data[field_name] = value
 
             if value is not None:
-                data[field.db_field] = value
+                if use_db_field:
+                    data[field.db_field] = value
+                else:
+                    data[field.name] = value
 
         # If "_id" has not been set, then try and set it
         Document = _import_class("Document")
@@ -273,7 +360,7 @@ class BaseDocument(object):
 
         # Only add _cls if allow_inheritance is True
         if (not hasattr(self, '_meta') or
-           not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)):
+                not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)):
             data.pop('_cls')
 
         return data
@@ -295,7 +382,8 @@ class BaseDocument(object):
                    self._data.get(name)) for name in self._fields_ordered]
 
         EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
-        GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField")
+        GenericEmbeddedDocumentField = _import_class(
+            "GenericEmbeddedDocumentField")
 
         for field, value in fields:
             if value is not None:
@@ -317,25 +405,30 @@ class BaseDocument(object):
             pk = "None"
             if hasattr(self, 'pk'):
                 pk = self.pk
-            elif self._instance:
+            elif self._instance and hasattr(self._instance, 'pk'):
                 pk = self._instance.pk
             message = "ValidationError (%s:%s) " % (self._class_name, pk)
             raise ValidationError(message, errors=errors)
 
     def to_json(self, *args, **kwargs):
-        """Converts a document to JSON"""
-        return json_util.dumps(self.to_mongo(),  *args, **kwargs)
+        """Converts a document to JSON.
+        :param use_db_field: Set to True by default but enables the output of the json structure with the field names and not the mongodb store db_names in case of set to False
+        """
+        use_db_field = kwargs.pop('use_db_field', True)
+        return json_util.dumps(self.to_mongo(use_db_field),  *args, **kwargs)
 
     @classmethod
-    def from_json(cls, json_data):
+    def from_json(cls, json_data, created=False):
         """Converts json data to an unsaved document instance"""
-        return cls._from_son(json_util.loads(json_data))
+        return cls._from_son(json_util.loads(json_data), created=created)
 
     def __expand_dynamic_values(self, name, value):
         """expand any dynamic values to their correct types / values"""
         if not isinstance(value, (dict, list, tuple)):
             return value
 
+        EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
+
         is_list = False
         if not hasattr(value, 'items'):
             is_list = True
@@ -358,8 +451,11 @@ class BaseDocument(object):
 
         # Convert lists / values so we can watch for any changes on them
         if (isinstance(value, (list, tuple)) and
-           not isinstance(value, BaseList)):
-            value = BaseList(value, self, name)
+                not isinstance(value, BaseList)):
+            if issubclass(type(self), EmbeddedDocumentListField):
+                value = EmbeddedDocumentList(value, self, name)
+            else:
+                value = BaseList(value, self, name)
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             value = BaseDict(value, self, name)
 
@@ -370,9 +466,18 @@ class BaseDocument(object):
         """
         if not key:
             return
-        key = self._db_field_map.get(key, key)
-        if (hasattr(self, '_changed_fields') and
-           key not in self._changed_fields):
+
+        if not hasattr(self, '_changed_fields'):
+            return
+
+        if '.' in key:
+            key, rest = key.split('.', 1)
+            key = self._db_field_map.get(key, key)
+            key = '%s.%s' % (key, rest)
+        else:
+            key = self._db_field_map.get(key, key)
+
+        if key not in self._changed_fields:
             self._changed_fields.append(key)
 
     def _clear_changed_fields(self):
@@ -392,6 +497,8 @@ class BaseDocument(object):
                 else:
                     data = getattr(data, part, None)
                 if hasattr(data, "_changed_fields"):
+                    if hasattr(data, "_is_document") and data._is_document:
+                        continue
                     data._changed_fields = []
         self._changed_fields = []
 
@@ -405,12 +512,17 @@ class BaseDocument(object):
 
         for index, value in iterator:
             list_key = "%s%s." % (key, index)
+            # don't check anything lower if this key is already marked
+            # as changed.
+            if list_key[:-1] in changed_fields:
+                continue
             if hasattr(value, '_get_changed_fields'):
                 changed = value._get_changed_fields(inspected)
                 changed_fields += ["%s%s" % (list_key, k)
-                                    for k in changed if k]
+                                   for k in changed if k]
             elif isinstance(value, (list, tuple, dict)):
-                self._nestable_types_changed_fields(changed_fields, list_key, value, inspected)
+                self._nestable_types_changed_fields(
+                    changed_fields, list_key, value, inspected)
 
     def _get_changed_fields(self, inspected=None):
         """Returns a list of all fields that have explicitly been changed.
@@ -420,6 +532,7 @@ class BaseDocument(object):
         ReferenceField = _import_class("ReferenceField")
         changed_fields = []
         changed_fields += getattr(self, '_changed_fields', [])
+
         inspected = inspected or set()
         if hasattr(self, 'id') and isinstance(self.id, Hashable):
             if self.id in inspected:
@@ -439,16 +552,17 @@ class BaseDocument(object):
             if isinstance(field, ReferenceField):
                 continue
             elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
-               and db_field_name not in changed_fields):
+                  and db_field_name not in changed_fields):
                  # Find all embedded fields that have been changed
                 changed = data._get_changed_fields(inspected)
                 changed_fields += ["%s%s" % (key, k) for k in changed if k]
             elif (isinstance(data, (list, tuple, dict)) and
                     db_field_name not in changed_fields):
                 if (hasattr(field, 'field') and
-                    isinstance(field.field, ReferenceField)):
+                        isinstance(field.field, ReferenceField)):
                     continue
-                self._nestable_types_changed_fields(changed_fields, key, data, inspected)
+                self._nestable_types_changed_fields(
+                    changed_fields, key, data, inspected)
         return changed_fields
 
     def _delta(self):
@@ -472,7 +586,10 @@ class BaseDocument(object):
                     if isinstance(d, (ObjectId, DBRef)):
                         break
                     elif isinstance(d, list) and p.isdigit():
-                        d = d[int(p)]
+                        try:
+                            d = d[int(p)]
+                        except IndexError:
+                            d = None
                     elif hasattr(d, 'get'):
                         d = d.get(p)
                     new_path.append(p)
@@ -491,7 +608,7 @@ class BaseDocument(object):
             # If we've set a value that ain't the default value dont unset it.
             default = None
             if (self._dynamic and len(parts) and parts[0] in
-               self._dynamic_fields):
+                    self._dynamic_fields):
                 del(set_data[path])
                 unset_data[path] = 1
                 continue
@@ -537,18 +654,16 @@ class BaseDocument(object):
         return cls._meta.get('collection', None)
 
     @classmethod
-    def _from_son(cls, son, _auto_dereference=True):
+    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
         """Create an instance of a Document (subclass) from a PyMongo SON.
         """
+        if not only_fields:
+            only_fields = []
 
         # get the class name from the document, falling back to the given
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
         data = dict(("%s" % key, value) for key, value in son.iteritems())
-        if not UNICODE_KWARGS:
-            # python 2.6.4 and lower cannot handle unicode keys
-            # passed to class constructor example: cls(**data)
-            to_str_keys_recursive(data)
 
         # Return correct subclass for document type
         if class_name != cls._class_name:
@@ -578,19 +693,24 @@ class BaseDocument(object):
                     default = default()
                 if isinstance(default, BaseDocument):
                     changed_fields.append(field_name)
+                elif not only_fields or field_name in only_fields:
+                    changed_fields.append(field_name)
 
         if errors_dict:
             errors = "\n".join(["%s - %s" % (k, v)
-                     for k, v in errors_dict.items()])
+                                for k, v in errors_dict.items()])
             msg = ("Invalid data to create a `%s` instance.\n%s"
                    % (cls._class_name, errors))
             raise InvalidDocumentError(msg)
 
-        obj = cls(__auto_convert=False, **data)
+        if cls.STRICT:
+            data = dict((k, v)
+                        for k, v in data.iteritems() if k in cls._fields)
+        obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
         obj._changed_fields = changed_fields
-        obj._created = False
         if not _auto_dereference:
             obj._fields = fields
+
         return obj
 
     @classmethod
@@ -609,7 +729,7 @@ class BaseDocument(object):
 
             spec_fields = [v['fields']
                            for k, v in enumerate(index_specs)]
-            # Merge unqiue_indexes with existing specs
+            # Merge unique_indexes with existing specs
             for k, v in enumerate(indices):
                 if v['fields'] in spec_fields:
                     index_specs[spec_fields.index(v['fields'])].update(v)
@@ -640,6 +760,9 @@ class BaseDocument(object):
                                           ALLOW_INHERITANCE)
         include_cls = (allow_inheritance and not spec.get('sparse', False) and
                        spec.get('cls',  True))
+
+        # 733: don't include cls if index_cls is False unless there is an explicit cls with the index
+        include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
         if "cls" in spec:
             spec.pop('cls')
         for key in spec['fields']:
@@ -647,15 +770,18 @@ class BaseDocument(object):
             if isinstance(key, (list, tuple)):
                 continue
 
-            # ASCENDING from +,
+            # ASCENDING from +
             # DESCENDING from -
             # GEO2D from *
+            # TEXT from $
             direction = pymongo.ASCENDING
             if key.startswith("-"):
                 direction = pymongo.DESCENDING
             elif key.startswith("*"):
                 direction = pymongo.GEO2D
-            if key.startswith(("+", "-", "*")):
+            elif key.startswith("$"):
+                direction = pymongo.TEXT
+            if key.startswith(("+", "-", "*", "$")):
                 key = key[1:]
 
             # Use real field name, do it manually because we need field
@@ -666,8 +792,14 @@ class BaseDocument(object):
                 fields = []
             else:
                 fields = cls._lookup_field(parts)
-                parts = [field if field == '_id' else field.db_field
-                         for field in fields]
+                parts = []
+                for field in fields:
+                    try:
+                        if field != "_id":
+                            field = field.db_field
+                    except AttributeError:
+                        pass
+                    parts.append(field)
                 key = '.'.join(parts)
             index_list.append((key, direction))
 
@@ -691,10 +823,9 @@ class BaseDocument(object):
         """
         unique_indexes = []
         for field_name, field in cls._fields.items():
-            sparse = False
+            sparse = field.sparse
             # Generate a list of indexes needed by uniqueness constraints
             if field.unique:
-                field.required = True
                 unique_fields = [field.db_field]
 
                 # Add any unique_with fields to the back of the index spec
@@ -722,9 +853,12 @@ class BaseDocument(object):
                 index = {'fields': fields, 'unique': True, 'sparse': sparse}
                 unique_indexes.append(index)
 
+            if field.__class__.__name__ == "ListField":
+                field = field.field
+
             # Grab any embedded document field unique indexes
             if (field.__class__.__name__ == "EmbeddedDocumentField" and
-               field.document_type != cls):
+                    field.document_type != cls):
                 field_namespace = "%s." % field_name
                 doc_cls = field.document_type
                 unique_indexes += doc_cls._unique_with_indexes(field_namespace)
@@ -740,7 +874,8 @@ class BaseDocument(object):
         geo_field_type_names = ["EmbeddedDocumentField", "GeoPointField",
                                 "PointField", "LineStringField", "PolygonField"]
 
-        geo_field_types = tuple([_import_class(field) for field in geo_field_type_names])
+        geo_field_types = tuple([_import_class(field)
+                                 for field in geo_field_type_names])
 
         for field in cls._fields.values():
             if not isinstance(field, geo_field_types):
@@ -750,13 +885,14 @@ class BaseDocument(object):
                 if field_cls in inspected:
                     continue
                 if hasattr(field_cls, '_geo_indices'):
-                    geo_indices += field_cls._geo_indices(inspected, parent_field=field.db_field)
+                    geo_indices += field_cls._geo_indices(
+                        inspected, parent_field=field.db_field)
             elif field._geo_index:
                 field_name = field.db_field
                 if parent_field:
                     field_name = "%s.%s" % (parent_field, field_name)
                 geo_indices.append({'fields':
-                                   [(field_name, field._geo_index)]})
+                                    [(field_name, field._geo_index)]})
         return geo_indices
 
     @classmethod
@@ -789,6 +925,19 @@ class BaseDocument(object):
                 elif cls._dynamic:
                     DynamicField = _import_class('DynamicField')
                     field = DynamicField(db_field=field_name)
+                elif cls._meta.get("allow_inheritance", False) or cls._meta.get("abstract", False):
+                    # 744: in case the field is defined in a subclass
+                    field = None
+                    for subcls in cls.__subclasses__():
+                        try:
+                            field = subcls._lookup_field([field_name])[0]
+                        except LookUpError:
+                            continue
+
+                        if field is not None:
+                            break
+                    else:
+                        raise LookUpError('Cannot resolve field "%s"' % field_name)
                 else:
                     raise LookUpError('Cannot resolve field "%s"'
                                       % field_name)
@@ -804,8 +953,17 @@ class BaseDocument(object):
                    # Look up subfield on the previous field
                     new_field = field.lookup_member(field_name)
                 if not new_field and isinstance(field, ComplexBaseField):
-                    fields.append(field_name)
-                    continue
+                    if hasattr(field.field, 'document_type') and cls._dynamic \
+                            and field.field.document_type._dynamic:
+                        DynamicField = _import_class('DynamicField')
+                        new_field = DynamicField(db_field=field_name)
+                    else:
+                        fields.append(field_name)
+                        continue
+                elif not new_field and hasattr(field, 'document_type') and cls._dynamic \
+                        and field.document_type._dynamic:
+                    DynamicField = _import_class('DynamicField')
+                    new_field = DynamicField(db_field=field_name)
                 elif not new_field:
                     raise LookUpError('Cannot resolve field "%s"'
                                       % field_name)
@@ -825,7 +983,11 @@ class BaseDocument(object):
         """Dynamically set the display value for a field with choices"""
         for attr_name, field in self._fields.items():
             if field.choices:
-                setattr(self,
+                if self._dynamic:
+                    obj = self
+                else:
+                    obj = type(self)
+                setattr(obj,
                         'get_%s_display' % attr_name,
                         partial(self.__get_field_display, field=field))
 

mongoengine/base/fields.py

@@ -9,12 +9,16 @@ from mongoengine.common import _import_class
 from mongoengine.errors import ValidationError
 
 from mongoengine.base.common import ALLOW_INHERITANCE
-from mongoengine.base.datastructures import BaseDict, BaseList
+from mongoengine.base.datastructures import (
+    BaseDict, BaseList, EmbeddedDocumentList
+)
 
-__all__ = ("BaseField", "ComplexBaseField", "ObjectIdField", "GeoJsonBaseField")
+__all__ = ("BaseField", "ComplexBaseField",
+           "ObjectIdField", "GeoJsonBaseField")
 
 
 class BaseField(object):
+
     """A base class for fields in a MongoDB document. Instances of this class
     may be added to subclasses of `Document` to define a document's schema.
 
@@ -35,7 +39,7 @@ class BaseField(object):
     def __init__(self, db_field=None, name=None, required=False, default=None,
                  unique=False, unique_with=None, primary_key=False,
                  validation=None, choices=None, verbose_name=None,
-                 help_text=None):
+                 help_text=None, null=False, sparse=False):
         """
         :param db_field: The database field to store this field in
             (defaults to the name of the field)
@@ -43,7 +47,7 @@ class BaseField(object):
         :param required: If the field is required. Whether it has to have a
             value or not. Defaults to False.
         :param default: (optional) The default value for this field if no value
-            has been set (or if the value has been unset).  It Can be a
+            has been set (or if the value has been unset).  It can be a
             callable.
         :param unique: Is the field value unique or not.  Defaults to False.
         :param unique_with: (optional) The other field this field should be
@@ -58,8 +62,13 @@ class BaseField(object):
             model forms from the document model.
         :param help_text: (optional) The help text for this field and is often
             used when generating model forms from the document model.
+        :param null: (optional) Is the field value can be null. If no and there is a default value
+            then the default value is set
+        :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False`
+            means that uniqueness won't be enforced for `None` values
         """
         self.db_field = (db_field or name) if not primary_key else '_id'
+
         if name:
             msg = "Fields' 'name' attribute deprecated in favour of 'db_field'"
             warnings.warn(msg, DeprecationWarning)
@@ -72,6 +81,8 @@ class BaseField(object):
         self.choices = choices
         self.verbose_name = verbose_name
         self.help_text = help_text
+        self.null = null
+        self.sparse = sparse
 
         # Adjust the appropriate creation counter, and save our local copy.
         if self.db_field == '_id':
@@ -97,15 +108,18 @@ class BaseField(object):
 
         # If setting to None and theres a default
         # Then set the value to the default value
-        if value is None and self.default is not None:
-            value = self.default
-            if callable(value):
-                value = value()
+        if value is None:
+            if self.null:
+                value = None
+            elif self.default is not None:
+                value = self.default
+                if callable(value):
+                    value = value()
 
         if instance._initialised:
             try:
                 if (self.name not in instance._data or
-                   instance._data[self.name] != value):
+                        instance._data[self.name] != value):
                     instance._mark_as_changed(self.name)
             except:
                 # Values cant be compared eg: naive and tz datetimes
@@ -113,7 +127,7 @@ class BaseField(object):
                 instance._mark_as_changed(self.name)
 
         EmbeddedDocument = _import_class('EmbeddedDocument')
-        if isinstance(value, EmbeddedDocument) and value._instance is None:
+        if isinstance(value, EmbeddedDocument):
             value._instance = weakref.proxy(instance)
         instance._data[self.name] = value
 
@@ -146,21 +160,23 @@ class BaseField(object):
     def _validate(self, value, **kwargs):
         Document = _import_class('Document')
         EmbeddedDocument = _import_class('EmbeddedDocument')
-        # check choices
+
+        # Check the Choices Constraint
         if self.choices:
-            is_cls = isinstance(value, (Document, EmbeddedDocument))
-            value_to_check = value.__class__ if is_cls else value
-            err_msg = 'an instance' if is_cls else 'one'
+
+            choice_list = self.choices
             if isinstance(self.choices[0], (list, tuple)):
-                option_keys = [k for k, v in self.choices]
-                if value_to_check not in option_keys:
-                    msg = ('Value must be %s of %s' %
-                           (err_msg, unicode(option_keys)))
-                    self.error(msg)
-            elif value_to_check not in self.choices:
-                msg = ('Value must be %s of %s' %
-                       (err_msg, unicode(self.choices)))
-                self.error(msg)
+                choice_list = [k for k, v in self.choices]
+
+            # Choices which are other types of Documents
+            if isinstance(value, (Document, EmbeddedDocument)):
+                if not any(isinstance(value, c) for c in choice_list):
+                    self.error(
+                        'Value must be instance of %s' % unicode(choice_list)
+                    )
+            # Choices which are types other than Documents
+            elif value not in choice_list:
+                self.error('Value must be one of %s' % unicode(choice_list))
 
         # check validation argument
         if self.validation is not None:
@@ -175,6 +191,7 @@ class BaseField(object):
 
 
 class ComplexBaseField(BaseField):
+
     """Handles complex fields, such as lists / dictionaries.
 
     Allows for nesting of embedded documents inside complex types.
@@ -195,9 +212,10 @@ class ComplexBaseField(BaseField):
 
         ReferenceField = _import_class('ReferenceField')
         GenericReferenceField = _import_class('GenericReferenceField')
+        EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
         dereference = (self._auto_dereference and
                        (self.field is None or isinstance(self.field,
-                        (GenericReferenceField, ReferenceField))))
+                                                         (GenericReferenceField, ReferenceField))))
 
         _dereference = _import_class("DeReference")()
 
@@ -211,17 +229,20 @@ class ComplexBaseField(BaseField):
         value = super(ComplexBaseField, self).__get__(instance, owner)
 
         # Convert lists / values so we can watch for any changes on them
-        if (isinstance(value, (list, tuple)) and
-           not isinstance(value, BaseList)):
-            value = BaseList(value, instance, self.name)
+        if isinstance(value, (list, tuple)):
+            if (issubclass(type(self), EmbeddedDocumentListField) and
+                    not isinstance(value, EmbeddedDocumentList)):
+                value = EmbeddedDocumentList(value, instance, self.name)
+            elif not isinstance(value, BaseList):
+                value = BaseList(value, instance, self.name)
             instance._data[self.name] = value
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             value = BaseDict(value, instance, self.name)
             instance._data[self.name] = value
 
         if (self._auto_dereference and instance._initialised and
-           isinstance(value, (BaseList, BaseDict))
-           and not value._dereferenced):
+                isinstance(value, (BaseList, BaseDict))
+                and not value._dereferenced):
             value = _dereference(
                 value, max_depth=1, instance=instance, name=self.name
             )
@@ -384,6 +405,7 @@ class ComplexBaseField(BaseField):
 
 
 class ObjectIdField(BaseField):
+
     """A field wrapper around MongoDB's ObjectIds.
     """
 
@@ -412,7 +434,9 @@ class ObjectIdField(BaseField):
 
 
 class GeoJsonBaseField(BaseField):
+
     """A geo json field storing a geojson style object.
+
     .. versionadded:: 0.8
     """
 
@@ -421,8 +445,8 @@ class GeoJsonBaseField(BaseField):
 
     def __init__(self, auto_index=True, *args, **kwargs):
         """
-        :param auto_index: Automatically create a "2dsphere" index. Defaults
-            to `True`.
+        :param bool auto_index: Automatically create a "2dsphere" index.\
+            Defaults to `True`.
         """
         self._name = "%sField" % self._type
         if not auto_index:
@@ -435,7 +459,8 @@ class GeoJsonBaseField(BaseField):
         if isinstance(value, dict):
             if set(value.keys()) == set(['type', 'coordinates']):
                 if value['type'] != self._type:
-                    self.error('%s type must be "%s"' % (self._name, self._type))
+                    self.error('%s type must be "%s"' %
+                               (self._name, self._type))
                 return self.validate(value['coordinates'])
             else:
                 self.error('%s can only accept a valid GeoJson dictionary'
@@ -450,7 +475,7 @@ class GeoJsonBaseField(BaseField):
         if error:
             self.error(error)
 
-    def _validate_polygon(self, value):
+    def _validate_polygon(self, value, top_level=True):
         if not isinstance(value, (list, tuple)):
             return 'Polygons must contain list of linestrings'
 
@@ -468,7 +493,10 @@ class GeoJsonBaseField(BaseField):
             if error and error not in errors:
                 errors.append(error)
         if errors:
-            return "Invalid Polygon:\n%s" % ", ".join(errors)
+            if top_level:
+                return "Invalid Polygon:\n%s" % ", ".join(errors)
+            else:
+                return "%s" % ", ".join(errors)
 
     def _validate_linestring(self, value, top_level=True):
         """Validates a linestring"""
@@ -502,6 +530,66 @@ class GeoJsonBaseField(BaseField):
               not isinstance(value[1], (float, int))):
             return "Both values (%s) in point must be float or int" % repr(value)
 
+    def _validate_multipoint(self, value):
+        if not isinstance(value, (list, tuple)):
+            return 'MultiPoint must be a list of Point'
+
+        # Quick and dirty validator
+        try:
+            value[0][0]
+        except:
+            return "Invalid MultiPoint must contain at least one valid point"
+
+        errors = []
+        for point in value:
+            error = self._validate_point(point)
+            if error and error not in errors:
+                errors.append(error)
+
+        if errors:
+            return "%s" % ", ".join(errors)
+
+    def _validate_multilinestring(self, value, top_level=True):
+        if not isinstance(value, (list, tuple)):
+            return 'MultiLineString must be a list of LineString'
+
+        # Quick and dirty validator
+        try:
+            value[0][0][0]
+        except:
+            return "Invalid MultiLineString must contain at least one valid linestring"
+
+        errors = []
+        for linestring in value:
+            error = self._validate_linestring(linestring, False)
+            if error and error not in errors:
+                errors.append(error)
+
+        if errors:
+            if top_level:
+                return "Invalid MultiLineString:\n%s" % ", ".join(errors)
+            else:
+                return "%s" % ", ".join(errors)
+
+    def _validate_multipolygon(self, value):
+        if not isinstance(value, (list, tuple)):
+            return 'MultiPolygon must be a list of Polygon'
+
+        # Quick and dirty validator
+        try:
+            value[0][0][0][0]
+        except:
+            return "Invalid MultiPolygon must contain at least one valid Polygon"
+
+        errors = []
+        for polygon in value:
+            error = self._validate_polygon(polygon, False)
+            if error and error not in errors:
+                errors.append(error)
+
+        if errors:
+            return "Invalid MultiPolygon:\n%s" % ", ".join(errors)
+
     def to_mongo(self, value):
         if isinstance(value, dict):
             return value

mongoengine/base/metaclasses.py

@@ -16,6 +16,7 @@ __all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass')
 
 
 class DocumentMetaclass(type):
+
     """Metaclass for all documents.
     """
 
@@ -29,6 +30,7 @@ class DocumentMetaclass(type):
             return super_new(cls, name, bases, attrs)
 
         attrs['_is_document'] = attrs.get('_is_document', False)
+        attrs['_cached_reference_fields'] = []
 
         # EmbeddedDocuments could have meta data for inheritance
         if 'meta' in attrs:
@@ -44,6 +46,11 @@ class DocumentMetaclass(type):
                 elif hasattr(base, '_meta'):
                     meta.merge(base._meta)
             attrs['_meta'] = meta
+            attrs['_meta']['abstract'] = False  # 789: EmbeddedDocument shouldn't inherit abstract
+
+        if attrs['_meta'].get('allow_inheritance', ALLOW_INHERITANCE):
+            StringField = _import_class('StringField')
+            attrs['_cls'] = StringField()
 
         # Handle document Fields
 
@@ -90,7 +97,7 @@ class DocumentMetaclass(type):
         # Set _fields and db_field maps
         attrs['_fields'] = doc_fields
         attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k))
-                                      for k, v in doc_fields.iteritems()])
+                                       for k, v in doc_fields.iteritems()])
         attrs['_reverse_db_field_map'] = dict(
             (v, k) for k, v in attrs['_db_field_map'].iteritems())
 
@@ -105,7 +112,7 @@ class DocumentMetaclass(type):
         class_name = [name]
         for base in flattened_bases:
             if (not getattr(base, '_is_base_cls', True) and
-               not getattr(base, '_meta', {}).get('abstract', True)):
+                    not getattr(base, '_meta', {}).get('abstract', True)):
                 # Collate heirarchy for _cls and _subclasses
                 class_name.append(base.__name__)
 
@@ -115,7 +122,7 @@ class DocumentMetaclass(type):
                 allow_inheritance = base._meta.get('allow_inheritance',
                                                    ALLOW_INHERITANCE)
                 if (allow_inheritance is not True and
-                   not base._meta.get('abstract')):
+                        not base._meta.get('abstract')):
                     raise ValueError('Document %s may not be subclassed' %
                                      base.__name__)
 
@@ -141,7 +148,8 @@ class DocumentMetaclass(type):
                 base._subclasses += (_cls,)
             base._types = base._subclasses   # TODO depreciate _types
 
-        Document, EmbeddedDocument, DictField = cls._import_classes()
+        (Document, EmbeddedDocument, DictField,
+         CachedReferenceField) = cls._import_classes()
 
         if issubclass(new_class, Document):
             new_class._collection = None
@@ -170,6 +178,20 @@ class DocumentMetaclass(type):
             f = field
             f.owner_document = new_class
             delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING)
+            if isinstance(f, CachedReferenceField):
+
+                if issubclass(new_class, EmbeddedDocument):
+                    raise InvalidDocumentError(
+                        "CachedReferenceFields is not allowed in EmbeddedDocuments")
+                if not f.document_type:
+                    raise InvalidDocumentError(
+                        "Document is not avaiable to sync")
+
+                if f.auto_sync:
+                    f.start_listener()
+
+                f.document_type._cached_reference_fields.append(f)
+
             if isinstance(f, ComplexBaseField) and hasattr(f, 'field'):
                 delete_rule = getattr(f.field,
                                       'reverse_delete_rule',
@@ -191,7 +213,7 @@ class DocumentMetaclass(type):
                                                      field.name, delete_rule)
 
             if (field.name and hasattr(Document, field.name) and
-               EmbeddedDocument not in new_class.mro()):
+                    EmbeddedDocument not in new_class.mro()):
                 msg = ("%s is a document method and not a valid "
                        "field name" % field.name)
                 raise InvalidDocumentError(msg)
@@ -224,10 +246,12 @@ class DocumentMetaclass(type):
         Document = _import_class('Document')
         EmbeddedDocument = _import_class('EmbeddedDocument')
         DictField = _import_class('DictField')
-        return (Document, EmbeddedDocument, DictField)
+        CachedReferenceField = _import_class('CachedReferenceField')
+        return (Document, EmbeddedDocument, DictField, CachedReferenceField)
 
 
 class TopLevelDocumentMetaclass(DocumentMetaclass):
+
     """Metaclass for top-level documents (i.e. documents that have their own
     collection in the database.
     """
@@ -275,21 +299,21 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
         # Find the parent document class
         parent_doc_cls = [b for b in flattened_bases
-                        if b.__class__ == TopLevelDocumentMetaclass]
+                          if b.__class__ == TopLevelDocumentMetaclass]
         parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0]
 
         # Prevent classes setting collection different to their parents
         # If parent wasn't an abstract class
         if (parent_doc_cls and 'collection' in attrs.get('_meta', {})
-            and not parent_doc_cls._meta.get('abstract', True)):
-                msg = "Trying to set a collection on a subclass (%s)" % name
-                warnings.warn(msg, SyntaxWarning)
-                del(attrs['_meta']['collection'])
+                and not parent_doc_cls._meta.get('abstract', True)):
+            msg = "Trying to set a collection on a subclass (%s)" % name
+            warnings.warn(msg, SyntaxWarning)
+            del(attrs['_meta']['collection'])
 
         # Ensure abstract documents have abstract bases
         if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'):
             if (parent_doc_cls and
-                not parent_doc_cls._meta.get('abstract', False)):
+                    not parent_doc_cls._meta.get('abstract', False)):
                 msg = "Abstract document cannot have non-abstract base"
                 raise ValueError(msg)
             return super_new(cls, name, bases, attrs)
@@ -306,7 +330,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
             # Set collection in the meta if its callable
             if (getattr(base, '_is_document', False) and
-                not base._meta.get('abstract')):
+                    not base._meta.get('abstract')):
                 collection = meta.get('collection', None)
                 if callable(collection):
                     meta['collection'] = collection(base)
@@ -318,7 +342,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         simple_class = all([b._meta.get('abstract')
                             for b in flattened_bases if hasattr(b, '_meta')])
         if (not simple_class and meta['allow_inheritance'] is False and
-           not meta['abstract']):
+                not meta['abstract']):
             raise ValueError('Only direct subclasses of Document may set '
                              '"allow_inheritance" to False')
 
@@ -359,7 +383,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
                     new_class.id = field
 
         # Set primary key if not defined by the document
-        new_class._auto_id_field = False
+        new_class._auto_id_field = getattr(parent_doc_cls,
+                                           '_auto_id_field', False)
         if not new_class._meta.get('id_field'):
             new_class._auto_id_field = True
             new_class._meta['id_field'] = 'id'
@@ -377,7 +402,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         for exc in exceptions_to_merge:
             name = exc.__name__
             parents = tuple(getattr(base, name) for base in flattened_bases
-                         if hasattr(base, name)) or (exc,)
+                            if hasattr(base, name)) or (exc,)
             # Create new exception and set to new_class
             exception = type(name, parents, {'__module__': module})
             setattr(new_class, name, exception)
@@ -386,6 +411,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
 
 
 class MetaDict(dict):
+
     """Custom dictionary for meta classes.
     Handles the merging of set indexes
     """
@@ -400,5 +426,6 @@ class MetaDict(dict):
 
 
 class BasesTuple(tuple):
+
     """Special class to handle introspection of bases tuple in __new__"""
     pass

mongoengine/common.py

@@ -1,4 +1,5 @@
 _class_registry_cache = {}
+_field_list_cache = []
 
 
 def _import_class(cls_name):
@@ -20,12 +21,16 @@ def _import_class(cls_name):
 
     doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument',
                    'MapReduceDocument')
-    field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField',
-                     'FileField', 'GenericReferenceField',
-                     'GenericEmbeddedDocumentField', 'GeoPointField',
-                     'PointField', 'LineStringField', 'ListField',
-                     'PolygonField', 'ReferenceField', 'StringField',
-                     'ComplexBaseField', 'GeoJsonBaseField')
+
+    # Field Classes
+    if not _field_list_cache:
+        from mongoengine.fields import __all__ as fields
+        _field_list_cache.extend(fields)
+        from mongoengine.base.fields import __all__ as fields
+        _field_list_cache.extend(fields)
+
+    field_classes = _field_list_cache
+
     queryset_classes = ('OperationError',)
     deref_classes = ('DeReference',)
 

mongoengine/connection.py

@@ -18,9 +18,10 @@ _connections = {}
 _dbs = {}
 
 
-def register_connection(alias, name, host=None, port=None,
-                        is_slave=False, read_preference=False, slaves=None,
-                        username=None, password=None, **kwargs):
+def register_connection(alias, name=None, host=None, port=None,
+                        read_preference=False,
+                        username=None, password=None, authentication_source=None,
+                        **kwargs):
     """Add a connection.
 
     :param alias: the name that will be used to refer to this connection
@@ -28,28 +29,24 @@ def register_connection(alias, name, host=None, port=None,
     :param name: the name of the specific database to use
     :param host: the host name of the :program:`mongod` instance to connect to
     :param port: the port that the :program:`mongod` instance is running on
-    :param is_slave: whether the connection can act as a slave
-      ** Depreciated pymongo 2.0.1+
     :param read_preference: The read preference for the collection
        ** Added pymongo 2.1
-    :param slaves: a list of aliases of slave connections; each of these must
-        be a registered connection that has :attr:`is_slave` set to ``True``
     :param username: username to authenticate with
     :param password: password to authenticate with
+    :param authentication_source: database to authenticate against
     :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver
 
     """
     global _connection_settings
 
     conn_settings = {
-        'name': name,
+        'name': name or 'test',
         'host': host or 'localhost',
         'port': port or 27017,
-        'is_slave': is_slave,
-        'slaves': slaves or [],
+        'read_preference': read_preference,
         'username': username,
         'password': password,
-        'read_preference': read_preference
+        'authentication_source': authentication_source
     }
 
     # Handle uri style connections
@@ -64,6 +61,10 @@ def register_connection(alias, name, host=None, port=None,
         if "replicaSet" in conn_settings['host']:
             conn_settings['replicaSet'] = True
 
+    # Deprecated parameters that should not be passed on
+    kwargs.pop('slaves', None)
+    kwargs.pop('is_slave', None)
+
     conn_settings.update(kwargs)
     _connection_settings[alias] = conn_settings
 
@@ -93,20 +94,10 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
             raise ConnectionError(msg)
         conn_settings = _connection_settings[alias].copy()
 
-        if hasattr(pymongo, 'version_tuple'):  # Support for 2.1+
-            conn_settings.pop('name', None)
-            conn_settings.pop('slaves', None)
-            conn_settings.pop('is_slave', None)
-            conn_settings.pop('username', None)
-            conn_settings.pop('password', None)
-        else:
-            # Get all the slave connections
-            if 'slaves' in conn_settings:
-                slaves = []
-                for slave_alias in conn_settings['slaves']:
-                    slaves.append(get_connection(slave_alias))
-                conn_settings['slaves'] = slaves
-                conn_settings.pop('read_preference', None)
+        conn_settings.pop('name', None)
+        conn_settings.pop('username', None)
+        conn_settings.pop('password', None)
+        conn_settings.pop('authentication_source', None)
 
         connection_class = MongoClient
         if 'replicaSet' in conn_settings:
@@ -119,7 +110,18 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
             connection_class = MongoReplicaSetClient
 
         try:
-            _connections[alias] = connection_class(**conn_settings)
+            connection = None
+            # check for shared connections
+            connection_settings_iterator = ((db_alias, settings.copy()) for db_alias, settings in _connection_settings.iteritems())
+            for db_alias, connection_settings in connection_settings_iterator:
+                connection_settings.pop('name', None)
+                connection_settings.pop('username', None)
+                connection_settings.pop('password', None)
+                if conn_settings == connection_settings and _connections.get(db_alias, None):
+                    connection = _connections[db_alias]
+                    break
+
+            _connections[alias] = connection if connection else connection_class(**conn_settings)
         except Exception, e:
             raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e))
     return _connections[alias]
@@ -137,12 +139,13 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         # Authenticate if necessary
         if conn_settings['username'] and conn_settings['password']:
             db.authenticate(conn_settings['username'],
-                            conn_settings['password'])
+                            conn_settings['password'],
+                            source=conn_settings['authentication_source'])
         _dbs[alias] = db
     return _dbs[alias]
 
 
-def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs):
+def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs):
     """Connect to the database specified by the 'db' argument.
 
     Connection settings may be provided here as well if the database is not

mongoengine/context_managers.py

@@ -1,6 +1,5 @@
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from mongoengine.queryset import QuerySet
 
 
 __all__ = ("switch_db", "switch_collection", "no_dereference",
@@ -162,12 +161,6 @@ class no_sub_classes(object):
         return self.cls
 
 
-class QuerySetNoDeRef(QuerySet):
-    """Special no_dereference QuerySet"""
-    def __dereference(items, max_depth=1, instance=None, name=None):
-            return items
-
-
 class query_counter(object):
     """ Query_counter context manager to get the number of queries. """
 

mongoengine/dereference.py

@@ -1,6 +1,9 @@
 from bson import DBRef, SON
 
-from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document)
+from base import (
+    BaseDict, BaseList, EmbeddedDocumentList,
+    TopLevelDocumentMetaclass, get_document
+)
 from fields import (ReferenceField, ListField, DictField, MapField)
 from connection import get_db
 from queryset import QuerySet
@@ -12,7 +15,7 @@ class DeReference(object):
     def __call__(self, items, max_depth=1, instance=None, name=None):
         """
         Cheaply dereferences the items to a set depth.
-        Also handles the convertion of complex data types.
+        Also handles the conversion of complex data types.
 
         :param items: The iterable (dict, list, queryset) to be dereferenced.
         :param max_depth: The maximum depth to recurse to
@@ -36,7 +39,7 @@ class DeReference(object):
         if instance and isinstance(instance, (Document, EmbeddedDocument,
                                               TopLevelDocumentMetaclass)):
             doc_type = instance._fields.get(name)
-            if hasattr(doc_type, 'field'):
+            while hasattr(doc_type, 'field'):
                 doc_type = doc_type.field
 
             if isinstance(doc_type, ReferenceField):
@@ -51,9 +54,19 @@ class DeReference(object):
                     return items
                 elif not field.dbref:
                     if not hasattr(items, 'items'):
-                        items = [field.to_python(v)
-                             if not isinstance(v, (DBRef, Document)) else v
-                             for v in items]
+
+                        def _get_items(items):
+                            new_items = []
+                            for v in items:
+                                if isinstance(v, list):
+                                    new_items.append(_get_items(v))
+                                elif not isinstance(v, (DBRef, Document)):
+                                    new_items.append(field.to_python(v))
+                                else:
+                                    new_items.append(v)
+                            return new_items
+
+                        items = _get_items(items)
                     else:
                         items = dict([
                             (k, field.to_python(v))
@@ -85,7 +98,7 @@ class DeReference(object):
         # Recursively find dbreferences
         depth += 1
         for k, item in iterator:
-            if isinstance(item, Document):
+            if isinstance(item, (Document, EmbeddedDocument)):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -114,11 +127,11 @@ class DeReference(object):
         """Fetch all references and convert to their document objects
         """
         object_map = {}
-        for col, dbrefs in self.reference_map.iteritems():
+        for collection, dbrefs in self.reference_map.iteritems():
             keys = object_map.keys()
             refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys]))
-            if hasattr(col, 'objects'):  # We have a document class for the refs
-                references = col.objects.in_bulk(refs)
+            if hasattr(collection, 'objects'):  # We have a document class for the refs
+                references = collection.objects.in_bulk(refs)
                 for key, doc in references.iteritems():
                     object_map[key] = doc
             else:  # Generic reference: use the refs data to convert to document
@@ -126,19 +139,19 @@ class DeReference(object):
                     continue
 
                 if doc_type:
-                    references = doc_type._get_db()[col].find({'_id': {'$in': refs}})
+                    references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
                         doc = doc_type._from_son(ref)
                         object_map[doc.id] = doc
                 else:
-                    references = get_db()[col].find({'_id': {'$in': refs}})
+                    references = get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
                         if '_cls' in ref:
                             doc = get_document(ref["_cls"])._from_son(ref)
                         elif doc_type is None:
                             doc = get_document(
                                 ''.join(x.capitalize()
-                                    for x in col.split('_')))._from_son(ref)
+                                    for x in collection.split('_')))._from_son(ref)
                         else:
                             doc = doc_type._from_son(ref)
                         object_map[doc.id] = doc
@@ -170,11 +183,18 @@ class DeReference(object):
                 return self.object_map.get(items['_ref'].id, items)
             elif '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
+                _cls = doc._data.pop('_cls', None)
+                del items['_cls']
                 doc._data = self._attach_objects(doc._data, depth, doc, None)
+                if _cls is not None:
+                    doc._data['_cls'] = _cls
                 return doc
 
         if not hasattr(items, 'items'):
             is_list = True
+            list_type = BaseList
+            if isinstance(items, EmbeddedDocumentList):
+                list_type = EmbeddedDocumentList
             as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
@@ -192,7 +212,7 @@ class DeReference(object):
 
             if k in self.object_map and not is_list:
                 data[k] = self.object_map[k]
-            elif isinstance(v, Document):
+            elif isinstance(v, (Document, EmbeddedDocument)):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -204,13 +224,14 @@ class DeReference(object):
                     elif isinstance(v, (list, tuple)) and depth <= self.max_depth:
                         data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name)
             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
-                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name)
+                item_name = '%s.%s' % (name, k) if name else name
+                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
             elif hasattr(v, 'id'):
                 data[k] = self.object_map.get(v.id, v)
 
         if instance and name:
             if is_list:
-                return tuple(data) if as_tuple else BaseList(data, instance, name)
+                return tuple(data) if as_tuple else list_type(data, instance, name)
             return BaseDict(data, instance, name)
         depth += 1
         return data

mongoengine/django/mongo_auth/models.py

@@ -3,7 +3,11 @@ from django.contrib.auth.hashers import make_password
 from django.contrib.auth.models import UserManager
 from django.core.exceptions import ImproperlyConfigured
 from django.db import models
-from django.utils.importlib import import_module
+try:
+    from django.utils.module_loading import import_module
+except ImportError:
+    """Handle older versions of Django"""
+    from django.utils.importlib import import_module
 from django.utils.translation import ugettext_lazy as _
 
 

mongoengine/django/tests.py

@@ -1,39 +1,31 @@
 #coding: utf-8
-from nose.plugins.skip import SkipTest
 
-from mongoengine.python_support import PY3
+from unittest import TestCase
+
 from mongoengine import connect
-
-try:
-    from django.test import TestCase
-    from django.conf import settings
-except Exception as err:
-    if PY3:
-        from unittest import TestCase
-        # Dummy value so no error
-        class settings:
-            MONGO_DATABASE_NAME = 'dummy'
-    else:
-        raise err
+from mongoengine.connection import get_db
 
 
 class MongoTestCase(TestCase):
-
-    def setUp(self):
-        if PY3:
-            raise SkipTest('django does not have Python 3 support')
-
     """
     TestCase class that clear the collection between the tests
     """
-    db_name = 'test_%s' % settings.MONGO_DATABASE_NAME
+
+    @property
+    def db_name(self):
+        from django.conf import settings
+        return 'test_%s' % getattr(settings, 'MONGO_DATABASE_NAME', 'dummy')
+
     def __init__(self, methodName='runtest'):
-        self.db = connect(self.db_name).get_db()
+        connect(self.db_name)
+        self.db = get_db()
         super(MongoTestCase, self).__init__(methodName)
 
-    def _post_teardown(self):
-        super(MongoTestCase, self)._post_teardown()
+    def dropCollections(self):
         for collection in self.db.collection_names():
-            if collection == 'system.indexes':
+            if collection.startswith('system.'):
                 continue
             self.db.drop_collection(collection)
+
+    def tearDown(self):
+        self.dropCollections()

mongoengine/document.py

@@ -9,11 +9,19 @@ from bson import ObjectId
 from bson.dbref import DBRef
 from mongoengine import signals
 from mongoengine.common import _import_class
-from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass,
-                              BaseDocument, BaseDict, BaseList,
-                              ALLOW_INHERITANCE, get_document)
-from mongoengine.errors import ValidationError
-from mongoengine.queryset import OperationError, NotUniqueError, QuerySet
+from mongoengine.base import (
+    DocumentMetaclass,
+    TopLevelDocumentMetaclass,
+    BaseDocument,
+    BaseDict,
+    BaseList,
+    EmbeddedDocumentList,
+    ALLOW_INHERITANCE,
+    get_document
+)
+from mongoengine.errors import ValidationError, InvalidQueryError, InvalidDocumentError
+from mongoengine.queryset import (OperationError, NotUniqueError,
+                                  QuerySet, transform)
 from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
 from mongoengine.context_managers import switch_db, switch_collection
 
@@ -40,6 +48,7 @@ class InvalidCollectionError(Exception):
 
 
 class EmbeddedDocument(BaseDocument):
+
     """A :class:`~mongoengine.Document` that isn't stored in its own
     collection.  :class:`~mongoengine.EmbeddedDocument`\ s should be used as
     fields on :class:`~mongoengine.Document`\ s through the
@@ -54,27 +63,35 @@ class EmbeddedDocument(BaseDocument):
     dictionary.
     """
 
+    __slots__ = ('_instance')
+
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
-    my_metaclass  = DocumentMetaclass
+    my_metaclass = DocumentMetaclass
     __metaclass__ = DocumentMetaclass
 
-    _instance = None
-
     def __init__(self, *args, **kwargs):
         super(EmbeddedDocument, self).__init__(*args, **kwargs)
+        self._instance = None
         self._changed_fields = []
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
-            return self.to_mongo() == other.to_mongo()
+            return self._data == other._data
         return False
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def save(self, *args, **kwargs):
+        self._instance.save(*args, **kwargs)
+
+    def reload(self, *args, **kwargs):
+        self._instance.reload(*args, **kwargs)
+
 
 class Document(BaseDocument):
+
     """The base class used for defining the structure and properties of
     collections of documents stored in MongoDB. Inherit from this class, and
     add fields as class attributes to define a document's structure.
@@ -109,7 +126,7 @@ class Document(BaseDocument):
     a **+** or **-** sign.
 
     Automatic index creation can be disabled by specifying
-    attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to
+    :attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to
     False then indexes will not be created by MongoEngine.  This is useful in
     production systems where index creation is performed as part of a
     deployment system.
@@ -122,12 +139,15 @@ class Document(BaseDocument):
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
-    my_metaclass  = TopLevelDocumentMetaclass
+    my_metaclass = TopLevelDocumentMetaclass
     __metaclass__ = TopLevelDocumentMetaclass
 
+    __slots__ = ('__objects')
+
     def pk():
         """Primary key alias
         """
+
         def fget(self):
             return getattr(self, self._meta['id_field'])
 
@@ -161,7 +181,7 @@ class Document(BaseDocument):
                     if options.get('max') != max_documents or \
                        options.get('size') != max_size:
                         msg = (('Cannot create collection "%s" as a capped '
-                               'collection as it already exists')
+                                'collection as it already exists')
                                % cls._collection)
                         raise InvalidCollectionError(msg)
                 else:
@@ -178,9 +198,47 @@ class Document(BaseDocument):
                 cls.ensure_indexes()
         return cls._collection
 
+    def modify(self, query={}, **update):
+        """Perform an atomic update of the document in the database and reload
+        the document object using updated version.
+
+        Returns True if the document has been updated or False if the document
+        in the database doesn't match the query.
+
+        .. note:: All unsaved changes that has been made to the document are
+            rejected if the method returns True.
+
+        :param query: the update will be performed only if the document in the
+            database matches the query
+        :param update: Django-style update keyword arguments
+        """
+
+        if self.pk is None:
+            raise InvalidDocumentError("The document does not have a primary key.")
+
+        id_field = self._meta["id_field"]
+        query = query.copy() if isinstance(query, dict) else query.to_query(self)
+
+        if id_field not in query:
+            query[id_field] = self.pk
+        elif query[id_field] != self.pk:
+            raise InvalidQueryError("Invalid document modify query: it must modify only this document.")
+
+        updated = self._qs(**query).modify(new=True, **update)
+        if updated is None:
+            return False
+
+        for field in self._fields_ordered:
+            setattr(self, field, self._reload(field, updated[field]))
+
+        self._changed_fields = updated._changed_fields
+        self._created = False
+
+        return True
+
     def save(self, force_insert=False, validate=True, clean=True,
              write_concern=None,  cascade=None, cascade_kwargs=None,
-             _refs=None, **kwargs):
+             _refs=None, save_condition=None, **kwargs):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -203,6 +261,8 @@ class Document(BaseDocument):
         :param cascade_kwargs: (optional) kwargs dictionary to be passed throw
             to cascading saves.  Implies ``cascade=True``.
         :param _refs: A list of processed references used in cascading saves
+        :param save_condition: only perform save if matching record in db
+            satisfies condition(s) (e.g., version number)
 
         .. versionchanged:: 0.5
             In existing documents it only saves changed fields using
@@ -217,6 +277,9 @@ class Document(BaseDocument):
             meta['cascade'] = True.  Also you can pass different kwargs to
             the cascade save using cascade_kwargs which overwrites the
             existing kwargs with custom values.
+        .. versionchanged:: 0.8.5
+            Optional save_condition that only overwrites existing documents
+            if the condition is satisfied in the current db record.
         """
         signals.pre_save.send(self.__class__, document=self)
 
@@ -230,10 +293,13 @@ class Document(BaseDocument):
 
         created = ('_id' not in doc or self._created or force_insert)
 
-        signals.pre_save_post_validation.send(self.__class__, document=self, created=created)
+        signals.pre_save_post_validation.send(self.__class__, document=self,
+                                              created=created)
 
         try:
             collection = self._get_collection()
+            if self._meta.get('auto_create_index', True):
+                self.ensure_indexes()
             if created:
                 if force_insert:
                     object_id = collection.insert(doc, **write_concern)
@@ -243,7 +309,12 @@ class Document(BaseDocument):
                 object_id = doc['_id']
                 updates, removals = self._delta()
                 # Need to add shard key to query, or you get an error
-                select_dict = {'_id': object_id}
+                if save_condition is not None:
+                    select_dict = transform.query(self.__class__,
+                                                  **save_condition)
+                else:
+                    select_dict = {}
+                select_dict['_id'] = object_id
                 shard_key = self.__class__._meta.get('shard_key', tuple())
                 for k in shard_key:
                     actual_key = self._db_field_map.get(k, k)
@@ -263,12 +334,14 @@ class Document(BaseDocument):
                 if removals:
                     update_query["$unset"] = removals
                 if updates or removals:
+                    upsert = save_condition is None
                     last_error = collection.update(select_dict, update_query,
-                                                   upsert=True, **write_concern)
+                                                   upsert=upsert, **write_concern)
                     created = is_new_object(last_error)
 
             if cascade is None:
-                cascade = self._meta.get('cascade', False) or cascade_kwargs is not None
+                cascade = self._meta.get(
+                    'cascade', False) or cascade_kwargs is not None
 
             if cascade:
                 kwargs = {
@@ -293,12 +366,12 @@ class Document(BaseDocument):
                 raise NotUniqueError(message % unicode(err))
             raise OperationError(message % unicode(err))
         id_field = self._meta['id_field']
-        if id_field not in self._meta.get('shard_key', []):
+        if created or id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
+        signals.post_save.send(self.__class__, document=self, created=created)
         self._clear_changed_fields()
         self._created = False
-        signals.post_save.send(self.__class__, document=self, created=created)
         return self
 
     def cascade_save(self, *args, **kwargs):
@@ -361,7 +434,8 @@ class Document(BaseDocument):
                     del(query["_cls"])
                 return self._qs.filter(**query).update_one(**kwargs)
             else:
-                raise OperationError('attempt to update a document not yet saved')
+                raise OperationError(
+                    'attempt to update a document not yet saved')
 
         # Need to add shard key to query, or you get an error
         return self._qs.filter(**self._object_key).update_one(**kwargs)
@@ -380,7 +454,8 @@ class Document(BaseDocument):
         signals.pre_delete.send(self.__class__, document=self)
 
         try:
-            self._qs.filter(**self._object_key).delete(write_concern=write_concern, _from_doc_delete=True)
+            self._qs.filter(
+                **self._object_key).delete(write_concern=write_concern, _from_doc_delete=True)
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
@@ -396,10 +471,11 @@ class Document(BaseDocument):
             user.switch_db('archive-db')
             user.save()
 
-        If you need to read from another database see
-        :class:`~mongoengine.context_managers.switch_db`
+        :param str db_alias: The database alias to use for saving the document
 
-        :param db_alias: The database alias to use for saving the document
+        .. seealso::
+            Use :class:`~mongoengine.context_managers.switch_collection`
+            if you need to read from another collection
         """
         with switch_db(self.__class__, db_alias) as cls:
             collection = cls._get_collection()
@@ -422,11 +498,12 @@ class Document(BaseDocument):
             user.switch_collection('old-users')
             user.save()
 
-        If you need to read from another database see
-        :class:`~mongoengine.context_managers.switch_db`
-
-        :param collection_name: The database alias to use for saving the
+        :param str collection_name: The database alias to use for saving the
             document
+
+        .. seealso::
+            Use :class:`~mongoengine.context_managers.switch_db`
+            if you need to read from another database
         """
         with switch_collection(self.__class__, collection_name) as cls:
             collection = cls._get_collection()
@@ -447,27 +524,47 @@ class Document(BaseDocument):
         DeReference()([self], max_depth + 1)
         return self
 
-    def reload(self, max_depth=1):
+    def reload(self, *fields, **kwargs):
         """Reloads all attributes from the database.
 
+        :param fields: (optional) args list of fields to reload
+        :param max_depth: (optional) depth of dereferencing to follow
+
         .. versionadded:: 0.1.2
         .. versionchanged:: 0.6  Now chainable
+        .. versionchanged:: 0.9  Can provide specific fields to reload
         """
+        max_depth = 1
+        if fields and isinstance(fields[0], int):
+            max_depth = fields[0]
+            fields = fields[1:]
+        elif "max_depth" in kwargs:
+            max_depth = kwargs["max_depth"]
+
         if not self.pk:
             raise self.DoesNotExist("Document does not exist")
         obj = self._qs.read_preference(ReadPreference.PRIMARY).filter(
-                    **self._object_key).limit(1).select_related(max_depth=max_depth)
-
+            **self._object_key).only(*fields).limit(1
+                                                    ).select_related(max_depth=max_depth)
 
         if obj:
             obj = obj[0]
         else:
             raise self.DoesNotExist("Document does not exist")
+
         for field in self._fields_ordered:
-            setattr(self, field, self._reload(field, obj[field]))
+            if not fields or field in fields:
+                try:
+                    setattr(self, field, self._reload(field, obj[field]))
+                except KeyError:
+                    # If field is removed from the database while the object
+                    # is in memory, a reload would cause a KeyError
+                    # i.e. obj.update(unset__field=1) followed by obj.reload()
+                    delattr(self, field)
+
         self._changed_fields = obj._changed_fields
         self._created = False
-        return obj
+        return self
 
     def _reload(self, key, value):
         """Used by :meth:`~mongoengine.Document.reload` to ensure the
@@ -476,6 +573,9 @@ class Document(BaseDocument):
         if isinstance(value, BaseDict):
             value = [(k, self._reload(k, v)) for k, v in value.items()]
             value = BaseDict(value, self, key)
+        elif isinstance(value, EmbeddedDocumentList):
+            value = [self._reload(key, v) for v in value]
+            value = EmbeddedDocumentList(value, self, key)
         elif isinstance(value, BaseList):
             value = [self._reload(key, v) for v in value]
             value = BaseList(value, self, key)
@@ -498,8 +598,8 @@ class Document(BaseDocument):
         object.
         """
         classes = [get_document(class_name)
-                    for class_name in cls._subclasses
-                    if class_name != cls.__name__] + [cls]
+                   for class_name in cls._subclasses
+                   if class_name != cls.__name__] + [cls]
         documents = [get_document(class_name)
                      for class_name in document_cls._subclasses
                      if class_name != document_cls.__name__] + [document_cls]
@@ -521,7 +621,7 @@ class Document(BaseDocument):
 
     @classmethod
     def ensure_index(cls, key_or_list, drop_dups=False, background=False,
-        **kwargs):
+                     **kwargs):
         """Ensure that the given indexes are in place.
 
         :param key_or_list: a single index key or a list of index keys (to
@@ -552,7 +652,9 @@ class Document(BaseDocument):
         index_cls = cls._meta.get('index_cls', True)
 
         collection = cls._get_collection()
-        if collection.read_preference > 1:
+        # 746: when connection is via mongos, the read preference is not necessarily an indication that
+        # this code runs on a secondary
+        if not collection.is_mongos and collection.read_preference > 1:
             return
 
         # determine if an index which we are creating includes
@@ -576,7 +678,7 @@ class Document(BaseDocument):
         # If _cls is being used (for polymorphism), it needs an index,
         # only if another index doesn't begin with _cls
         if (index_cls and not cls_indexed and
-           cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
+                cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
             collection.ensure_index('_cls', background=background,
                                     **index_opts)
 
@@ -589,26 +691,27 @@ class Document(BaseDocument):
         if cls._meta.get('abstract'):
             return []
 
-        # get all the base classes, subclasses and sieblings
+        # get all the base classes, subclasses and siblings
         classes = []
+
         def get_classes(cls):
 
             if (cls not in classes and
-               isinstance(cls, TopLevelDocumentMetaclass)):
+                    isinstance(cls, TopLevelDocumentMetaclass)):
                 classes.append(cls)
 
             for base_cls in cls.__bases__:
                 if (isinstance(base_cls, TopLevelDocumentMetaclass) and
-                   base_cls != Document and
-                   not base_cls._meta.get('abstract') and
-                   base_cls._get_collection().full_name == cls._get_collection().full_name and
-                   base_cls not in classes):
+                        base_cls != Document and
+                        not base_cls._meta.get('abstract') and
+                        base_cls._get_collection().full_name == cls._get_collection().full_name and
+                        base_cls not in classes):
                     classes.append(base_cls)
                     get_classes(base_cls)
             for subclass in cls.__subclasses__():
                 if (isinstance(base_cls, TopLevelDocumentMetaclass) and
-                   subclass._get_collection().full_name == cls._get_collection().full_name and
-                   subclass not in classes):
+                        subclass._get_collection().full_name == cls._get_collection().full_name and
+                        subclass not in classes):
                     classes.append(subclass)
                     get_classes(subclass)
 
@@ -636,8 +739,8 @@ class Document(BaseDocument):
         if [(u'_id', 1)] not in indexes:
             indexes.append([(u'_id', 1)])
         if (cls._meta.get('index_cls', True) and
-           cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
-             indexes.append([(u'_cls', 1)])
+                cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True):
+            indexes.append([(u'_cls', 1)])
 
         return indexes
 
@@ -648,7 +751,8 @@ class Document(BaseDocument):
         """
 
         required = cls.list_indexes()
-        existing = [info['key'] for info in cls._get_collection().index_information().values()]
+        existing = [info['key']
+                    for info in cls._get_collection().index_information().values()]
         missing = [index for index in required if index not in existing]
         extra = [index for index in existing if index not in required]
 
@@ -666,6 +770,7 @@ class Document(BaseDocument):
 
 
 class DynamicDocument(Document):
+
     """A Dynamic Document class allowing flexible, expandable and uncontrolled
     schemas.  As a :class:`~mongoengine.Document` subclass, acts in the same
     way as an ordinary document but has expando style properties.  Any data
@@ -681,7 +786,7 @@ class DynamicDocument(Document):
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
-    my_metaclass  = TopLevelDocumentMetaclass
+    my_metaclass = TopLevelDocumentMetaclass
     __metaclass__ = TopLevelDocumentMetaclass
 
     _dynamic = True
@@ -697,6 +802,7 @@ class DynamicDocument(Document):
 
 
 class DynamicEmbeddedDocument(EmbeddedDocument):
+
     """A Dynamic Embedded Document class allowing flexible, expandable and
     uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more
     information about dynamic documents.
@@ -704,7 +810,7 @@ class DynamicEmbeddedDocument(EmbeddedDocument):
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
     # my_metaclass is defined so that metaclass can be queried in Python 2 & 3
-    my_metaclass  = DocumentMetaclass
+    my_metaclass = DocumentMetaclass
     __metaclass__ = DocumentMetaclass
 
     _dynamic = True
@@ -723,6 +829,7 @@ class DynamicEmbeddedDocument(EmbeddedDocument):
 
 
 class MapReduceDocument(object):
+
     """A document returned from a map/reduce query.
 
     :param collection: An instance of :class:`~pymongo.Collection`
@@ -753,7 +860,7 @@ class MapReduceDocument(object):
             try:
                 self.key = id_field_type(self.key)
             except:
-                raise Exception("Could not cast key as %s" % \
+                raise Exception("Could not cast key as %s" %
                                 id_field_type.__name__)
 
         if not hasattr(self, "_key_object"):

mongoengine/errors.py

@@ -5,7 +5,8 @@ from mongoengine.python_support import txt_type
 
 __all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError',
            'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError',
-           'OperationError', 'NotUniqueError', 'ValidationError')
+           'OperationError', 'NotUniqueError', 'FieldDoesNotExist',
+           'ValidationError')
 
 
 class NotRegistered(Exception):
@@ -40,6 +41,10 @@ class NotUniqueError(OperationError):
     pass
 
 
+class FieldDoesNotExist(Exception):
+    pass
+
+
 class ValidationError(AssertionError):
     """Validation exception.
 

mongoengine/fields.py

@@ -34,22 +34,25 @@ except ImportError:
     Image = None
     ImageOps = None
 
-__all__ = ['StringField',  'URLField',  'EmailField',  'IntField',  'LongField',
-           'FloatField',  'DecimalField',  'BooleanField',  'DateTimeField',
-           'ComplexDateTimeField',  'EmbeddedDocumentField', 'ObjectIdField',
-           'GenericEmbeddedDocumentField',  'DynamicField',  'ListField',
-           'SortedListField',  'DictField',  'MapField',  'ReferenceField',
-           'GenericReferenceField',  'BinaryField',  'GridFSError',
-           'GridFSProxy',  'FileField',  'ImageGridFsProxy',
-           'ImproperlyConfigured',  'ImageField',  'GeoPointField', 'PointField',
-           'LineStringField', 'PolygonField', 'SequenceField',  'UUIDField',
-           'GeoJsonBaseField']
+__all__ = [
+    'StringField', 'URLField', 'EmailField', 'IntField', 'LongField',
+    'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField',
+    'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField',
+    'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
+    'SortedListField', 'EmbeddedDocumentListField', 'DictField',
+    'MapField', 'ReferenceField', 'CachedReferenceField',
+    'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
+    'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
+    'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
+    'SequenceField', 'UUIDField', 'MultiPointField', 'MultiLineStringField',
+    'MultiPolygonField', 'GeoJsonBaseField']
 
 
 RECURSIVE_REFERENCE_CONSTANT = 'self'
 
 
 class StringField(BaseField):
+
     """A unicode string field.
     """
 
@@ -109,6 +112,7 @@ class StringField(BaseField):
 
 
 class URLField(StringField):
+
     """A field that validates input as an URL.
 
     .. versionadded:: 0.3
@@ -116,7 +120,8 @@ class URLField(StringField):
 
     _URL_REGEX = re.compile(
         r'^(?:http|ftp)s?://'  # http:// or https://
-        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
+        # domain...
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
         r'localhost|'  # localhost...
         r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
         r'(?::\d+)?'  # optional port
@@ -145,15 +150,19 @@ class URLField(StringField):
 
 
 class EmailField(StringField):
+
     """A field that validates input as an E-Mail-Address.
 
     .. versionadded:: 0.4
     """
 
     EMAIL_REGEX = re.compile(
-        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"  # dot-atom
-        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'  # quoted-string
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,6}$', re.IGNORECASE  # domain
+        # dot-atom
+        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
+        # quoted-string
+        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
+        # domain (max length of an ICAAN TLD is 22 characters)
+        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,22}$', re.IGNORECASE
     )
 
     def validate(self, value):
@@ -163,6 +172,7 @@ class EmailField(StringField):
 
 
 class IntField(BaseField):
+
     """An 32-bit integer field.
     """
 
@@ -197,6 +207,7 @@ class IntField(BaseField):
 
 
 class LongField(BaseField):
+
     """An 64-bit integer field.
     """
 
@@ -231,6 +242,7 @@ class LongField(BaseField):
 
 
 class FloatField(BaseField):
+
     """An floating point number field.
     """
 
@@ -265,6 +277,7 @@ class FloatField(BaseField):
 
 
 class DecimalField(BaseField):
+
     """A fixed-point decimal number field.
 
     .. versionchanged:: 0.8
@@ -278,7 +291,7 @@ class DecimalField(BaseField):
         :param max_value: Validation rule for the maximum acceptable value.
         :param force_string: Store as a string.
         :param precision: Number of decimal places to store.
-        :param rounding: The rounding rule from the python decimal libary:
+        :param rounding: The rounding rule from the python decimal library:
 
             - decimal.ROUND_CEILING (towards Infinity)
             - decimal.ROUND_DOWN (towards zero)
@@ -295,7 +308,7 @@ class DecimalField(BaseField):
         self.min_value = min_value
         self.max_value = max_value
         self.force_string = force_string
-        self.precision = decimal.Decimal(".%s" % ("0" * precision))
+        self.precision = precision
         self.rounding = rounding
 
         super(DecimalField, self).__init__(**kwargs)
@@ -309,9 +322,9 @@ class DecimalField(BaseField):
             value = decimal.Decimal("%s" % value)
         except decimal.InvalidOperation:
             return value
-        return value.quantize(self.precision, rounding=self.rounding)
+        return value.quantize(decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding)
 
-    def to_mongo(self, value):
+    def to_mongo(self, value, use_db_field=True):
         if value is None:
             return value
         if self.force_string:
@@ -338,6 +351,7 @@ class DecimalField(BaseField):
 
 
 class BooleanField(BaseField):
+
     """A boolean field type.
 
     .. versionadded:: 0.1.2
@@ -356,15 +370,16 @@ class BooleanField(BaseField):
 
 
 class DateTimeField(BaseField):
+
     """A datetime field.
 
     Uses the python-dateutil library if available alternatively use time.strptime
     to parse the dates.  Note: python-dateutil's parser is fully featured and when
-    installed you can utilise it to convert varing types of date formats into valid
+    installed you can utilise it to convert varying types of date formats into valid
     python datetime objects.
 
     Note: Microseconds are rounded to the nearest millisecond.
-      Pre UTC microsecond support is effecively broken.
+      Pre UTC microsecond support is effectively broken.
       Use :class:`~mongoengine.fields.ComplexDateTimeField` if you
       need accurate microsecond support.
     """
@@ -391,7 +406,7 @@ class DateTimeField(BaseField):
         if dateutil:
             try:
                 return dateutil.parser.parse(value)
-            except ValueError:
+            except (TypeError, ValueError):
                 return None
 
         # split usecs, because they are not recognized by strptime.
@@ -406,15 +421,15 @@ class DateTimeField(BaseField):
         kwargs = {'microsecond': usecs}
         try:  # Seconds are optional, so try converting seconds first.
             return datetime.datetime(*time.strptime(value,
-                                     '%Y-%m-%d %H:%M:%S')[:6], **kwargs)
+                                                    '%Y-%m-%d %H:%M:%S')[:6], **kwargs)
         except ValueError:
             try:  # Try without seconds.
                 return datetime.datetime(*time.strptime(value,
-                                         '%Y-%m-%d %H:%M')[:5], **kwargs)
+                                                        '%Y-%m-%d %H:%M')[:5], **kwargs)
             except ValueError:  # Try without hour/minutes/seconds.
                 try:
                     return datetime.datetime(*time.strptime(value,
-                                             '%Y-%m-%d')[:3], **kwargs)
+                                                            '%Y-%m-%d')[:3], **kwargs)
                 except ValueError:
                     return None
 
@@ -423,6 +438,7 @@ class DateTimeField(BaseField):
 
 
 class ComplexDateTimeField(StringField):
+
     """
     ComplexDateTimeField handles microseconds exactly instead of rounding
     like DateTimeField does.
@@ -494,7 +510,7 @@ class ComplexDateTimeField(StringField):
     def __get__(self, instance, owner):
         data = super(ComplexDateTimeField, self).__get__(instance, owner)
         if data is None:
-            return datetime.datetime.now()
+            return None if self.null else datetime.datetime.now()
         if isinstance(data, datetime.datetime):
             return data
         return self._convert_from_string(data)
@@ -525,6 +541,7 @@ class ComplexDateTimeField(StringField):
 
 
 class EmbeddedDocumentField(BaseField):
+
     """An embedded document field - with a declared document_type.
     Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`.
     """
@@ -551,10 +568,11 @@ class EmbeddedDocumentField(BaseField):
             return self.document_type._from_son(value)
         return value
 
-    def to_mongo(self, value):
+    def to_mongo(self, value, use_db_field=True, fields=[]):
         if not isinstance(value, self.document_type):
             return value
-        return self.document_type.to_mongo(value)
+        return self.document_type.to_mongo(value, use_db_field,
+                                           fields=fields)
 
     def validate(self, value, clean=True):
         """Make sure that the document instance is an instance of the
@@ -574,6 +592,7 @@ class EmbeddedDocumentField(BaseField):
 
 
 class GenericEmbeddedDocumentField(BaseField):
+
     """A generic embedded document field - allows any
     :class:`~mongoengine.EmbeddedDocument` to be stored.
 
@@ -601,24 +620,25 @@ class GenericEmbeddedDocumentField(BaseField):
 
         value.validate(clean=clean)
 
-    def to_mongo(self, document):
+    def to_mongo(self, document, use_db_field=True):
         if document is None:
             return None
 
-        data = document.to_mongo()
+        data = document.to_mongo(use_db_field)
         if not '_cls' in data:
             data['_cls'] = document._class_name
         return data
 
 
 class DynamicField(BaseField):
+
     """A truly dynamic field type capable of handling different and varying
     types of data.
 
     Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
 
     def to_mongo(self, value):
-        """Convert a Python type to a MongoDBcompatible type.
+        """Convert a Python type to a MongoDB compatible type.
         """
 
         if isinstance(value, basestring):
@@ -628,7 +648,7 @@ class DynamicField(BaseField):
             cls = value.__class__
             val = value.to_mongo()
             # If we its a document thats not inherited add _cls
-            if (isinstance(value, Document)):
+            if (isinstance(value,   Document)):
                 val = {"_ref": value.to_dbref(), "_cls": cls.__name__}
             if (isinstance(value, EmbeddedDocument)):
                 val['_cls'] = cls.__name__
@@ -675,6 +695,7 @@ class DynamicField(BaseField):
 
 
 class ListField(ComplexBaseField):
+
     """A list field that wraps a standard field, allowing multiple instances
     of the field to be used as a list in the database.
 
@@ -693,21 +714,48 @@ class ListField(ComplexBaseField):
         """Make sure that a list of valid fields is being used.
         """
         if (not isinstance(value, (list, tuple, QuerySet)) or
-           isinstance(value, basestring)):
+                isinstance(value, basestring)):
             self.error('Only lists and tuples may be used in a list field')
         super(ListField, self).validate(value)
 
     def prepare_query_value(self, op, value):
         if self.field:
             if op in ('set', 'unset') and (not isinstance(value, basestring)
-               and not isinstance(value, BaseDocument)
-               and hasattr(value, '__iter__')):
+                                           and not isinstance(value, BaseDocument)
+                                           and hasattr(value, '__iter__')):
                 return [self.field.prepare_query_value(op, v) for v in value]
             return self.field.prepare_query_value(op, value)
         return super(ListField, self).prepare_query_value(op, value)
 
 
+class EmbeddedDocumentListField(ListField):
+    """A :class:`~mongoengine.ListField` designed specially to hold a list of
+    embedded documents to provide additional query helpers.
+
+    .. note::
+        The only valid list values are subclasses of
+        :class:`~mongoengine.EmbeddedDocument`.
+
+    .. versionadded:: 0.9
+
+    """
+
+    def __init__(self, document_type, *args, **kwargs):
+        """
+        :param document_type: The type of
+         :class:`~mongoengine.EmbeddedDocument` the list will hold.
+        :param args: Arguments passed directly into the parent
+         :class:`~mongoengine.ListField`.
+        :param kwargs: Keyword arguments passed directly into the parent
+         :class:`~mongoengine.ListField`.
+        """
+        super(EmbeddedDocumentListField, self).__init__(
+            field=EmbeddedDocumentField(document_type), **kwargs
+        )
+
+
 class SortedListField(ListField):
+
     """A ListField that sorts the contents of its list before writing to
     the database in order to ensure that a sorted list is always
     retrieved.
@@ -739,6 +787,7 @@ class SortedListField(ListField):
                           reverse=self._order_reverse)
         return sorted(value, reverse=self._order_reverse)
 
+
 def key_not_string(d):
     """ Helper function to recursively determine if any key in a dictionary is
     not a string.
@@ -747,6 +796,7 @@ def key_not_string(d):
         if not isinstance(k, basestring) or (isinstance(v, dict) and key_not_string(v)):
             return True
 
+
 def key_has_dot_or_dollar(d):
     """ Helper function to recursively determine if any key in a dictionary
     contains a dot or a dollar sign.
@@ -755,12 +805,14 @@ def key_has_dot_or_dollar(d):
         if ('.' in k or '$' in k) or (isinstance(v, dict) and key_has_dot_or_dollar(v)):
             return True
 
+
 class DictField(ComplexBaseField):
+
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
 
     .. note::
-        Required means it cannot be empty - as the default for ListFields is []
+        Required means it cannot be empty - as the default for DictFields is {}
 
     .. versionadded:: 0.3
     .. versionchanged:: 0.5 - Can now handle complex / varying types of data
@@ -801,12 +853,17 @@ class DictField(ComplexBaseField):
             return StringField().prepare_query_value(op, value)
 
         if hasattr(self.field, 'field'):
+            if op in ('set', 'unset') and isinstance(value, dict):
+                return dict(
+                    (k, self.field.prepare_query_value(op, v))
+                    for k, v in value.items())
             return self.field.prepare_query_value(op, value)
 
         return super(DictField, self).prepare_query_value(op, value)
 
 
 class MapField(DictField):
+
     """A field that maps a name to a specified field type. Similar to
     a DictField, except the 'value' of each item must match the specified
     field type.
@@ -822,12 +879,13 @@ class MapField(DictField):
 
 
 class ReferenceField(BaseField):
+
     """A reference to a document that will be automatically dereferenced on
     access (lazily).
 
     Use the `reverse_delete_rule` to handle what should happen if the document
     the field is referencing is deleted.  EmbeddedDocuments, DictFields and
-    MapFields do not support reverse_delete_rules and an `InvalidDocumentError`
+    MapFields does not support reverse_delete_rule and an `InvalidDocumentError`
     will be raised if trying to set on one of these Document / Field types.
 
     The options are:
@@ -851,7 +909,7 @@ class ReferenceField(BaseField):
         Bar.register_delete_rule(Foo, 'bar', NULLIFY)
 
     .. note ::
-        `reverse_delete_rules` do not trigger pre / post delete signals to be
+        `reverse_delete_rule` does not trigger pre / post delete signals to be
         triggered.
 
     .. versionchanged:: 0.5 added `reverse_delete_rule`
@@ -932,7 +990,7 @@ class ReferenceField(BaseField):
         """Convert a MongoDB-compatible type to a Python type.
         """
         if (not self.dbref and
-           not isinstance(value, (DBRef, Document, EmbeddedDocument))):
+                not isinstance(value, (DBRef, Document, EmbeddedDocument))):
             collection = self.document_type._get_collection_name()
             value = DBRef(collection, self.document_type.id.to_python(value))
         return value
@@ -955,7 +1013,147 @@ class ReferenceField(BaseField):
         return self.document_type._fields.get(member_name)
 
 
+class CachedReferenceField(BaseField):
+
+    """
+    A referencefield with cache fields to porpuse pseudo-joins
+    .. versionadded:: 0.9
+    """
+
+    def __init__(self, document_type, fields=[], auto_sync=True, **kwargs):
+        """Initialises the Cached Reference Field.
+
+        :param fields:  A list of fields to be cached in document
+        :param auto_sync: if True documents are auto updated.
+        """
+        if not isinstance(document_type, basestring) and \
+                not issubclass(document_type, (Document, basestring)):
+
+            self.error('Argument to CachedReferenceField constructor must be a'
+                       ' document class or a string')
+
+        self.auto_sync = auto_sync
+        self.document_type_obj = document_type
+        self.fields = fields
+        super(CachedReferenceField, self).__init__(**kwargs)
+
+    def start_listener(self):
+        from mongoengine import signals
+        signals.post_save.connect(self.on_document_pre_save,
+                                  sender=self.document_type)
+
+    def on_document_pre_save(self, sender, document, created, **kwargs):
+        if not created:
+            update_kwargs = dict(
+                ('set__%s__%s' % (self.name, k), v)
+                for k, v in document._delta()[0].items()
+                if k in self.fields)
+
+            if update_kwargs:
+                filter_kwargs = {}
+                filter_kwargs[self.name] = document
+
+                self.owner_document.objects(
+                    **filter_kwargs).update(**update_kwargs)
+
+    def to_python(self, value):
+        if isinstance(value, dict):
+            collection = self.document_type._get_collection_name()
+            value = DBRef(
+                collection, self.document_type.id.to_python(value['_id']))
+
+        return value
+
+    @property
+    def document_type(self):
+        if isinstance(self.document_type_obj, basestring):
+            if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
+                self.document_type_obj = self.owner_document
+            else:
+                self.document_type_obj = get_document(self.document_type_obj)
+        return self.document_type_obj
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        # Get value from document instance if available
+        value = instance._data.get(self.name)
+        self._auto_dereference = instance._fields[self.name]._auto_dereference
+        # Dereference DBRefs
+        if self._auto_dereference and isinstance(value, DBRef):
+            value = self.document_type._get_db().dereference(value)
+            if value is not None:
+                instance._data[self.name] = self.document_type._from_son(value)
+
+        return super(CachedReferenceField, self).__get__(instance, owner)
+
+    def to_mongo(self, document):
+        id_field_name = self.document_type._meta['id_field']
+        id_field = self.document_type._fields[id_field_name]
+        doc_tipe = self.document_type
+
+        if isinstance(document, Document):
+            # We need the id from the saved object to create the DBRef
+            id_ = document.pk
+            if id_ is None:
+                self.error('You can only reference documents once they have'
+                           ' been saved to the database')
+        else:
+            self.error('Only accept a document object')
+
+        value = SON((
+            ("_id", id_field.to_mongo(id_)),
+        ))
+
+        value.update(dict(document.to_mongo(fields=self.fields)))
+        return value
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return None
+
+        if isinstance(value, Document):
+            if value.pk is None:
+                self.error('You can only reference documents once they have'
+                           ' been saved to the database')
+            return {'_id': value.pk}
+
+        raise NotImplementedError
+
+    def validate(self, value):
+
+        if not isinstance(value, (self.document_type)):
+            self.error("A CachedReferenceField only accepts documents")
+
+        if isinstance(value, Document) and value.id is None:
+            self.error('You can only reference documents once they have been '
+                       'saved to the database')
+
+    def lookup_member(self, member_name):
+        return self.document_type._fields.get(member_name)
+
+    def sync_all(self):
+        """
+        Sync all cached fields on demand.
+        Caution: this operation may be slower.
+        """
+        update_key = 'set__%s' % self.name
+
+        for doc in self.document_type.objects:
+            filter_kwargs = {}
+            filter_kwargs[self.name] = doc
+
+            update_kwargs = {}
+            update_kwargs[update_key] = doc
+
+            self.owner_document.objects(
+                **filter_kwargs).update(**update_kwargs)
+
+
 class GenericReferenceField(BaseField):
+
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
 
@@ -974,6 +1172,7 @@ class GenericReferenceField(BaseField):
             return self
 
         value = instance._data.get(self.name)
+
         self._auto_dereference = instance._fields[self.name]._auto_dereference
         if self._auto_dereference and isinstance(value, (dict, SON)):
             instance._data[self.name] = self.dereference(value)
@@ -1001,7 +1200,7 @@ class GenericReferenceField(BaseField):
             doc = doc_cls._from_son(doc)
         return doc
 
-    def to_mongo(self, document):
+    def to_mongo(self, document, use_db_field=True):
         if document is None:
             return None
 
@@ -1036,6 +1235,7 @@ class GenericReferenceField(BaseField):
 
 
 class BinaryField(BaseField):
+
     """A binary data field.
     """
 
@@ -1056,7 +1256,7 @@ class BinaryField(BaseField):
         if not isinstance(value, (bin_type, txt_type, Binary)):
             self.error("BinaryField only accepts instances of "
                        "(%s, %s, Binary)" % (
-                       bin_type.__name__, txt_type.__name__))
+                           bin_type.__name__, txt_type.__name__))
 
         if self.max_bytes is not None and len(value) > self.max_bytes:
             self.error('Binary value is too long')
@@ -1067,6 +1267,7 @@ class GridFSError(Exception):
 
 
 class GridFSProxy(object):
+
     """Proxy object to handle writing and reading of files to and from GridFS
 
     .. versionadded:: 0.4
@@ -1121,7 +1322,8 @@ class GridFSProxy(object):
         return '<%s: %s>' % (self.__class__.__name__, self.grid_id)
 
     def __str__(self):
-        name = getattr(self.get(), 'filename', self.grid_id) if self.get() else '(no file)'
+        name = getattr(
+            self.get(), 'filename', self.grid_id) if self.get() else '(no file)'
         return '<%s: %s>' % (self.__class__.__name__, name)
 
     def __eq__(self, other):
@@ -1135,7 +1337,8 @@ class GridFSProxy(object):
     @property
     def fs(self):
         if not self._fs:
-            self._fs = gridfs.GridFS(get_db(self.db_alias), self.collection_name)
+            self._fs = gridfs.GridFS(
+                get_db(self.db_alias), self.collection_name)
         return self._fs
 
     def get(self, id=None):
@@ -1154,6 +1357,7 @@ class GridFSProxy(object):
     def new_file(self, **kwargs):
         self.newfile = self.fs.new_file(**kwargs)
         self.grid_id = self.newfile._id
+        self._mark_as_changed()
 
     def put(self, file_obj, **kwargs):
         if self.grid_id:
@@ -1209,6 +1413,7 @@ class GridFSProxy(object):
 
 
 class FileField(BaseField):
+
     """A GridFS storage field.
 
     .. versionadded:: 0.4
@@ -1253,7 +1458,8 @@ class FileField(BaseField):
                     pass
 
             # Create a new proxy object as we don't already have one
-            instance._data[key] = self.get_proxy_obj(key=key, instance=instance)
+            instance._data[key] = self.get_proxy_obj(
+                key=key, instance=instance)
             instance._data[key].put(value)
         else:
             instance._data[key] = value
@@ -1291,11 +1497,13 @@ class FileField(BaseField):
 
 
 class ImageGridFsProxy(GridFSProxy):
+
     """
     Proxy for ImageField
 
     versionadded: 0.6
     """
+
     def put(self, file_obj, **kwargs):
         """
         Insert a image in database
@@ -1312,6 +1520,16 @@ class ImageGridFsProxy(GridFSProxy):
         except Exception, e:
             raise ValidationError('Invalid image: %s' % e)
 
+        # Progressive JPEG
+        progressive = img.info.get('progressive') or False
+
+        if (kwargs.get('progressive') and
+                isinstance(kwargs.get('progressive'), bool) and
+                img_format == 'JPEG'):
+            progressive = True
+        else:
+            progressive = False
+
         if (field.size and (img.size[0] > field.size['width'] or
                             img.size[1] > field.size['height'])):
             size = field.size
@@ -1331,7 +1549,8 @@ class ImageGridFsProxy(GridFSProxy):
             size = field.thumbnail_size
 
             if size['force']:
-                thumbnail = ImageOps.fit(img, (size['width'], size['height']), Image.ANTIALIAS)
+                thumbnail = ImageOps.fit(
+                    img, (size['width'], size['height']), Image.ANTIALIAS)
             else:
                 thumbnail = img.copy()
                 thumbnail.thumbnail((size['width'],
@@ -1339,14 +1558,14 @@ class ImageGridFsProxy(GridFSProxy):
                                     Image.ANTIALIAS)
 
         if thumbnail:
-            thumb_id = self._put_thumbnail(thumbnail, img_format)
+            thumb_id = self._put_thumbnail(thumbnail, img_format, progressive)
         else:
             thumb_id = None
 
         w, h = img.size
 
         io = StringIO()
-        img.save(io, img_format)
+        img.save(io, img_format, progressive=progressive)
         io.seek(0)
 
         return super(ImageGridFsProxy, self).put(io,
@@ -1357,18 +1576,18 @@ class ImageGridFsProxy(GridFSProxy):
                                                  **kwargs)
 
     def delete(self, *args, **kwargs):
-        #deletes thumbnail
+        # deletes thumbnail
         out = self.get()
         if out and out.thumbnail_id:
             self.fs.delete(out.thumbnail_id)
 
         return super(ImageGridFsProxy, self).delete(*args, **kwargs)
 
-    def _put_thumbnail(self, thumbnail, format, **kwargs):
+    def _put_thumbnail(self, thumbnail, format, progressive, **kwargs):
         w, h = thumbnail.size
 
         io = StringIO()
-        thumbnail.save(io, format)
+        thumbnail.save(io, format, progressive=progressive)
         io.seek(0)
 
         return self.fs.put(io, width=w,
@@ -1417,6 +1636,7 @@ class ImproperlyConfigured(Exception):
 
 
 class ImageField(FileField):
+
     """
     A Image File storage field.
 
@@ -1455,7 +1675,8 @@ class ImageField(FileField):
 
 
 class SequenceField(BaseField):
-    """Provides a sequental counter see:
+
+    """Provides a sequential counter see:
      http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers
 
     .. note::
@@ -1524,7 +1745,7 @@ class SequenceField(BaseField):
         data = collection.find_one({"_id": sequence_id})
 
         if data:
-            return self.value_decorator(data['next']+1)
+            return self.value_decorator(data['next'] + 1)
 
         return self.value_decorator(1)
 
@@ -1554,6 +1775,14 @@ class SequenceField(BaseField):
 
         return super(SequenceField, self).__set__(instance, value)
 
+    def prepare_query_value(self, op, value):
+        """
+        This method is overridden in order to convert the query value into to required
+        type. We need to do this in order to be able to successfully compare query
+        values passed as string, the base implementation returns the value as is.
+        """
+        return self.value_decorator(value)
+
     def to_python(self, value):
         if value is None:
             value = self.generate()
@@ -1561,6 +1790,7 @@ class SequenceField(BaseField):
 
 
 class UUIDField(BaseField):
+
     """A UUID field.
 
     .. versionadded:: 0.6
@@ -1613,7 +1843,13 @@ class UUIDField(BaseField):
 
 
 class GeoPointField(BaseField):
-    """A list storing a latitude and longitude.
+
+    """A list storing a longitude and latitude coordinate.
+
+    .. note:: this represents a generic point in a 2D plane and a legacy way of
+        representing a geo point. It admits 2d indexes but not "2dsphere" indexes
+        in MongoDB > 2.4 which are more natural for modeling geospatial points.
+        See :ref:`geospatial-indexes`
 
     .. versionadded:: 0.4
     """
@@ -1628,14 +1864,17 @@ class GeoPointField(BaseField):
                        'of (x, y)')
 
         if not len(value) == 2:
-            self.error("Value (%s) must be a two-dimensional point" % repr(value))
+            self.error("Value (%s) must be a two-dimensional point" %
+                       repr(value))
         elif (not isinstance(value[0], (float, int)) or
               not isinstance(value[1], (float, int))):
-            self.error("Both values (%s) in point must be float or int" % repr(value))
+            self.error(
+                "Both values (%s) in point must be float or int" % repr(value))
 
 
 class PointField(GeoJsonBaseField):
-    """A geo json field storing a latitude and longitude.
+
+    """A GeoJSON field storing a longitude and latitude coordinate.
 
     The data is represented as:
 
@@ -1648,13 +1887,15 @@ class PointField(GeoJsonBaseField):
     to set the value.
 
     Requires mongodb >= 2.4
+
     .. versionadded:: 0.8
     """
     _type = "Point"
 
 
 class LineStringField(GeoJsonBaseField):
-    """A geo json field storing a line of latitude and longitude coordinates.
+
+    """A GeoJSON field storing a line of longitude and latitude coordinates.
 
     The data is represented as:
 
@@ -1666,13 +1907,15 @@ class LineStringField(GeoJsonBaseField):
     You can either pass a dict with the full information or a list of points.
 
     Requires mongodb >= 2.4
+
     .. versionadded:: 0.8
     """
     _type = "LineString"
 
 
 class PolygonField(GeoJsonBaseField):
-    """A geo json field storing a polygon of latitude and longitude coordinates.
+
+    """A GeoJSON field storing a polygon of longitude and latitude coordinates.
 
     The data is represented as:
 
@@ -1687,6 +1930,77 @@ class PolygonField(GeoJsonBaseField):
     holes.
 
     Requires mongodb >= 2.4
+
     .. versionadded:: 0.8
     """
     _type = "Polygon"
+
+
+class MultiPointField(GeoJsonBaseField):
+
+    """A GeoJSON field storing a list of Points.
+
+    The data is represented as:
+
+    .. code-block:: js
+
+        { "type" : "MultiPoint" ,
+          "coordinates" : [[x1, y1], [x2, y2]]}
+
+    You can either pass a dict with the full information or a list
+    to set the value.
+
+    Requires mongodb >= 2.6
+
+    .. versionadded:: 0.9
+    """
+    _type = "MultiPoint"
+
+
+class MultiLineStringField(GeoJsonBaseField):
+
+    """A GeoJSON field storing a list of LineStrings.
+
+    The data is represented as:
+
+    .. code-block:: js
+
+        { "type" : "MultiLineString" ,
+          "coordinates" : [[[x1, y1], [x1, y1] ... [xn, yn]],
+                           [[x1, y1], [x1, y1] ... [xn, yn]]]}
+
+    You can either pass a dict with the full information or a list of points.
+
+    Requires mongodb >= 2.6
+
+    .. versionadded:: 0.9
+    """
+    _type = "MultiLineString"
+
+
+class MultiPolygonField(GeoJsonBaseField):
+
+    """A GeoJSON field storing  list of Polygons.
+
+    The data is represented as:
+
+    .. code-block:: js
+
+        { "type" : "MultiPolygon" ,
+          "coordinates" : [[
+                [[x1, y1], [x1, y1] ... [xn, yn]],
+                [[x1, y1], [x1, y1] ... [xn, yn]]
+            ], [
+                [[x1, y1], [x1, y1] ... [xn, yn]],
+                [[x1, y1], [x1, y1] ... [xn, yn]]
+            ]
+        }
+
+    You can either pass a dict with the full information or a list
+    of Polygons.
+
+    Requires mongodb >= 2.6
+
+    .. versionadded:: 0.9
+    """
+    _type = "MultiPolygon"

mongoengine/python_support.py

@@ -3,8 +3,6 @@
 import sys
 
 PY3 = sys.version_info[0] == 3
-PY25 = sys.version_info[:2] == (2, 5)
-UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264
 
 if PY3:
     import codecs
@@ -29,33 +27,3 @@ else:
     txt_type = unicode
 
 str_types = (bin_type, txt_type)
-
-if PY25:
-    def product(*args, **kwds):
-        pools = map(tuple, args) * kwds.get('repeat', 1)
-        result = [[]]
-        for pool in pools:
-            result = [x + [y] for x in result for y in pool]
-        for prod in result:
-            yield tuple(prod)
-    reduce = reduce
-else:
-    from itertools import product
-    from functools import reduce
-
-
-# For use with Python 2.5
-# converts all keys from unicode to str for d and all nested dictionaries
-def to_str_keys_recursive(d):
-    if isinstance(d, list):
-        for val in d:
-            if isinstance(val, (dict, list)):
-                to_str_keys_recursive(val)
-    elif isinstance(d, dict):
-        for key, val in d.items():
-            if isinstance(val, (dict, list)):
-                to_str_keys_recursive(val)
-            if isinstance(key, unicode):
-                d[str(key)] = d.pop(key)
-    else:
-        raise ValueError("non list/dict parameter not allowed")

mongoengine/queryset/base.py

@@ -7,17 +7,20 @@ import pprint
 import re
 import warnings
 
+from bson import SON
 from bson.code import Code
 from bson import json_util
 import pymongo
+import pymongo.errors
 from pymongo.common import validate_read_preference
 
 from mongoengine import signals
+from mongoengine.connection import get_db
+from mongoengine.context_managers import switch_db
 from mongoengine.common import _import_class
 from mongoengine.base.common import get_document
 from mongoengine.errors import (OperationError, NotUniqueError,
                                 InvalidQueryError, LookUpError)
-
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
@@ -36,6 +39,7 @@ RE_TYPE = type(re.compile(''))
 
 
 class BaseQuerySet(object):
+
     """A set of results returned from a query. Wraps a MongoDB cursor,
     providing :class:`~mongoengine.Document` objects as the results.
     """
@@ -50,7 +54,7 @@ class BaseQuerySet(object):
         self._initial_query = {}
         self._where_clause = None
         self._loaded_fields = QueryFieldList()
-        self._ordering = []
+        self._ordering = None
         self._snapshot = False
         self._timeout = True
         self._class_check = True
@@ -61,6 +65,7 @@ class BaseQuerySet(object):
         self._none = False
         self._as_pymongo = False
         self._as_pymongo_coerce = False
+        self._search_text = None
 
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -68,12 +73,15 @@ class BaseQuerySet(object):
             if len(self._document._subclasses) == 1:
                 self._initial_query = {"_cls": self._document._subclasses[0]}
             else:
-                self._initial_query = {"_cls": {"$in": self._document._subclasses}}
+                self._initial_query = {
+                    "_cls": {"$in": self._document._subclasses}}
             self._loaded_fields = QueryFieldList(always_include=['_cls'])
         self._cursor_obj = None
         self._limit = None
         self._skip = None
         self._hint = -1  # Using -1 as None is a valid value for hint
+        self.only_fields = []
+        self._max_time_ms = None
 
     def __call__(self, q_obj=None, class_check=True, slave_okay=False,
                  read_preference=None, **query):
@@ -144,16 +152,35 @@ class BaseQuerySet(object):
             if queryset._scalar:
                 return queryset._get_scalar(
                     queryset._document._from_son(queryset._cursor[key],
-                                                 _auto_dereference=self._auto_dereference))
+                                                 _auto_dereference=self._auto_dereference,
+                                                 only_fields=self.only_fields))
+
             if queryset._as_pymongo:
-                return queryset._get_as_pymongo(queryset._cursor.next())
+                return queryset._get_as_pymongo(queryset._cursor[key])
             return queryset._document._from_son(queryset._cursor[key],
-                                                _auto_dereference=self._auto_dereference)
+                                               _auto_dereference=self._auto_dereference, only_fields=self.only_fields)
+
         raise AttributeError
 
     def __iter__(self):
         raise NotImplementedError
 
+    def _has_data(self):
+        """ Retrieves whether cursor has any data. """
+
+        queryset = self.order_by()
+        return False if queryset.first() is None else True
+
+    def __nonzero__(self):
+        """ Avoid to open all records in an if stmt in Py2. """
+
+        return self._has_data()
+
+    def __bool__(self):
+        """ Avoid to open all records in an if stmt in Py3. """
+
+        return self._has_data()
+
     # Core functions
 
     def all(self):
@@ -165,6 +192,32 @@ class BaseQuerySet(object):
         """
         return self.__call__(*q_objs, **query)
 
+    def search_text(self, text, language=None):
+        """
+        Start a text search, using text indexes.
+        Require: MongoDB server version 2.6+.
+
+        :param language:  The language that determines the list of stop words
+            for the search and the rules for the stemmer and tokenizer.
+            If not specified, the search uses the default language of the index.
+            For supported languages, see `Text Search Languages <http://docs.mongodb.org/manual/reference/text-search-languages/#text-search-languages>`.
+        """
+        queryset = self.clone()
+        if queryset._search_text:
+            raise OperationError(
+                "It is not possible to use search_text two times.")
+
+        query_kwargs = SON({'$search': text})
+        if language:
+            query_kwargs['$language'] = language
+
+        queryset._query_obj &= Q(__raw__={'$text': query_kwargs})
+        queryset._mongo_query = None
+        queryset._cursor_obj = None
+        queryset._search_text = text
+
+        return queryset
+
     def get(self, *q_objs, **query):
         """Retrieve the the matching object raising
         :class:`~mongoengine.queryset.MultipleObjectsReturned` or
@@ -175,7 +228,7 @@ class BaseQuerySet(object):
         .. versionadded:: 0.3
         """
         queryset = self.clone()
-        queryset = queryset.limit(2)
+        queryset = queryset.order_by().limit(2)
         queryset = queryset.filter(*q_objs, **query)
 
         try:
@@ -215,7 +268,7 @@ class BaseQuerySet(object):
         .. note:: This requires two separate operations and therefore a
             race condition exists.  Because there are no transactions in
             mongoDB other approaches should be investigated, to ensure you
-            don't accidently duplicate data when using this method.  This is
+            don't accidentally duplicate data when using this method.  This is
             now scheduled to be removed before 1.0
 
         :param write_concern: optional extra keyword arguments used if we
@@ -303,10 +356,10 @@ class BaseQuerySet(object):
         try:
             ids = self._collection.insert(raw, **write_concern)
         except pymongo.errors.DuplicateKeyError, err:
-            message = 'Could not save document (%s)';
+            message = 'Could not save document (%s)'
             raise NotUniqueError(message % unicode(err))
         except pymongo.errors.OperationFailure, err:
-            message = 'Could not save document (%s)';
+            message = 'Could not save document (%s)'
             if re.match('^E1100[01] duplicate key', unicode(err)):
                 # E11000 - duplicate key error index
                 # E11001 - duplicate key on update
@@ -327,7 +380,7 @@ class BaseQuerySet(object):
             self._document, documents=results, loaded=True)
         return return_one and results[0] or results
 
-    def count(self, with_limit_and_skip=True):
+    def count(self, with_limit_and_skip=False):
         """Count the selected elements in the query.
 
         :param with_limit_and_skip (optional): take any :meth:`limit` or
@@ -349,6 +402,8 @@ class BaseQuerySet(object):
             will force an fsync on the primary server.
         :param _from_doc_delete: True when called from document delete therefore
             signals will have been triggered so don't loop.
+
+        :returns number of deleted documents
         """
         queryset = self.clone()
         doc = queryset._document
@@ -366,15 +421,19 @@ class BaseQuerySet(object):
                                 has_delete_signal) and not _from_doc_delete
 
         if call_document_delete:
+            cnt = 0
             for doc in queryset:
                 doc.delete(write_concern=write_concern)
-            return
+                cnt += 1
+            return cnt
 
         delete_rules = doc._meta.get('delete_rules') or {}
         # Check for DENY rules before actually deleting/nullifying any other
         # references
         for rule_entry in delete_rules:
             document_cls, field_name = rule_entry
+            if document_cls._meta.get('abstract'):
+                continue
             rule = doc._meta['delete_rules'][rule_entry]
             if rule == DENY and document_cls.objects(
                     **{field_name + '__in': self}).count() > 0:
@@ -384,12 +443,14 @@ class BaseQuerySet(object):
 
         for rule_entry in delete_rules:
             document_cls, field_name = rule_entry
+            if document_cls._meta.get('abstract'):
+                continue
             rule = doc._meta['delete_rules'][rule_entry]
             if rule == CASCADE:
                 ref_q = document_cls.objects(**{field_name + '__in': self})
                 ref_q_count = ref_q.count()
                 if (doc != document_cls and ref_q_count > 0
-                   or (doc == document_cls and ref_q_count > 0)):
+                        or (doc == document_cls and ref_q_count > 0)):
                     ref_q.delete(write_concern=write_concern)
             elif rule == NULLIFY:
                 document_cls.objects(**{field_name + '__in': self}).update(
@@ -399,7 +460,8 @@ class BaseQuerySet(object):
                     write_concern=write_concern,
                     **{'pull_all__%s' % field_name: self})
 
-        queryset._collection.remove(queryset._query, write_concern=write_concern)
+        result = queryset._collection.remove(queryset._query, **write_concern)
+        return result["n"]
 
     def update(self, upsert=False, multi=True, write_concern=None,
                full_result=False, **update):
@@ -443,6 +505,8 @@ class BaseQuerySet(object):
                 return result
             elif result:
                 return result['n']
+        except pymongo.errors.DuplicateKeyError, err:
+            raise NotUniqueError(u'Update failed (%s)' % unicode(err))
         except pymongo.errors.OperationFailure, err:
             if unicode(err) == u'multi not coded yet':
                 message = u'update() method requires MongoDB 1.1.3+'
@@ -466,6 +530,60 @@ class BaseQuerySet(object):
         return self.update(
             upsert=upsert, multi=False, write_concern=write_concern, **update)
 
+    def modify(self, upsert=False, full_response=False, remove=False, new=False, **update):
+        """Update and return the updated document.
+
+        Returns either the document before or after modification based on `new`
+        parameter. If no documents match the query and `upsert` is false,
+        returns ``None``. If upserting and `new` is false, returns ``None``.
+
+        If the full_response parameter is ``True``, the return value will be
+        the entire response object from the server, including the 'ok' and
+        'lastErrorObject' fields, rather than just the modified document.
+        This is useful mainly because the 'lastErrorObject' document holds
+        information about the command's execution.
+
+        :param upsert: insert if document doesn't exist (default ``False``)
+        :param full_response: return the entire response object from the
+            server (default ``False``)
+        :param remove: remove rather than updating (default ``False``)
+        :param new: return updated rather than original document
+            (default ``False``)
+        :param update: Django-style update keyword arguments
+
+        .. versionadded:: 0.9
+        """
+
+        if remove and new:
+            raise OperationError("Conflicting parameters: remove and new")
+
+        if not update and not upsert and not remove:
+            raise OperationError(
+                "No update parameters, must either update or remove")
+
+        queryset = self.clone()
+        query = queryset._query
+        update = transform.update(queryset._document, **update)
+        sort = queryset._ordering
+
+        try:
+            result = queryset._collection.find_and_modify(
+                query, update, upsert=upsert, sort=sort, remove=remove, new=new,
+                full_response=full_response, **self._cursor_args)
+        except pymongo.errors.DuplicateKeyError, err:
+            raise NotUniqueError(u"Update failed (%s)" % err)
+        except pymongo.errors.OperationFailure, err:
+            raise OperationError(u"Update failed (%s)" % err)
+
+        if full_response:
+            if result["value"] is not None:
+                result["value"] = self._document._from_son(result["value"], only_fields=self.only_fields)
+        else:
+            if result is not None:
+                result = self._document._from_son(result, only_fields=self.only_fields)
+
+        return result
+
     def with_id(self, object_id):
         """Retrieve the object matching the id provided.  Uses `object_id` only
         and raises InvalidQueryError if a filter has been applied. Returns
@@ -497,13 +615,15 @@ class BaseQuerySet(object):
         if self._scalar:
             for doc in docs:
                 doc_map[doc['_id']] = self._get_scalar(
-                    self._document._from_son(doc))
+                    self._document._from_son(doc, only_fields=self.only_fields))
         elif self._as_pymongo:
             for doc in docs:
                 doc_map[doc['_id']] = self._get_as_pymongo(doc)
         else:
             for doc in docs:
-                doc_map[doc['_id']] = self._document._from_son(doc)
+                doc_map[doc['_id']] = self._document._from_son(doc,
+                        only_fields=self.only_fields,
+                        _auto_dereference=self._auto_dereference)
 
         return doc_map
 
@@ -522,6 +642,19 @@ class BaseQuerySet(object):
 
         return self
 
+    def using(self, alias):
+        """This method is for controlling which database the QuerySet will be evaluated against if you are using more than one database.
+
+        :param alias: The database alias
+
+        .. versionadded:: 0.9
+        """
+
+        with switch_db(self._document, alias) as cls:
+            collection = cls._get_collection()
+
+        return self.clone_into(self.__class__(self._document, collection))
+
     def clone(self):
         """Creates a copy of the current
           :class:`~mongoengine.queryset.QuerySet`
@@ -535,13 +668,15 @@ class BaseQuerySet(object):
           :class:`~mongoengine.queryset.base.BaseQuerySet` into another child class
         """
         if not isinstance(cls, BaseQuerySet):
-            raise OperationError('%s is not a subclass of BaseQuerySet' % cls.__name__)
+            raise OperationError(
+                '%s is not a subclass of BaseQuerySet' % cls.__name__)
 
         copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj',
                       '_where_clause', '_loaded_fields', '_ordering', '_snapshot',
                       '_timeout', '_class_check', '_slave_okay', '_read_preference',
                       '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce',
-                      '_limit', '_skip', '_hint', '_auto_dereference')
+                      '_limit', '_skip', '_hint', '_auto_dereference',
+                      '_search_text', 'only_fields', '_max_time_ms')
 
         for prop in copy_props:
             val = getattr(self, prop)
@@ -627,10 +762,31 @@ class BaseQuerySet(object):
             distinct = self._dereference(queryset._cursor.distinct(field), 1,
                                          name=field, instance=self._document)
 
+            doc_field = self._document._fields.get(field.split('.', 1)[0])
+            instance = False
             # We may need to cast to the correct type eg. ListField(EmbeddedDocumentField)
-            doc_field = getattr(self._document._fields.get(field), "field", None)
-            instance = getattr(doc_field, "document_type", False)
-            if instance:
+            EmbeddedDocumentField = _import_class('EmbeddedDocumentField')
+            ListField = _import_class('ListField')
+            GenericEmbeddedDocumentField = _import_class('GenericEmbeddedDocumentField')
+            if isinstance(doc_field, ListField):
+                doc_field = getattr(doc_field, "field", doc_field)
+            if isinstance(doc_field, (EmbeddedDocumentField, GenericEmbeddedDocumentField)):
+                instance = getattr(doc_field, "document_type", False)
+            # handle distinct on subdocuments
+            if '.' in field:
+                for field_part in field.split('.')[1:]:
+                    # if looping on embedded document, get the document type instance
+                    if instance and isinstance(doc_field, (EmbeddedDocumentField, GenericEmbeddedDocumentField)):
+                        doc_field = instance
+                    # now get the subdocument
+                    doc_field = getattr(doc_field, field_part, doc_field)
+                    # We may need to cast to the correct type eg. ListField(EmbeddedDocumentField)
+                    if isinstance(doc_field, ListField):
+                        doc_field = getattr(doc_field, "field", doc_field)
+                    if isinstance(doc_field, (EmbeddedDocumentField, GenericEmbeddedDocumentField)):
+                        instance = getattr(doc_field, "document_type", False)
+            if instance and isinstance(doc_field, (EmbeddedDocumentField,
+                                                   GenericEmbeddedDocumentField)):
                 distinct = [instance(**doc) for doc in distinct]
             return distinct
 
@@ -653,6 +809,7 @@ class BaseQuerySet(object):
         .. versionchanged:: 0.5 - Added subfield support
         """
         fields = dict([(f, QueryFieldList.ONLY) for f in fields])
+        self.only_fields = fields.keys()
         return self.fields(True, **fields)
 
     def exclude(self, *fields):
@@ -709,7 +866,8 @@ class BaseQuerySet(object):
         for value, group in itertools.groupby(fields, lambda x: x[1]):
             fields = [field for field, value in group]
             fields = queryset._fields_to_dbfields(fields)
-            queryset._loaded_fields += QueryFieldList(fields, value=value, _only_called=_only_called)
+            queryset._loaded_fields += QueryFieldList(
+                fields, value=value, _only_called=_only_called)
 
         return queryset
 
@@ -830,6 +988,13 @@ class BaseQuerySet(object):
         queryset._as_pymongo_coerce = coerce_types
         return queryset
 
+    def max_time_ms(self, ms):
+        """Wait `ms` milliseconds before killing the query on the server
+
+        :param ms: the number of milliseconds before killing the query on the server
+        """
+        return self._chainable_method("max_time_ms", ms)
+
     # JSON Helpers
 
     def to_json(self, *args, **kwargs):
@@ -839,10 +1004,35 @@ class BaseQuerySet(object):
     def from_json(self, json_data):
         """Converts json data to unsaved objects"""
         son_data = json_util.loads(json_data)
-        return [self._document._from_son(data) for data in son_data]
+        return [self._document._from_son(data, only_fields=self.only_fields) for data in son_data]
+
+    def aggregate(self, *pipeline, **kwargs):
+        """
+        Perform a aggregate function based in your queryset params
+        :param pipeline: list of aggregation commands,\
+            see: http://docs.mongodb.org/manual/core/aggregation-pipeline/
+
+        .. versionadded:: 0.9
+        """
+        initial_pipeline = []
+
+        if self._query:
+            initial_pipeline.append({'$match': self._query})
+
+        if self._ordering:
+            initial_pipeline.append({'$sort': dict(self._ordering)})
+
+        if self._limit is not None:
+            initial_pipeline.append({'$limit': self._limit})
+
+        if self._skip is not None:
+            initial_pipeline.append({'$skip': self._skip})
+
+        pipeline = initial_pipeline + list(pipeline)
+
+        return self._collection.aggregate(pipeline, cursor={}, **kwargs)
 
     # JS functionality
-
     def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None,
                    scope=None):
         """Perform a map/reduce query using the current query spec
@@ -923,10 +1113,38 @@ class BaseQuerySet(object):
             map_reduce_function = 'inline_map_reduce'
         else:
             map_reduce_function = 'map_reduce'
-            mr_args['out'] = output
+
+            if isinstance(output, basestring):
+                mr_args['out'] = output
+
+            elif isinstance(output, dict):
+                ordered_output = []
+
+                for part in ('replace', 'merge', 'reduce'):
+                    value = output.get(part)
+                    if value:
+                        ordered_output.append((part, value))
+                        break
+
+                else:
+                    raise OperationError("actionData not specified for output")
+
+                db_alias = output.get('db_alias')
+                remaing_args = ['db', 'sharded', 'nonAtomic']
+
+                if db_alias:
+                    ordered_output.append(('db', get_db(db_alias).name))
+                    del remaing_args[0]
+
+                for part in remaing_args:
+                    value = output.get(part)
+                    if value:
+                        ordered_output.append((part, value))
+
+                mr_args['out'] = SON(ordered_output)
 
         results = getattr(queryset._collection, map_reduce_function)(
-                          map_f, reduce_f, **mr_args)
+            map_f, reduce_f, **mr_args)
 
         if map_reduce_function == 'map_reduce':
             results = results.find()
@@ -1138,7 +1356,8 @@ class BaseQuerySet(object):
         if self._as_pymongo:
             return self._get_as_pymongo(raw_doc)
         doc = self._document._from_son(raw_doc,
-                                       _auto_dereference=self._auto_dereference)
+                                       _auto_dereference=self._auto_dereference, only_fields=self.only_fields)
+
         if self._scalar:
             return self._get_scalar(doc)
 
@@ -1147,6 +1366,7 @@ class BaseQuerySet(object):
     def rewind(self):
         """Rewind the cursor to its unevaluated state.
 
+
         .. versionadded:: 0.3
         """
         self._iter = False
@@ -1173,6 +1393,13 @@ class BaseQuerySet(object):
             cursor_args['slave_okay'] = self._slave_okay
         if self._loaded_fields:
             cursor_args['fields'] = self._loaded_fields.as_dict()
+
+        if self._search_text:
+            if 'fields' not in cursor_args:
+                cursor_args['fields'] = {}
+
+            cursor_args['fields']['_text_score'] = {'$meta': "textScore"}
+
         return cursor_args
 
     @property
@@ -1189,8 +1416,9 @@ class BaseQuerySet(object):
             if self._ordering:
                 # Apply query ordering
                 self._cursor_obj.sort(self._ordering)
-            elif self._document._meta['ordering']:
-                # Otherwise, apply the ordering from the document model
+            elif self._ordering is None and self._document._meta['ordering']:
+                # Otherwise, apply the ordering from the document model, unless
+                # it's been explicitly cleared via order_by with no arguments
                 order = self._get_order_by(self._document._meta['ordering'])
                 self._cursor_obj.sort(order)
 
@@ -1213,8 +1441,11 @@ class BaseQuerySet(object):
     def _query(self):
         if self._mongo_query is None:
             self._mongo_query = self._query_obj.to_query(self._document)
-            if self._class_check:
-                self._mongo_query.update(self._initial_query)
+            if self._class_check and self._initial_query:
+                if "_cls" in self._mongo_query:
+                    self._mongo_query = {"$and": [self._initial_query, self._mongo_query]}
+                else:
+                    self._mongo_query.update(self._initial_query)
         return self._mongo_query
 
     @property
@@ -1362,7 +1593,7 @@ class BaseQuerySet(object):
                 for subdoc in subclasses:
                     try:
                         subfield = ".".join(f.db_field for f in
-                                        subdoc._lookup_field(field.split('.')))
+                                            subdoc._lookup_field(field.split('.')))
                         ret.append(subfield)
                         found = True
                         break
@@ -1380,6 +1611,11 @@ class BaseQuerySet(object):
         for key in keys:
             if not key:
                 continue
+
+            if key == '$text_score':
+                key_list.append(('_text_score', {'$meta': "textScore"}))
+                continue
+
             direction = pymongo.ASCENDING
             if key[0] == '-':
                 direction = pymongo.DESCENDING
@@ -1392,7 +1628,7 @@ class BaseQuerySet(object):
                 pass
             key_list.append((key, direction))
 
-        if self._cursor_obj:
+        if self._cursor_obj and key_list:
             self._cursor_obj.sort(key_list)
         return key_list
 
@@ -1450,6 +1686,7 @@ class BaseQuerySet(object):
                     # type of this field and use the corresponding
                     # .to_python(...)
                     from mongoengine.fields import EmbeddedDocumentField
+
                     obj = self._document
                     for chunk in path.split('.'):
                         obj = getattr(obj, chunk, None)
@@ -1460,6 +1697,7 @@ class BaseQuerySet(object):
                     if obj and data is not None:
                         data = obj.to_python(data)
             return data
+
         return clean(row)
 
     def _sub_js_fields(self, code):
@@ -1468,6 +1706,7 @@ class BaseQuerySet(object):
         substituted for the MongoDB name of the field (specified using the
         :attr:`name` keyword argument in a field's constructor).
         """
+
         def field_sub(match):
             # Extract just the field name, and look up the field objects
             field_name = match.group(1).split('.')
@@ -1487,6 +1726,13 @@ class BaseQuerySet(object):
                       code)
         return code
 
+    def _chainable_method(self, method_name, val):
+        queryset = self.clone()
+        method = getattr(queryset._cursor, method_name)
+        method(val)
+        setattr(queryset, "_" + method_name, val)
+        return queryset
+
     # Deprecated
     def ensure_index(self, **kwargs):
         """Deprecated use :func:`Document.ensure_index`"""

mongoengine/queryset/queryset.py

@@ -94,7 +94,7 @@ class QuerySet(BaseQuerySet):
             except StopIteration:
                 self._has_more = False
 
-    def count(self, with_limit_and_skip=True):
+    def count(self, with_limit_and_skip=False):
         """Count the selected elements in the query.
 
         :param with_limit_and_skip (optional): take any :meth:`limit` or
@@ -155,3 +155,10 @@ class QuerySetNoCache(BaseQuerySet):
             queryset = self.clone()
         queryset.rewind()
         return queryset
+
+
+class QuerySetNoDeRef(QuerySet):
+    """Special no_dereference QuerySet"""
+
+    def __dereference(items, max_depth=1, instance=None, name=None):
+        return items

mongoengine/queryset/transform.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 import pymongo
 from bson import SON
 
+from mongoengine.connection import get_connection
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidQueryError, LookUpError
 
@@ -10,22 +11,22 @@ __all__ = ('query', 'update')
 
 
 COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod',
-                        'all', 'size', 'exists', 'not')
-GEO_OPERATORS        = ('within_distance', 'within_spherical_distance',
-                        'within_box', 'within_polygon', 'near', 'near_sphere',
-                        'max_distance', 'geo_within', 'geo_within_box',
-                        'geo_within_polygon', 'geo_within_center',
-                        'geo_within_sphere', 'geo_intersects')
-STRING_OPERATORS     = ('contains', 'icontains', 'startswith',
-                        'istartswith', 'endswith', 'iendswith',
-                        'exact', 'iexact')
-CUSTOM_OPERATORS     = ('match',)
-MATCH_OPERATORS      = (COMPARISON_OPERATORS + GEO_OPERATORS +
-                        STRING_OPERATORS + CUSTOM_OPERATORS)
+                        'all', 'size', 'exists', 'not', 'elemMatch', 'type')
+GEO_OPERATORS = ('within_distance', 'within_spherical_distance',
+                 'within_box', 'within_polygon', 'near', 'near_sphere',
+                 'max_distance', 'geo_within', 'geo_within_box',
+                 'geo_within_polygon', 'geo_within_center',
+                 'geo_within_sphere', 'geo_intersects')
+STRING_OPERATORS = ('contains', 'icontains', 'startswith',
+                    'istartswith', 'endswith', 'iendswith',
+                    'exact', 'iexact')
+CUSTOM_OPERATORS = ('match',)
+MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS +
+                   STRING_OPERATORS + CUSTOM_OPERATORS)
 
-UPDATE_OPERATORS     = ('set', 'unset', 'inc', 'dec', 'pop', 'push',
-                        'push_all', 'pull', 'pull_all', 'add_to_set',
-                        'set_on_insert')
+UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push',
+                    'push_all', 'pull', 'pull_all', 'add_to_set',
+                    'set_on_insert', 'min', 'max')
 
 
 def query(_doc_cls=None, _field_operation=False, **query):
@@ -38,7 +39,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
             mongo_query.update(value)
             continue
 
-        parts = key.split('__')
+        parts = key.rsplit('__')
         indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
         parts = [part for part in parts if not part.isdigit()]
         # Check for an operator and transform to mongo-style if there is
@@ -59,14 +60,20 @@ def query(_doc_cls=None, _field_operation=False, **query):
                 raise InvalidQueryError(e)
             parts = []
 
+            CachedReferenceField = _import_class('CachedReferenceField')
+
             cleaned_fields = []
             for field in fields:
                 append_field = True
                 if isinstance(field, basestring):
                     parts.append(field)
                     append_field = False
+                # is last and CachedReferenceField
+                elif isinstance(field, CachedReferenceField) and fields[-1] == field:
+                    parts.append('%s._id' % field.db_field)
                 else:
                     parts.append(field.db_field)
+
                 if append_field:
                     cleaned_fields.append(field)
 
@@ -78,13 +85,17 @@ def query(_doc_cls=None, _field_operation=False, **query):
             if op in singular_ops:
                 if isinstance(field, basestring):
                     if (op in STRING_OPERATORS and
-                       isinstance(value, basestring)):
+                            isinstance(value, basestring)):
                         StringField = _import_class('StringField')
                         value = StringField.prepare_query_value(op, value)
                     else:
                         value = field
                 else:
                     value = field.prepare_query_value(op, value)
+
+                    if isinstance(field, CachedReferenceField) and value:
+                        value = value['_id']
+
             elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
                 # 'in', 'nin' and 'all' require a list of values
                 value = [field.prepare_query_value(op, v) for v in value]
@@ -94,7 +105,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
             if op in GEO_OPERATORS:
                 value = _geo_operator(field, op, value)
             elif op in CUSTOM_OPERATORS:
-                if op == 'match':
+                if op in ('elem_match', 'match'):
                     value = field.prepare_query_value(op, value)
                     value = {"$elemMatch": value}
                 else:
@@ -115,14 +126,28 @@ def query(_doc_cls=None, _field_operation=False, **query):
             if key in mongo_query and isinstance(mongo_query[key], dict):
                 mongo_query[key].update(value)
                 # $maxDistance needs to come last - convert to SON
-                if '$maxDistance' in mongo_query[key]:
-                    value_dict = mongo_query[key]
+                value_dict = mongo_query[key]
+                if ('$maxDistance' in value_dict and '$near' in value_dict):
                     value_son = SON()
-                    for k, v in value_dict.iteritems():
-                        if k == '$maxDistance':
-                            continue
-                        value_son[k] = v
-                    value_son['$maxDistance'] = value_dict['$maxDistance']
+                    if isinstance(value_dict['$near'], dict):
+                        for k, v in value_dict.iteritems():
+                            if k == '$maxDistance':
+                                continue
+                            value_son[k] = v
+                        if (get_connection().max_wire_version <= 1):
+                            value_son['$maxDistance'] = value_dict[
+                                '$maxDistance']
+                        else:
+                            value_son['$near'] = SON(value_son['$near'])
+                            value_son['$near'][
+                                '$maxDistance'] = value_dict['$maxDistance']
+                    else:
+                        for k, v in value_dict.iteritems():
+                            if k == '$maxDistance':
+                                continue
+                            value_son[k] = v
+                        value_son['$maxDistance'] = value_dict['$maxDistance']
+
                     mongo_query[key] = value_son
             else:
                 # Store for manually merging later
@@ -135,7 +160,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
         if isinstance(v, list):
             value = [{k: val} for val in v]
             if '$and' in mongo_query.keys():
-                mongo_query['$and'].append(value)
+                mongo_query['$and'].extend(value)
             else:
                 mongo_query['$and'] = value
 
@@ -151,6 +176,9 @@ def update(_doc_cls=None, **update):
             mongo_update.update(value)
             continue
         parts = key.split('__')
+        # if there is no operator, default to "set"
+        if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS:
+            parts.insert(0, 'set')
         # Check for an operator and transform to mongo-style if there is
         op = None
         if parts[0] in UPDATE_OPERATORS:
@@ -248,7 +276,8 @@ def update(_doc_cls=None, **update):
             if ListField in field_classes:
                 # Join all fields via dot notation to the last ListField
                 # Then process as normal
-                last_listField = len(cleaned_fields) - field_classes.index(ListField)
+                last_listField = len(
+                    cleaned_fields) - field_classes.index(ListField)
                 key = ".".join(parts[:last_listField])
                 parts = parts[last_listField:]
                 parts.insert(0, key)

mongoengine/queryset/visitor.py

@@ -1,8 +1,9 @@
 import copy
 
-from mongoengine.errors import InvalidQueryError
-from mongoengine.python_support import product, reduce
+from itertools import product
+from functools import reduce
 
+from mongoengine.errors import InvalidQueryError
 from mongoengine.queryset import transform
 
 __all__ = ('Q',)
@@ -28,7 +29,7 @@ class DuplicateQueryConditionsError(InvalidQueryError):
 
 
 class SimplificationVisitor(QNodeVisitor):
-    """Simplifies query trees by combinging unnecessary 'and' connection nodes
+    """Simplifies query trees by combining unnecessary 'and' connection nodes
     into a single Q-object.
     """
 

requirements.txt

@@ -1 +1,2 @@
-pymongo
+pymongo>=2.7.1
+nose

setup.py

@@ -41,9 +41,11 @@ CLASSIFIERS = [
     "Programming Language :: Python :: 2.6",
     "Programming Language :: Python :: 2.7",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.1",
     "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
     'Topic :: Database',
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
@@ -51,12 +53,15 @@ CLASSIFIERS = [
 extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])}
 if sys.version_info[0] == 3:
     extra_opts['use_2to3'] = True
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'Pillow>=2.0.0', 'django>=1.5.1']
     if "test" in sys.argv or "nosetests" in sys.argv:
         extra_opts['packages'] = find_packages()
         extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'Pillow>=2.0.0', 'jinja2>=2.6', 'python-dateutil']
+
+    if sys.version_info[0] == 2 and sys.version_info[1] == 6:
+        extra_opts['tests_require'].append('unittest2')
 
 setup(name='mongoengine',
       version=VERSION,
@@ -72,7 +77,7 @@ setup(name='mongoengine',
       long_description=LONG_DESCRIPTION,
       platforms=['any'],
       classifiers=CLASSIFIERS,
-      install_requires=['pymongo>=2.5'],
+      install_requires=['pymongo>=2.7.1'],
       test_suite='nose.collector',
       **extra_opts
 )

tests/document/class_methods.py

@@ -36,9 +36,9 @@ class ClassMethodsTest(unittest.TestCase):
     def test_definition(self):
         """Ensure that document may be defined using fields.
         """
-        self.assertEqual(['age', 'id', 'name'],
+        self.assertEqual(['_cls', 'age', 'id', 'name'],
                          sorted(self.Person._fields.keys()))
-        self.assertEqual(["IntField", "ObjectIdField", "StringField"],
+        self.assertEqual(["IntField", "ObjectIdField", "StringField", "StringField"],
                         sorted([x.__class__.__name__ for x in
                                 self.Person._fields.values()]))
 

tests/document/delta.py

@@ -207,22 +207,21 @@ class DeltaTest(unittest.TestCase):
         doc.embedded_field.list_field[2].string_field = 'hello world'
         doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2]
         self.assertEqual(doc._get_changed_fields(),
-                         ['embedded_field.list_field'])
-        self.assertEqual(doc.embedded_field._delta(), ({
-            'list_field': ['1', 2, {
-            '_cls': 'Embedded',
-            'string_field': 'hello world',
-            'int_field': 1,
-            'list_field': ['1', 2, {'hello': 'world'}],
-            'dict_field': {'hello': 'world'}}]}, {}))
-        self.assertEqual(doc._delta(), ({
-            'embedded_field.list_field': ['1', 2, {
+                         ['embedded_field.list_field.2'])
+        self.assertEqual(doc.embedded_field._delta(), ({'list_field.2': {
                 '_cls': 'Embedded',
                 'string_field': 'hello world',
                 'int_field': 1,
                 'list_field': ['1', 2, {'hello': 'world'}],
                 'dict_field': {'hello': 'world'}}
-            ]}, {}))
+            }, {}))
+        self.assertEqual(doc._delta(), ({'embedded_field.list_field.2': {
+                '_cls': 'Embedded',
+                'string_field': 'hello world',
+                'int_field': 1,
+                'list_field': ['1', 2, {'hello': 'world'}],
+                'dict_field': {'hello': 'world'}}
+            }, {}))
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].string_field,
@@ -253,7 +252,7 @@ class DeltaTest(unittest.TestCase):
 
         del(doc.embedded_field.list_field[2].list_field[2]['hello'])
         self.assertEqual(doc._delta(),
-                         ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {}))
+                         ({}, {'embedded_field.list_field.2.list_field.2.hello': 1}))
         doc.save()
         doc = doc.reload(10)
 
@@ -548,22 +547,21 @@ class DeltaTest(unittest.TestCase):
         doc.embedded_field.list_field[2].string_field = 'hello world'
         doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2]
         self.assertEqual(doc._get_changed_fields(),
-            ['db_embedded_field.db_list_field'])
-        self.assertEqual(doc.embedded_field._delta(), ({
-            'db_list_field': ['1', 2, {
+            ['db_embedded_field.db_list_field.2'])
+        self.assertEqual(doc.embedded_field._delta(), ({'db_list_field.2': {
             '_cls': 'Embedded',
             'db_string_field': 'hello world',
             'db_int_field': 1,
             'db_list_field': ['1', 2, {'hello': 'world'}],
-            'db_dict_field': {'hello': 'world'}}]}, {}))
+            'db_dict_field': {'hello': 'world'}}}, {}))
         self.assertEqual(doc._delta(), ({
-            'db_embedded_field.db_list_field': ['1', 2, {
+            'db_embedded_field.db_list_field.2': {
                 '_cls': 'Embedded',
                 'db_string_field': 'hello world',
                 'db_int_field': 1,
                 'db_list_field': ['1', 2, {'hello': 'world'}],
                 'db_dict_field': {'hello': 'world'}}
-            ]}, {}))
+            }, {}))
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].string_field,
@@ -594,8 +592,7 @@ class DeltaTest(unittest.TestCase):
 
         del(doc.embedded_field.list_field[2].list_field[2]['hello'])
         self.assertEqual(doc._delta(),
-            ({'db_embedded_field.db_list_field.2.db_list_field':
-                [1, 2, {}]}, {}))
+            ({}, {'db_embedded_field.db_list_field.2.db_list_field.2.hello': 1}))
         doc.save()
         doc = doc.reload(10)
 
@@ -735,5 +732,47 @@ class DeltaTest(unittest.TestCase):
         mydoc._clear_changed_fields()
         self.assertEqual([], mydoc._get_changed_fields())
 
+    def test_referenced_object_changed_attributes(self):
+        """Ensures that when you save a new reference to a field, the referenced object isn't altered"""
+
+        class Organization(Document):
+            name = StringField()
+
+        class User(Document):
+            name = StringField()
+            org = ReferenceField('Organization', required=True)
+
+        Organization.drop_collection()
+        User.drop_collection()
+
+        org1 = Organization(name='Org 1')
+        org1.save()
+
+        org2 = Organization(name='Org 2')
+        org2.save()
+
+        user = User(name='Fred', org=org1)
+        user.save()
+
+        org1.reload()
+        org2.reload()
+        user.reload()
+        self.assertEqual(org1.name, 'Org 1')
+        self.assertEqual(org2.name, 'Org 2')
+        self.assertEqual(user.name, 'Fred')
+
+        user.name = 'Harold'
+        user.org = org2
+
+        org2.name = 'New Org 2'
+        self.assertEqual(org2.name, 'New Org 2')
+
+        user.save()
+        org2.save()
+
+        self.assertEqual(org2.name, 'New Org 2')
+        org2.reload()
+        self.assertEqual(org2.name, 'New Org 2')
+
 if __name__ == '__main__':
     unittest.main()

tests/document/dynamic.py

@@ -81,6 +81,13 @@ class DynamicTest(unittest.TestCase):
         obj = collection.find_one()
         self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'name'])
 
+    def test_reload_after_unsetting(self):
+        p = self.Person()
+        p.misc = 22
+        p.save()
+        p.update(unset__misc=1)
+        p.reload()
+
     def test_dynamic_document_queries(self):
         """Ensure we can query dynamic fields"""
         p = self.Person()
@@ -292,6 +299,59 @@ class DynamicTest(unittest.TestCase):
         person.save()
         self.assertEqual(Person.objects.first().age, 35)
 
+    def test_dynamic_embedded_works_with_only(self):
+        """Ensure custom fieldnames on a dynamic embedded document are found by qs.only()"""
+
+        class Address(DynamicEmbeddedDocument):
+            city = StringField()
+
+        class Person(DynamicDocument):
+            address = EmbeddedDocumentField(Address)
+
+        Person.drop_collection()
+
+        Person(name="Eric", address=Address(city="San Francisco", street_number="1337")).save()
+
+        self.assertEqual(Person.objects.first().address.street_number, '1337')
+        self.assertEqual(Person.objects.only('address__street_number').first().address.street_number, '1337')
+
+    def test_dynamic_and_embedded_dict_access(self):
+        """Ensure embedded dynamic documents work with dict[] style access"""
+
+        class Address(EmbeddedDocument):
+            city = StringField()
+
+        class Person(DynamicDocument):
+            name = StringField()
+
+        Person.drop_collection()
+
+        Person(name="Ross", address=Address(city="London")).save()
+
+        person = Person.objects.first()
+        person.attrval = "This works"
+
+        person["phone"] = "555-1212" # but this should too
+
+        # Same thing two levels deep
+        person["address"]["city"] = "Lundenne"
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Lundenne")
+
+        self.assertEqual(Person.objects.first().phone, "555-1212")
+
+        person = Person.objects.first()
+        person.address = Address(city="Londinium")
+        person.save()
+
+        self.assertEqual(Person.objects.first().address.city, "Londinium")
+
+
+        person = Person.objects.first()
+        person["age"] = 35
+        person.save()
+        self.assertEqual(Person.objects.first().age, 35)
 
 if __name__ == '__main__':
     unittest.main()

tests/document/indexes.py

@@ -18,7 +18,7 @@ __all__ = ("IndexesTest", )
 class IndexesTest(unittest.TestCase):
 
     def setUp(self):
-        connect(db='mongoenginetest')
+        self.connection = connect(db='mongoenginetest')
         self.db = get_db()
 
         class Person(Document):
@@ -175,6 +175,16 @@ class IndexesTest(unittest.TestCase):
         info = A._get_collection().index_information()
         self.assertEqual(len(info.keys()), 2)
 
+        class B(A):
+            c = StringField()
+            d = StringField()
+            meta = {
+                'indexes': [{'fields': ['c']}, {'fields': ['d'], 'cls': True}],
+                'allow_inheritance': True
+            }
+        self.assertEqual([('c', 1)], B._meta['index_specs'][1]['fields'])
+        self.assertEqual([('_cls', 1), ('d', 1)], B._meta['index_specs'][2]['fields'])
+
     def test_build_index_spec_is_not_destructive(self):
 
         class MyDoc(Document):
@@ -485,9 +495,12 @@ class IndexesTest(unittest.TestCase):
 
         self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10)
 
-        def invalid_index():
-            BlogPost.objects.hint('tags')
-        self.assertRaises(TypeError, invalid_index)
+        if pymongo.version >= '2.8':
+            self.assertEqual(BlogPost.objects.hint('tags').count(), 10)
+        else:
+            def invalid_index():
+                BlogPost.objects.hint('tags')
+            self.assertRaises(TypeError, invalid_index)
 
         def invalid_index_2():
             return BlogPost.objects.hint(('tags', 1))
@@ -567,6 +580,38 @@ class IndexesTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_unique_embedded_document_in_list(self):
+        """
+        Ensure that the uniqueness constraints are applied to fields in
+        embedded documents, even when the embedded documents in in a
+        list field.
+        """
+        class SubDocument(EmbeddedDocument):
+            year = IntField(db_field='yr')
+            slug = StringField(unique=True)
+
+        class BlogPost(Document):
+            title = StringField()
+            subs = ListField(EmbeddedDocumentField(SubDocument))
+
+        BlogPost.drop_collection()
+
+        post1 = BlogPost(
+            title='test1', subs=[
+                SubDocument(year=2009, slug='conflict'),
+                SubDocument(year=2009, slug='conflict')
+            ]
+        )
+        post1.save()
+
+        post2 = BlogPost(
+            title='test2', subs=[SubDocument(year=2014, slug='conflict')]
+        )
+
+        self.assertRaises(NotUniqueError, post2.save)
+
+        BlogPost.drop_collection()
+
     def test_unique_with_embedded_document_and_embedded_unique(self):
         """Ensure that uniqueness constraints are applied to fields on
         embedded documents.  And work with unique_with as well.
@@ -727,5 +772,59 @@ class IndexesTest(unittest.TestCase):
                          report.to_mongo())
         self.assertEqual(report, Report.objects.get(pk=my_key))
 
+    def test_string_indexes(self):
+
+        class MyDoc(Document):
+            provider_ids = DictField()
+            meta = {
+                "indexes": ["provider_ids.foo", "provider_ids.bar"],
+            }
+
+        info = MyDoc.objects._collection.index_information()
+        info = [value['key'] for key, value in info.iteritems()]
+        self.assertTrue([('provider_ids.foo', 1)] in info)
+        self.assertTrue([('provider_ids.bar', 1)] in info)
+
+    def test_text_indexes(self):
+
+        class Book(Document):
+            title = DictField()
+            meta = {
+                "indexes": ["$title"],
+            }
+
+        indexes = Book.objects._collection.index_information()
+        self.assertTrue("title_text" in indexes)
+        key = indexes["title_text"]["key"]
+        self.assertTrue(('_fts', 'text') in key)
+
+    def test_indexes_after_database_drop(self):
+        """
+        Test to ensure that indexes are re-created on a collection even
+        after the database has been dropped.
+
+        Issue #812
+        """
+        class BlogPost(Document):
+            title = StringField()
+            slug = StringField(unique=True)
+
+        BlogPost.drop_collection()
+
+        # Create Post #1
+        post1 = BlogPost(title='test1', slug='test')
+        post1.save()
+
+        # Drop the Database
+        self.connection.drop_database(BlogPost._get_db().name)
+
+        # Re-create Post #1
+        post1 = BlogPost(title='test1', slug='test')
+        post1.save()
+
+        # Create Post #2
+        post2 = BlogPost(title='test2', slug='test')
+        self.assertRaises(NotUniqueError, post2.save)
+
 if __name__ == '__main__':
     unittest.main()

tests/document/inheritance.py

@@ -163,7 +163,7 @@ class InheritanceTest(unittest.TestCase):
         class Employee(Person):
             salary = IntField()
 
-        self.assertEqual(['age', 'id', 'name', 'salary'],
+        self.assertEqual(['_cls', 'age', 'id', 'name', 'salary'],
                          sorted(Employee._fields.keys()))
         self.assertEqual(Employee._get_collection_name(),
                          Person._get_collection_name())
@@ -180,7 +180,7 @@ class InheritanceTest(unittest.TestCase):
         class Employee(Person):
             salary = IntField()
 
-        self.assertEqual(['age', 'id', 'name', 'salary'],
+        self.assertEqual(['_cls', 'age', 'id', 'name', 'salary'],
                          sorted(Employee._fields.keys()))
         self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(),
                          ['_cls', 'name', 'age'])
@@ -397,6 +397,16 @@ class InheritanceTest(unittest.TestCase):
                 meta = {'abstract': True}
         self.assertRaises(ValueError, create_bad_abstract)
 
+    def test_abstract_embedded_documents(self):
+        # 789: EmbeddedDocument shouldn't inherit abstract
+        class A(EmbeddedDocument):
+            meta = {"abstract": True}
+
+        class B(A):
+            pass
+
+        self.assertFalse(B._meta["abstract"])
+
     def test_inherited_collections(self):
         """Ensure that subclassed documents don't override parents'
         collections

tests/document/instance.py

@@ -9,13 +9,13 @@ import unittest
 import uuid
 
 from datetime import datetime
-from bson import DBRef
+from bson import DBRef, ObjectId
 from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest,
                             PickleDyanmicEmbedded, PickleDynamicTest)
 
 from mongoengine import *
 from mongoengine.errors import (NotRegistered, InvalidDocumentError,
-                                InvalidQueryError)
+                                InvalidQueryError, NotUniqueError)
 from mongoengine.queryset import NULLIFY, Q
 from mongoengine.connection import get_db
 from mongoengine.base import get_document
@@ -34,15 +34,21 @@ class InstanceTest(unittest.TestCase):
         connect(db='mongoenginetest')
         self.db = get_db()
 
+        class Job(EmbeddedDocument):
+            name = StringField()
+            years = IntField()
+
         class Person(Document):
             name = StringField()
             age = IntField()
+            job = EmbeddedDocumentField(Job)
 
             non_field = True
 
             meta = {"allow_inheritance": True}
 
         self.Person = Person
+        self.Job = Job
 
     def tearDown(self):
         for collection in self.db.collection_names():
@@ -50,6 +56,11 @@ class InstanceTest(unittest.TestCase):
                 continue
             self.db.drop_collection(collection)
 
+    def assertDbEqual(self, docs):
+        self.assertEqual(
+            list(self.Person._get_collection().find().sort("id")),
+            sorted(docs, key=lambda doc: doc["_id"]))
+
     def test_capped_collection(self):
         """Ensure that capped collections work properly.
         """
@@ -57,7 +68,7 @@ class InstanceTest(unittest.TestCase):
             date = DateTimeField(default=datetime.now)
             meta = {
                 'max_documents': 10,
-                'max_size': 90000,
+                'max_size': 4096,
             }
 
         Log.drop_collection()
@@ -75,7 +86,7 @@ class InstanceTest(unittest.TestCase):
         options = Log.objects._collection.options()
         self.assertEqual(options['capped'], True)
         self.assertEqual(options['max'], 10)
-        self.assertEqual(options['size'], 90000)
+        self.assertTrue(options['size'] >= 4096)
 
         # Check that the document cannot be redefined with different options
         def recreate_log_document():
@@ -103,6 +114,19 @@ class InstanceTest(unittest.TestCase):
 
         self.assertEqual('<Article: привет мир>', repr(doc))
 
+    def test_repr_none(self):
+        """Ensure None values handled correctly
+        """
+        class Article(Document):
+            title = StringField()
+
+            def __str__(self):
+                return None
+
+        doc = Article(title=u'привет мир')
+
+        self.assertEqual('<Article: None>', repr(doc))
+
     def test_queryset_resurrects_dropped_collection(self):
         self.Person.drop_collection()
 
@@ -122,10 +146,18 @@ class InstanceTest(unittest.TestCase):
         """
         class Animal(Document):
             meta = {'allow_inheritance': True}
-        class Fish(Animal): pass
-        class Mammal(Animal): pass
-        class Dog(Mammal): pass
-        class Human(Mammal): pass
+
+        class Fish(Animal):
+            pass
+
+        class Mammal(Animal):
+            pass
+
+        class Dog(Mammal):
+            pass
+
+        class Human(Mammal):
+            pass
 
         class Zoo(Document):
             animals = ListField(ReferenceField(Animal))
@@ -353,6 +385,14 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 20)
 
+        person.reload('age')
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 21)
+
+        person.reload()
+        self.assertEqual(person.name, "Mr Test User")
+        self.assertEqual(person.age, 21)
+
         person.reload()
         self.assertEqual(person.name, "Mr Test User")
         self.assertEqual(person.age, 21)
@@ -398,10 +438,11 @@ class InstanceTest(unittest.TestCase):
         doc.embedded_field.dict_field['woot'] = "woot"
 
         self.assertEqual(doc._get_changed_fields(), [
-            'list_field', 'dict_field', 'embedded_field.list_field',
-            'embedded_field.dict_field'])
+            'list_field', 'dict_field.woot', 'embedded_field.list_field',
+            'embedded_field.dict_field.woot'])
         doc.save()
 
+        self.assertEqual(len(doc.list_field), 4)
         doc = doc.reload(10)
         self.assertEqual(doc._get_changed_fields(), [])
         self.assertEqual(len(doc.list_field), 4)
@@ -409,6 +450,16 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(len(doc.embedded_field.list_field), 4)
         self.assertEqual(len(doc.embedded_field.dict_field), 2)
 
+        doc.list_field.append(1)
+        doc.save()
+        doc.dict_field['extra'] = 1
+        doc = doc.reload(10, 'list_field')
+        self.assertEqual(doc._get_changed_fields(), [])
+        self.assertEqual(len(doc.list_field), 5)
+        self.assertEqual(len(doc.dict_field), 3)
+        self.assertEqual(len(doc.embedded_field.list_field), 4)
+        self.assertEqual(len(doc.embedded_field.dict_field), 2)
+
     def test_reload_doesnt_exist(self):
         class Foo(Document):
             pass
@@ -418,7 +469,7 @@ class InstanceTest(unittest.TestCase):
             f.reload()
         except Foo.DoesNotExist:
             pass
-        except Exception as ex:
+        except Exception:
             self.assertFalse("Threw wrong exception")
 
         f.save()
@@ -427,13 +478,13 @@ class InstanceTest(unittest.TestCase):
             f.reload()
         except Foo.DoesNotExist:
             pass
-        except Exception as ex:
+        except Exception:
             self.assertFalse("Threw wrong exception")
 
     def test_dictionary_access(self):
         """Ensure that dictionary-style field access works properly.
         """
-        person = self.Person(name='Test User', age=30)
+        person = self.Person(name='Test User', age=30, job=self.Job())
         self.assertEqual(person['name'], 'Test User')
 
         self.assertRaises(KeyError, person.__getitem__, 'salary')
@@ -443,7 +494,7 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(person['name'], 'Another User')
 
         # Length = length(assigned fields + id)
-        self.assertEqual(len(person), 3)
+        self.assertEqual(len(person), 5)
 
         self.assertTrue('age' in person)
         person.age = None
@@ -462,8 +513,9 @@ class InstanceTest(unittest.TestCase):
 
         self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(),
                          ['_cls', 'name', 'age'])
-        self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(),
-                         ['_cls', 'name', 'age', 'salary'])
+        self.assertEqual(
+            Employee(name="Bob", age=35, salary=0).to_mongo().keys(),
+            ['_cls', 'name', 'age', 'salary'])
 
     def test_embedded_document_to_mongo_id(self):
         class SubDoc(EmbeddedDocument):
@@ -515,9 +567,6 @@ class InstanceTest(unittest.TestCase):
 
         class Email(EmbeddedDocument):
             email = EmailField()
-            def clean(self):
-                print "instance:"
-                print self._instance
 
         class Account(Document):
             email = EmbeddedDocumentField(Email)
@@ -601,6 +650,64 @@ class InstanceTest(unittest.TestCase):
         t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5))
         t.save(clean=False)
 
+    def test_modify_empty(self):
+        doc = self.Person(name="bob", age=10).save()
+        self.assertRaises(
+            InvalidDocumentError, lambda: self.Person().modify(set__age=10))
+        self.assertDbEqual([dict(doc.to_mongo())])
+
+    def test_modify_invalid_query(self):
+        doc1 = self.Person(name="bob", age=10).save()
+        doc2 = self.Person(name="jim", age=20).save()
+        docs = [dict(doc1.to_mongo()), dict(doc2.to_mongo())]
+
+        self.assertRaises(
+            InvalidQueryError,
+            lambda: doc1.modify(dict(id=doc2.id), set__value=20))
+
+        self.assertDbEqual(docs)
+
+    def test_modify_match_another_document(self):
+        doc1 = self.Person(name="bob", age=10).save()
+        doc2 = self.Person(name="jim", age=20).save()
+        docs = [dict(doc1.to_mongo()), dict(doc2.to_mongo())]
+
+        assert not doc1.modify(dict(name=doc2.name), set__age=100)
+
+        self.assertDbEqual(docs)
+
+    def test_modify_not_exists(self):
+        doc1 = self.Person(name="bob", age=10).save()
+        doc2 = self.Person(id=ObjectId(), name="jim", age=20)
+        docs = [dict(doc1.to_mongo())]
+
+        assert not doc2.modify(dict(name=doc2.name), set__age=100)
+
+        self.assertDbEqual(docs)
+
+    def test_modify_update(self):
+        other_doc = self.Person(name="bob", age=10).save()
+        doc = self.Person(
+            name="jim", age=20, job=self.Job(name="10gen", years=3)).save()
+
+        doc_copy = doc._from_son(doc.to_mongo())
+
+        # these changes must go away
+        doc.name = "liza"
+        doc.job.name = "Google"
+        doc.job.years = 3
+
+        assert doc.modify(
+            set__age=21, set__job__name="MongoDB", unset__job__years=True)
+        doc_copy.age = 21
+        doc_copy.job.name = "MongoDB"
+        del doc_copy.job.years
+
+        assert doc.to_json() == doc_copy.to_json()
+        assert doc._get_changed_fields() == []
+
+        self.assertDbEqual([dict(other_doc.to_mongo()), dict(doc.to_mongo())])
+
     def test_save(self):
         """Ensure that a document may be saved in the database.
         """
@@ -820,6 +927,80 @@ class InstanceTest(unittest.TestCase):
         p1.reload()
         self.assertEqual(p1.name, p.parent.name)
 
+    def test_save_atomicity_condition(self):
+
+        class Widget(Document):
+            toggle = BooleanField(default=False)
+            count = IntField(default=0)
+            save_id = UUIDField()
+
+        def flip(widget):
+            widget.toggle = not widget.toggle
+            widget.count += 1
+
+        def UUID(i):
+            return uuid.UUID(int=i)
+
+        Widget.drop_collection()
+
+        w1 = Widget(toggle=False, save_id=UUID(1))
+
+        # ignore save_condition on new record creation
+        w1.save(save_condition={'save_id': UUID(42)})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.save_id, UUID(1))
+        self.assertEqual(w1.count, 0)
+
+        # mismatch in save_condition prevents save
+        flip(w1)
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+        w1.save(save_condition={'save_id': UUID(42)})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.count, 0)
+
+        # matched save_condition allows save
+        flip(w1)
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+        w1.save(save_condition={'save_id': UUID(1)})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 1)
+
+        # save_condition can be used to ensure atomic read & updates
+        # i.e., prevent interleaved reads and writes from separate contexts
+        w2 = Widget.objects.get()
+        self.assertEqual(w1, w2)
+        old_id = w1.save_id
+
+        flip(w1)
+        w1.save_id = UUID(2)
+        w1.save(save_condition={'save_id': old_id})
+        w1.reload()
+        self.assertFalse(w1.toggle)
+        self.assertEqual(w1.count, 2)
+        flip(w2)
+        flip(w2)
+        w2.save(save_condition={'save_id': old_id})
+        w2.reload()
+        self.assertFalse(w2.toggle)
+        self.assertEqual(w2.count, 2)
+
+        # save_condition uses mongoengine-style operator syntax
+        flip(w1)
+        w1.save(save_condition={'count__lt': w1.count})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 3)
+        flip(w1)
+        w1.save(save_condition={'count__gte': w1.count})
+        w1.reload()
+        self.assertTrue(w1.toggle)
+        self.assertEqual(w1.count, 3)
+
     def test_update(self):
         """Ensure that an existing document is updated instead of be
         overwritten."""
@@ -984,11 +1165,23 @@ class InstanceTest(unittest.TestCase):
 
         self.assertRaises(OperationError, update_no_value_raises)
 
-        def update_no_op_raises():
+        def update_no_op_should_default_to_set():
             person = self.Person.objects.first()
             person.update(name="Dan")
+            person.reload()
+            return person.name
 
-        self.assertRaises(InvalidQueryError, update_no_op_raises)
+        self.assertEqual("Dan", update_no_op_should_default_to_set())
+
+    def test_update_unique_field(self):
+        class Doc(Document):
+            name = StringField(unique=True)
+
+        doc1 = Doc(name="first").save()
+        doc2 = Doc(name="second").save()
+
+        self.assertRaises(NotUniqueError, lambda:
+                          doc2.update(set__name=doc1.name))
 
     def test_embedded_update(self):
         """
@@ -1249,7 +1442,8 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4')
 
     def test_save_custom_pk(self):
-        """Ensure that a document may be saved with a custom _id using pk alias.
+        """
+        Ensure that a document may be saved with a custom _id using pk alias.
         """
         # Create person object and save it to the database
         person = self.Person(name='Test User', age=30,
@@ -1335,9 +1529,15 @@ class InstanceTest(unittest.TestCase):
         p4 = Page(comments=[Comment(user=u2, comment="Heavy Metal song")])
         p4.save()
 
-        self.assertEqual([p1, p2], list(Page.objects.filter(comments__user=u1)))
-        self.assertEqual([p1, p2, p4], list(Page.objects.filter(comments__user=u2)))
-        self.assertEqual([p1, p3], list(Page.objects.filter(comments__user=u3)))
+        self.assertEqual(
+            [p1, p2],
+            list(Page.objects.filter(comments__user=u1)))
+        self.assertEqual(
+            [p1, p2, p4],
+            list(Page.objects.filter(comments__user=u2)))
+        self.assertEqual(
+            [p1, p3],
+            list(Page.objects.filter(comments__user=u3)))
 
     def test_save_embedded_document(self):
         """Ensure that a document with an embedded document field may be
@@ -1412,7 +1612,8 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(promoted_employee.age, 50)
 
         # Ensure that the 'details' embedded object saved correctly
-        self.assertEqual(promoted_employee.details.position, 'Senior Developer')
+        self.assertEqual(
+            promoted_employee.details.position, 'Senior Developer')
 
         # Test removal
         promoted_employee.details = None
@@ -1548,7 +1749,8 @@ class InstanceTest(unittest.TestCase):
         post.save()
 
         reviewer.delete()
-        self.assertEqual(BlogPost.objects.count(), 1)  # No effect on the BlogPost
+        # No effect on the BlogPost
+        self.assertEqual(BlogPost.objects.count(), 1)
         self.assertEqual(BlogPost.objects.get().reviewer, None)
 
         # Delete the Person, which should lead to deletion of the BlogPost, too
@@ -1597,8 +1799,10 @@ class InstanceTest(unittest.TestCase):
 
         class BlogPost(Document):
             content = StringField()
-            authors = ListField(ReferenceField(self.Person, reverse_delete_rule=CASCADE))
-            reviewers = ListField(ReferenceField(self.Person, reverse_delete_rule=NULLIFY))
+            authors = ListField(ReferenceField(
+                self.Person, reverse_delete_rule=CASCADE))
+            reviewers = ListField(ReferenceField(
+                self.Person, reverse_delete_rule=NULLIFY))
 
         self.Person.drop_collection()
 
@@ -1693,13 +1897,17 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(Bar.objects.count(), 1)  # No effect on the BlogPost
         self.assertEqual(Bar.objects.get().foo, None)
 
-    def test_invalid_reverse_delete_rules_raise_errors(self):
+    def test_invalid_reverse_delete_rule_raise_errors(self):
 
         def throw_invalid_document_error():
             class Blog(Document):
                 content = StringField()
-                authors = MapField(ReferenceField(self.Person, reverse_delete_rule=CASCADE))
-                reviewers = DictField(field=ReferenceField(self.Person, reverse_delete_rule=NULLIFY))
+                authors = MapField(ReferenceField(
+                    self.Person, reverse_delete_rule=CASCADE))
+                reviewers = DictField(
+                    field=ReferenceField(
+                        self.Person,
+                        reverse_delete_rule=NULLIFY))
 
         self.assertRaises(InvalidDocumentError, throw_invalid_document_error)
 
@@ -1708,7 +1916,8 @@ class InstanceTest(unittest.TestCase):
                 father = ReferenceField('Person', reverse_delete_rule=DENY)
                 mother = ReferenceField('Person', reverse_delete_rule=DENY)
 
-        self.assertRaises(InvalidDocumentError, throw_invalid_document_error_embedded)
+        self.assertRaises(
+            InvalidDocumentError, throw_invalid_document_error_embedded)
 
     def test_reverse_delete_rule_cascade_recurs(self):
         """Ensure that a chain of documents is also deleted upon cascaded
@@ -1730,16 +1939,16 @@ class InstanceTest(unittest.TestCase):
         author = self.Person(name='Test User')
         author.save()
 
-        post = BlogPost(content = 'Watched some TV')
+        post = BlogPost(content='Watched some TV')
         post.author = author
         post.save()
 
-        comment = Comment(text = 'Kudos.')
+        comment = Comment(text='Kudos.')
         comment.post = post
         comment.save()
 
-        # Delete the Person, which should lead to deletion of the BlogPost, and,
-        # recursively to the Comment, too
+        # Delete the Person, which should lead to deletion of the BlogPost,
+        # and, recursively to the Comment, too
         author.delete()
         self.assertEqual(Comment.objects.count(), 0)
 
@@ -1762,7 +1971,7 @@ class InstanceTest(unittest.TestCase):
         author = self.Person(name='Test User')
         author.save()
 
-        post = BlogPost(content = 'Watched some TV')
+        post = BlogPost(content='Watched some TV')
         post.author = author
         post.save()
 
@@ -1878,7 +2087,8 @@ class InstanceTest(unittest.TestCase):
 
     def test_dynamic_document_pickle(self):
 
-        pickle_doc = PickleDynamicTest(name="test", number=1, string="One", lists=['1', '2'])
+        pickle_doc = PickleDynamicTest(
+            name="test", number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar")
         pickled_doc = pickle.dumps(pickle_doc)  # make sure pickling works even before the doc is saved
 
@@ -1900,7 +2110,8 @@ class InstanceTest(unittest.TestCase):
                          pickle_doc.embedded._dynamic_fields.keys())
 
     def test_picklable_on_signals(self):
-        pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2'])
+        pickle_doc = PickleSignalsTest(
+            number=1, string="One", lists=['1', '2'])
         pickle_doc.embedded = PickleEmbedded()
         pickle_doc.save()
         pickle_doc.delete()
@@ -2055,9 +2266,15 @@ class InstanceTest(unittest.TestCase):
         self.assertEqual(AuthorBooks._get_db(), get_db("testdb-3"))
 
         # Collections
-        self.assertEqual(User._get_collection(), get_db("testdb-1")[User._get_collection_name()])
-        self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()])
-        self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()])
+        self.assertEqual(
+            User._get_collection(),
+            get_db("testdb-1")[User._get_collection_name()])
+        self.assertEqual(
+            Book._get_collection(),
+            get_db("testdb-2")[Book._get_collection_name()])
+        self.assertEqual(
+            AuthorBooks._get_collection(),
+            get_db("testdb-3")[AuthorBooks._get_collection_name()])
 
     def test_db_alias_overrides(self):
         """db_alias can be overriden
@@ -2226,30 +2443,6 @@ class InstanceTest(unittest.TestCase):
         group = Group.objects.first()
         self.assertEqual("hello - default", group.name)
 
-    def test_no_overwritting_no_data_loss(self):
-
-        class User(Document):
-            username = StringField(primary_key=True)
-            name = StringField()
-
-            @property
-            def foo(self):
-                return True
-
-        User.drop_collection()
-
-        user = User(username="Ross", foo="bar")
-        self.assertTrue(user.foo)
-
-        User._get_collection().save({"_id": "Ross", "foo": "Bar",
-                                     "data": [1, 2, 3]})
-
-        user = User.objects.first()
-        self.assertEqual("Ross", user.username)
-        self.assertEqual(True, user.foo)
-        self.assertEqual("Bar", user._data["foo"])
-        self.assertEqual([1, 2, 3], user._data["data"])
-
     def test_spaces_in_keys(self):
 
         class Embedded(DynamicEmbeddedDocument):
@@ -2281,6 +2474,8 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
+        self.assertTrue(log.id is not None)
+
         log.log = "Saving"
         log.save()
 
@@ -2304,6 +2499,8 @@ class InstanceTest(unittest.TestCase):
         log.machine = "Localhost"
         log.save()
 
+        self.assertTrue(log.id is not None)
+
         log.log = "Saving"
         log.save()
 
@@ -2321,6 +2518,10 @@ class InstanceTest(unittest.TestCase):
             doc_name = StringField()
             doc = EmbeddedDocumentField(Embedded)
 
+            def __eq__(self, other):
+                return (self.doc_name == other.doc_name and
+                        self.doc == other.doc)
+
         classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc"))
         dict_doc = Doc(**{"doc_name": "my doc",
                           "doc": {"name": "embedded doc"}})
@@ -2337,6 +2538,10 @@ class InstanceTest(unittest.TestCase):
             doc_name = StringField()
             docs = ListField(EmbeddedDocumentField(Embedded))
 
+            def __eq__(self, other):
+                return (self.doc_name == other.doc_name and
+                        self.docs == other.docs)
+
         classic_doc = Doc(doc_name="my doc", docs=[
                           Embedded(name="embedded doc1"),
                           Embedded(name="embedded doc2")])
@@ -2411,7 +2616,7 @@ class InstanceTest(unittest.TestCase):
                 for parameter_name, parameter in self.parameters.iteritems():
                     parameter.expand()
 
-        class System(Document):
+        class NodesSystem(Document):
             name = StringField(required=True)
             nodes = MapField(ReferenceField(Node, dbref=False))
 
@@ -2419,19 +2624,21 @@ class InstanceTest(unittest.TestCase):
                 for node_name, node in self.nodes.iteritems():
                     node.expand()
                     node.save(*args, **kwargs)
-                super(System, self).save(*args, **kwargs)
+                super(NodesSystem, self).save(*args, **kwargs)
 
-        System.drop_collection()
+        NodesSystem.drop_collection()
         Node.drop_collection()
 
-        system = System(name="system")
+        system = NodesSystem(name="system")
         system.nodes["node"] = Node()
         system.save()
         system.nodes["node"].parameters["param"] = Parameter()
         system.save()
 
-        system = System.objects.first()
-        self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value)
+        system = NodesSystem.objects.first()
+        self.assertEqual(
+            "UNDEFINED",
+            system.nodes["node"].parameters["param"].macros["test"].value)
 
     def test_embedded_document_equality(self):
 
@@ -2452,5 +2659,145 @@ class InstanceTest(unittest.TestCase):
         f1.ref  # Dereferences lazily
         self.assertEqual(f1, f2)
 
+    def test_dbref_equality(self):
+        class Test2(Document):
+            name = StringField()
+
+        class Test3(Document):
+            name = StringField()
+
+        class Test(Document):
+            name = StringField()
+            test2 = ReferenceField('Test2')
+            test3 = ReferenceField('Test3')
+
+        Test.drop_collection()
+        Test2.drop_collection()
+        Test3.drop_collection()
+
+        t2 = Test2(name='a')
+        t2.save()
+
+        t3 = Test3(name='x')
+        t3.id = t2.id
+        t3.save()
+
+        t = Test(name='b', test2=t2, test3=t3)
+
+        f = Test._from_son(t.to_mongo())
+
+        dbref2 = f._data['test2']
+        obj2 = f.test2
+        self.assertTrue(isinstance(dbref2, DBRef))
+        self.assertTrue(isinstance(obj2, Test2))
+        self.assertTrue(obj2.id == dbref2.id)
+        self.assertTrue(obj2 == dbref2)
+        self.assertTrue(dbref2 == obj2)
+
+        dbref3 = f._data['test3']
+        obj3 = f.test3
+        self.assertTrue(isinstance(dbref3, DBRef))
+        self.assertTrue(isinstance(obj3, Test3))
+        self.assertTrue(obj3.id == dbref3.id)
+        self.assertTrue(obj3 == dbref3)
+        self.assertTrue(dbref3 == obj3)
+
+        self.assertTrue(obj2.id == obj3.id)
+        self.assertTrue(dbref2.id == dbref3.id)
+        self.assertFalse(dbref2 == dbref3)
+        self.assertFalse(dbref3 == dbref2)
+        self.assertTrue(dbref2 != dbref3)
+        self.assertTrue(dbref3 != dbref2)
+
+        self.assertFalse(obj2 == dbref3)
+        self.assertFalse(dbref3 == obj2)
+        self.assertTrue(obj2 != dbref3)
+        self.assertTrue(dbref3 != obj2)
+
+        self.assertFalse(obj3 == dbref2)
+        self.assertFalse(dbref2 == obj3)
+        self.assertTrue(obj3 != dbref2)
+        self.assertTrue(dbref2 != obj3)
+
+    def test_default_values(self):
+        class Person(Document):
+            created_on = DateTimeField(default=lambda: datetime.utcnow())
+            name = StringField()
+
+        p = Person(name='alon')
+        p.save()
+        orig_created_on = Person.objects().only('created_on')[0].created_on
+
+        p2 = Person.objects().only('name')[0]
+        p2.name = 'alon2'
+        p2.save()
+        p3 = Person.objects().only('created_on')[0]
+        self.assertEquals(orig_created_on, p3.created_on)
+
+        class Person(Document):
+            created_on = DateTimeField(default=lambda: datetime.utcnow())
+            name = StringField()
+            height = IntField(default=189)
+
+        p4 = Person.objects()[0]
+        p4.save()
+        self.assertEquals(p4.height, 189)
+        self.assertEquals(Person.objects(height=189).count(), 1)
+
+    def test_from_son(self):
+        # 771
+        class MyPerson(self.Person):
+            meta = dict(shard_key=["id"])
+        p = MyPerson.from_json('{"name": "name", "age": 27}', created=True)
+        self.assertEquals(p.id, None)
+        p.id = "12345"  # in case it is not working: "OperationError: Shard Keys are immutable..." will be raised here
+        p = MyPerson._from_son({"name": "name", "age": 27}, created=True)
+        self.assertEquals(p.id, None)
+        p.id = "12345"  # in case it is not working: "OperationError: Shard Keys are immutable..." will be raised here
+
+    def test_null_field(self):
+        # 734
+        class User(Document):
+            name = StringField()
+            height = IntField(default=184, null=True)
+            str_fld = StringField(null=True)
+            int_fld = IntField(null=True)
+            flt_fld = FloatField(null=True)
+            dt_fld = DateTimeField(null=True)
+            cdt_fld = ComplexDateTimeField(null=True)
+
+        User.objects.delete()
+        u = User(name='user')
+        u.save()
+        u_from_db = User.objects.get(name='user')
+        u_from_db.height = None
+        u_from_db.save()
+        self.assertEquals(u_from_db.height, None)
+        # 864
+        self.assertEqual(u_from_db.str_fld, None)
+        self.assertEqual(u_from_db.int_fld, None)
+        self.assertEqual(u_from_db.flt_fld, None)
+        self.assertEqual(u_from_db.dt_fld, None)
+        self.assertEqual(u_from_db.cdt_fld, None)
+
+        # 735
+        User.objects.delete()
+        u = User(name='user')
+        u.save()
+        User.objects(name='user').update_one(set__height=None, upsert=True)
+        u_from_db = User.objects.get(name='user')
+        self.assertEquals(u_from_db.height, None)
+
+    def test_not_saved_eq(self):
+        """Ensure we can compare documents not saved.
+        """
+        class Person(Document):
+            pass
+
+        p = Person()
+        p1 = Person()
+        self.assertNotEqual(p, p1)
+        self.assertEqual(p, p)
+
 if __name__ == '__main__':
     unittest.main()

tests/document/json_serialisation.py

@@ -20,6 +20,28 @@ class TestJson(unittest.TestCase):
     def setUp(self):
         connect(db='mongoenginetest')
 
+    def test_json_names(self):
+        """
+        Going to test reported issue:
+            https://github.com/MongoEngine/mongoengine/issues/654
+        where the reporter asks for the availability to perform
+        a to_json with the original class names and not the abreviated
+        mongodb document keys
+        """
+        class Embedded(EmbeddedDocument):
+            string = StringField(db_field='s')
+
+        class Doc(Document):
+            string = StringField(db_field='s')
+            embedded = EmbeddedDocumentField(Embedded, db_field='e')
+
+        doc = Doc( string="Hello", embedded=Embedded(string="Inner Hello"))
+        doc_json = doc.to_json(sort_keys=True, use_db_field=False,separators=(',', ':'))
+
+        expected_json = """{"embedded":{"string":"Inner Hello"},"string":"Hello"}"""
+
+        self.assertEqual( doc_json, expected_json)
+
     def test_json_simple(self):
 
         class Embedded(EmbeddedDocument):
@@ -29,6 +51,10 @@ class TestJson(unittest.TestCase):
             string = StringField()
             embedded_field = EmbeddedDocumentField(Embedded)
 
+            def __eq__(self, other):
+                return (self.string == other.string and
+                        self.embedded_field == other.embedded_field)
+
         doc = Doc(string="Hi", embedded_field=Embedded(string="Hi"))
 
         doc_json = doc.to_json(sort_keys=True, separators=(',', ':'))
@@ -77,6 +103,10 @@ class TestJson(unittest.TestCase):
             generic_embedded_document_field = GenericEmbeddedDocumentField(
                                         default=lambda: EmbeddedDoc())
 
+            def __eq__(self, other):
+                import json
+                return json.loads(self.to_json()) == json.loads(other.to_json())
+
         doc = Doc()
         self.assertEqual(doc, Doc.from_json(doc.to_json()))
 

tests/document/validation.py

@@ -141,6 +141,30 @@ class ValidatorErrorTest(unittest.TestCase):
             self.assertEqual(e.to_dict(), {
                 "e": {'val': 'OK could not be converted to int'}})
 
+    def test_embedded_weakref(self):
+
+        class SubDoc(EmbeddedDocument):
+            val = IntField(required=True)
+
+        class Doc(Document):
+            e = EmbeddedDocumentField(SubDoc, db_field='eb')
+
+        Doc.drop_collection()
+
+        d1 = Doc()
+        d2 = Doc()
+
+        s = SubDoc()
+
+        self.assertRaises(ValidationError, lambda: s.validate())
+
+        d1.e = s
+        d2.e = s
+
+        del d1
+
+        self.assertRaises(ValidationError, lambda: d2.validate())
+
 
 if __name__ == '__main__':
     unittest.main()

tests/fields/file_tests.py

@@ -114,6 +114,42 @@ class FileTest(unittest.TestCase):
         # Ensure deleted file returns None
         self.assertTrue(result.the_file.read() == None)
 
+    def test_file_fields_stream_after_none(self):
+        """Ensure that a file field can be written to after it has been saved as
+        None
+        """
+        class StreamFile(Document):
+            the_file = FileField()
+
+        StreamFile.drop_collection()
+
+        text = b('Hello, World!')
+        more_text = b('Foo Bar')
+        content_type = 'text/plain'
+
+        streamfile = StreamFile()
+        streamfile.save()
+        streamfile.the_file.new_file()
+        streamfile.the_file.write(text)
+        streamfile.the_file.write(more_text)
+        streamfile.the_file.close()
+        streamfile.save()
+
+        result = StreamFile.objects.first()
+        self.assertTrue(streamfile == result)
+        self.assertEqual(result.the_file.read(), text + more_text)
+        #self.assertEqual(result.the_file.content_type, content_type)
+        result.the_file.seek(0)
+        self.assertEqual(result.the_file.tell(), 0)
+        self.assertEqual(result.the_file.read(len(text)), text)
+        self.assertEqual(result.the_file.tell(), len(text))
+        self.assertEqual(result.the_file.read(len(more_text)), more_text)
+        self.assertEqual(result.the_file.tell(), len(text + more_text))
+        result.the_file.delete()
+
+        # Ensure deleted file returns None
+        self.assertTrue(result.the_file.read() == None)
+
     def test_file_fields_set(self):
 
         class SetFile(Document):
@@ -279,7 +315,7 @@ class FileTest(unittest.TestCase):
                 t.image.put(f)
                 self.fail("Should have raised an invalidation error")
             except ValidationError, e:
-                self.assertEqual("%s" % e, "Invalid image: cannot identify image file")
+                self.assertEqual("%s" % e, "Invalid image: cannot identify image file %s" % f)
 
         t = TestImage()
         t.image.put(open(TEST_IMAGE_PATH, 'rb'))

tests/fields/geo.py

@@ -19,8 +19,8 @@ class GeoFieldTest(unittest.TestCase):
     def _test_for_expected_error(self, Cls, loc, expected):
         try:
             Cls(loc=loc).validate()
-            self.fail()
-        except ValidationError, e:
+            self.fail('Should not validate the location {0}'.format(loc))
+        except ValidationError as e:
             self.assertEqual(expected, e.to_dict()['loc'])
 
     def test_geopoint_validation(self):
@@ -155,6 +155,117 @@ class GeoFieldTest(unittest.TestCase):
 
         Location(loc=[[[1, 2], [3, 4], [5, 6], [1, 2]]]).validate()
 
+    def test_multipoint_validation(self):
+        class Location(Document):
+            loc = MultiPointField()
+
+        invalid_coords = {"x": 1, "y": 2}
+        expected = 'MultiPointField can only accept a valid GeoJson dictionary or lists of (x, y)'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MadeUp", "coordinates": [[]]}
+        expected = 'MultiPointField type must be "MultiPoint"'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MultiPoint", "coordinates": [[1, 2, 3]]}
+        expected = "Value ([1, 2, 3]) must be a two-dimensional point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[]]
+        expected = "Invalid MultiPoint must contain at least one valid point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[1]], [[1, 2, 3]]]
+        for coord in invalid_coords:
+            expected = "Value (%s) must be a two-dimensional point" % repr(coord[0])
+            self._test_for_expected_error(Location, coord, expected)
+
+        invalid_coords = [[[{}, {}]], [("a", "b")]]
+        for coord in invalid_coords:
+            expected = "Both values (%s) in point must be float or int" % repr(coord[0])
+            self._test_for_expected_error(Location, coord, expected)
+
+        Location(loc=[[1, 2]]).validate()
+        Location(loc={
+            "type": "MultiPoint",
+            "coordinates": [
+                [1, 2],
+                [81.4471435546875, 23.61432859499169]
+            ]}).validate()
+
+    def test_multilinestring_validation(self):
+        class Location(Document):
+            loc = MultiLineStringField()
+
+        invalid_coords = {"x": 1, "y": 2}
+        expected = 'MultiLineStringField can only accept a valid GeoJson dictionary or lists of (x, y)'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MadeUp", "coordinates": [[]]}
+        expected = 'MultiLineStringField type must be "MultiLineString"'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MultiLineString", "coordinates": [[[1, 2, 3]]]}
+        expected = "Invalid MultiLineString:\nValue ([1, 2, 3]) must be a two-dimensional point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [5, "a"]
+        expected = "Invalid MultiLineString must contain at least one valid linestring"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[1]]]
+        expected = "Invalid MultiLineString:\nValue (%s) must be a two-dimensional point" % repr(invalid_coords[0][0])
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[1, 2, 3]]]
+        expected = "Invalid MultiLineString:\nValue (%s) must be a two-dimensional point" % repr(invalid_coords[0][0])
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[{}, {}]]], [[("a", "b")]]]
+        for coord in invalid_coords:
+            expected = "Invalid MultiLineString:\nBoth values (%s) in point must be float or int" % repr(coord[0][0])
+            self._test_for_expected_error(Location, coord, expected)
+
+        Location(loc=[[[1, 2], [3, 4], [5, 6], [1,2]]]).validate()
+
+    def test_multipolygon_validation(self):
+        class Location(Document):
+            loc = MultiPolygonField()
+
+        invalid_coords = {"x": 1, "y": 2}
+        expected = 'MultiPolygonField can only accept a valid GeoJson dictionary or lists of (x, y)'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MadeUp", "coordinates": [[]]}
+        expected = 'MultiPolygonField type must be "MultiPolygon"'
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = {"type": "MultiPolygon", "coordinates": [[[[1, 2, 3]]]]}
+        expected = "Invalid MultiPolygon:\nValue ([1, 2, 3]) must be a two-dimensional point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[5, "a"]]]]
+        expected = "Invalid MultiPolygon:\nBoth values ([5, 'a']) in point must be float or int"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[]]]]
+        expected = "Invalid MultiPolygon must contain at least one valid Polygon"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[1, 2, 3]]]]
+        expected = "Invalid MultiPolygon:\nValue ([1, 2, 3]) must be a two-dimensional point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[{}, {}]]], [[("a", "b")]]]
+        expected = "Invalid MultiPolygon:\nBoth values ([{}, {}]) in point must be float or int, Both values (('a', 'b')) in point must be float or int"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        invalid_coords = [[[[1, 2], [3, 4]]]]
+        expected = "Invalid MultiPolygon:\nLineStrings must start and end at the same point"
+        self._test_for_expected_error(Location, invalid_coords, expected)
+
+        Location(loc=[[[[1, 2], [3, 4], [5, 6], [1, 2]]]]).validate()
+
     def test_indexes_geopoint(self):
         """Ensure that indexes are created automatically for GeoPointFields.
         """

tests/queryset/__init__.py

@@ -3,3 +3,4 @@ from field_list import *
 from queryset import *
 from visitor import *
 from geo import *
+from modify import *

tests/queryset/geo.py

@@ -5,6 +5,8 @@ import unittest
 from datetime import datetime, timedelta
 from mongoengine import *
 
+from nose.plugins.skip import SkipTest
+
 __all__ = ("GeoQueriesTest",)
 
 
@@ -139,6 +141,7 @@ class GeoQueriesTest(unittest.TestCase):
     def test_spherical_geospatial_operators(self):
         """Ensure that spherical geospatial queries are working
         """
+        raise SkipTest("https://jira.mongodb.org/browse/SERVER-14039")
         class Point(Document):
             location = GeoPointField()
 

tests/queryset/modify.py

@@ -0,0 +1,102 @@
+import sys
+sys.path[0:0] = [""]
+
+import unittest
+
+from mongoengine import connect, Document, IntField
+
+__all__ = ("FindAndModifyTest",)
+
+
+class Doc(Document):
+    id = IntField(primary_key=True)
+    value = IntField()
+
+
+class FindAndModifyTest(unittest.TestCase):
+
+    def setUp(self):
+        connect(db="mongoenginetest")
+        Doc.drop_collection()
+
+    def assertDbEqual(self, docs):
+        self.assertEqual(list(Doc._collection.find().sort("id")), docs)
+
+    def test_modify(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(set__value=-1)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_with_new(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        new_doc = Doc.objects(id=1).modify(set__value=-1, new=True)
+        doc.value = -1
+        self.assertEqual(new_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_not_existing(self):
+        Doc(id=0, value=0).save()
+        self.assertEqual(Doc.objects(id=1).modify(set__value=-1), None)
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_modify_with_upsert(self):
+        Doc(id=0, value=0).save()
+        old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True)
+        self.assertEqual(old_doc, None)
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+
+    def test_modify_with_upsert_existing(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+    def test_modify_with_upsert_with_new(self):
+        Doc(id=0, value=0).save()
+        new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1)
+        self.assertEqual(new_doc.to_mongo(), {"_id": 1, "value": 1})
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}])
+
+    def test_modify_with_remove(self):
+        Doc(id=0, value=0).save()
+        doc = Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).modify(remove=True)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_find_and_modify_with_remove_not_existing(self):
+        Doc(id=0, value=0).save()
+        self.assertEqual(Doc.objects(id=1).modify(remove=True), None)
+        self.assertDbEqual([{"_id": 0, "value": 0}])
+
+    def test_modify_with_order_by(self):
+        Doc(id=0, value=3).save()
+        Doc(id=1, value=2).save()
+        Doc(id=2, value=1).save()
+        doc = Doc(id=3, value=0).save()
+
+        old_doc = Doc.objects().order_by("-id").modify(set__value=-1)
+        self.assertEqual(old_doc.to_json(), doc.to_json())
+        self.assertDbEqual([
+            {"_id": 0, "value": 3}, {"_id": 1, "value": 2},
+            {"_id": 2, "value": 1}, {"_id": 3, "value": -1}])
+
+    def test_modify_with_fields(self):
+        Doc(id=0, value=0).save()
+        Doc(id=1, value=1).save()
+
+        old_doc = Doc.objects(id=1).only("id").modify(set__value=-1)
+        self.assertEqual(old_doc.to_mongo(), {"_id": 1})
+        self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/queryset/transform.py

@@ -197,5 +197,17 @@ class TransformTest(unittest.TestCase):
         update = transform.update(Location, set__poly={"type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [40, 5]]]})
         self.assertEqual(update, {'$set': {'poly': {"type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [40, 5]]]}}})
 
+    def test_type(self):
+        class Doc(Document):
+            df = DynamicField()
+        Doc(df=True).save()
+        Doc(df=7).save()
+        Doc(df="df").save()
+        self.assertEqual(Doc.objects(df__type=1).count(), 0)  # double
+        self.assertEqual(Doc.objects(df__type=8).count(), 1)  # bool
+        self.assertEqual(Doc.objects(df__type=2).count(), 1)  # str
+        self.assertEqual(Doc.objects(df__type=16).count(), 1)  # int
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_connection.py

@@ -1,6 +1,11 @@
 import sys
 sys.path[0:0] = [""]
-import unittest
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
 import datetime
 
 import pymongo
@@ -34,6 +39,17 @@ class ConnectionTest(unittest.TestCase):
         conn = get_connection('testdb')
         self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
 
+    def test_sharing_connections(self):
+        """Ensure that connections are shared when the connection settings are exactly the same
+        """
+        connect('mongoenginetest', alias='testdb1')
+
+        expected_connection = get_connection('testdb1')
+
+        connect('mongoenginetest', alias='testdb2')
+        actual_connection = get_connection('testdb2')
+        self.assertEqual(expected_connection, actual_connection)
+
     def test_connect_uri(self):
         """Ensure that the connect() method works properly with uri's
         """
@@ -131,6 +147,18 @@ class ConnectionTest(unittest.TestCase):
         date_doc = DateDoc.objects.first()
         self.assertEqual(d, date_doc.the_date)
 
+    def test_multiple_connection_settings(self):
+        connect('mongoenginetest', alias='t1', host="localhost")
+
+        connect('mongoenginetest2', alias='t2', host="127.0.0.1")
+
+        mongo_connections = mongoengine.connection._connections
+        self.assertEqual(len(mongo_connections.items()), 2)
+        self.assertTrue('t1' in mongo_connections.keys())
+        self.assertTrue('t2' in mongo_connections.keys())
+        self.assertEqual(mongo_connections['t1'].host, 'localhost')
+        self.assertEqual(mongo_connections['t2'].host, '127.0.0.1')
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_datastructures.py

@@ -0,0 +1,107 @@
+import unittest
+from mongoengine.base.datastructures import StrictDict, SemiStrictDict 
+
+class TestStrictDict(unittest.TestCase):
+    def strict_dict_class(self, *args, **kwargs):
+        return StrictDict.create(*args, **kwargs)
+    def setUp(self):
+        self.dtype = self.strict_dict_class(("a", "b", "c"))
+    def test_init(self):
+        d = self.dtype(a=1, b=1, c=1)
+        self.assertEqual((d.a, d.b, d.c), (1, 1, 1))
+
+    def test_init_fails_on_nonexisting_attrs(self):
+        self.assertRaises(AttributeError, lambda: self.dtype(a=1, b=2, d=3))
+        
+    def test_eq(self):
+        d = self.dtype(a=1, b=1, c=1)
+        dd = self.dtype(a=1, b=1, c=1)
+        e = self.dtype(a=1, b=1, c=3)
+        f = self.dtype(a=1, b=1)
+        g = self.strict_dict_class(("a", "b", "c", "d"))(a=1, b=1, c=1, d=1)
+        h = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=1)
+        i = self.strict_dict_class(("a", "c", "b"))(a=1, b=1, c=2)
+        
+        self.assertEqual(d, dd)
+        self.assertNotEqual(d, e)
+        self.assertNotEqual(d, f)
+        self.assertNotEqual(d, g)
+        self.assertNotEqual(f, d)
+        self.assertEqual(d, h)
+        self.assertNotEqual(d, i)
+
+    def test_setattr_getattr(self):
+        d = self.dtype()
+        d.a = 1
+        self.assertEqual(d.a, 1)
+        self.assertRaises(AttributeError, lambda: d.b)
+    
+    def test_setattr_raises_on_nonexisting_attr(self):
+        d = self.dtype()
+        def _f():
+            d.x=1
+        self.assertRaises(AttributeError, _f)
+    
+    def test_setattr_getattr_special(self):
+        d = self.strict_dict_class(["items"])
+        d.items = 1
+        self.assertEqual(d.items, 1)
+    
+    def test_get(self):
+        d = self.dtype(a=1)
+        self.assertEqual(d.get('a'), 1)
+        self.assertEqual(d.get('b', 'bla'), 'bla')
+
+    def test_items(self):
+        d = self.dtype(a=1)
+        self.assertEqual(d.items(), [('a', 1)])
+        d = self.dtype(a=1, b=2)
+        self.assertEqual(d.items(), [('a', 1), ('b', 2)])
+
+    def test_mappings_protocol(self):
+        d = self.dtype(a=1, b=2)
+        assert dict(d) == {'a': 1, 'b': 2}
+        assert dict(**d) == {'a': 1, 'b': 2}
+
+
+class TestSemiSrictDict(TestStrictDict):
+    def strict_dict_class(self, *args, **kwargs):
+        return SemiStrictDict.create(*args, **kwargs)
+
+    def test_init_fails_on_nonexisting_attrs(self):
+        # disable irrelevant test
+        pass
+
+    def test_setattr_raises_on_nonexisting_attr(self):
+        # disable irrelevant test
+        pass
+
+    def test_setattr_getattr_nonexisting_attr_succeeds(self):
+        d = self.dtype()
+        d.x = 1
+        self.assertEqual(d.x, 1)
+
+    def test_init_succeeds_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual((d.a, d.b, d.c, d.x), (1, 1, 1, 2))
+   
+    def test_iter_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual(list(d), ['a', 'b', 'c', 'x'])
+
+    def test_iteritems_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        self.assertEqual(list(d.iteritems()), [('a', 1), ('b', 1), ('c', 1), ('x', 2)])
+
+    def tets_cmp_with_strict_dicts(self):
+        d = self.dtype(a=1, b=1, c=1)
+        dd = StrictDict.create(("a", "b", "c"))(a=1, b=1, c=1)
+        self.assertEqual(d, dd)
+
+    def test_cmp_with_strict_dict_with_nonexisting_attrs(self):
+        d = self.dtype(a=1, b=1, c=1, x=2)
+        dd = StrictDict.create(("a", "b", "c", "x"))(a=1, b=1, c=1, x=2)
+        self.assertEqual(d, dd)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_dereference.py

@@ -291,9 +291,37 @@ class FieldTest(unittest.TestCase):
                 self.assertEqual(employee.friends, friends)
                 self.assertEqual(q, 2)
 
+    def test_list_of_lists_of_references(self):
+
+        class User(Document):
+            name = StringField()
+
+        class Post(Document):
+            user_lists = ListField(ListField(ReferenceField(User)))
+
+        class SimpleList(Document):
+            users = ListField(ReferenceField(User))
+
+        User.drop_collection()
+        Post.drop_collection()
+
+        u1 = User.objects.create(name='u1')
+        u2 = User.objects.create(name='u2')
+        u3 = User.objects.create(name='u3')
+
+        SimpleList.objects.create(users=[u1, u2, u3])
+        self.assertEqual(SimpleList.objects.all()[0].users, [u1, u2, u3])
+
+        Post.objects.create(user_lists=[[u1, u2], [u3]])
+        self.assertEqual(Post.objects.all()[0].user_lists, [[u1, u2], [u3]])
+
     def test_circular_reference(self):
         """Ensure you can handle circular references
         """
+        class Relation(EmbeddedDocument):
+            name = StringField()
+            person = ReferenceField('Person')
+
         class Person(Document):
             name = StringField()
             relations = ListField(EmbeddedDocumentField('Relation'))
@@ -301,10 +329,6 @@ class FieldTest(unittest.TestCase):
             def __repr__(self):
                 return "<Person: %s>" % self.name
 
-        class Relation(EmbeddedDocument):
-            name = StringField()
-            person = ReferenceField('Person')
-
         Person.drop_collection()
         mother = Person(name="Mother")
         daughter = Person(name="Daughter")
@@ -923,6 +947,8 @@ class FieldTest(unittest.TestCase):
 
         class Asset(Document):
             name = StringField(max_length=250, required=True)
+            path = StringField()
+            title = StringField()
             parent = GenericReferenceField(default=None)
             parents = ListField(GenericReferenceField())
             children = ListField(GenericReferenceField())
@@ -1195,6 +1221,31 @@ class FieldTest(unittest.TestCase):
         page = Page.objects.first()
         self.assertEqual(page.tags[0], page.posts[0].tags[0])
 
+    def test_select_related_follows_embedded_referencefields(self):
+
+        class Song(Document):
+            title = StringField()
+
+        class PlaylistItem(EmbeddedDocument):
+            song = ReferenceField("Song")
+
+        class Playlist(Document):
+            items = ListField(EmbeddedDocumentField("PlaylistItem"))
+
+        Playlist.drop_collection()
+        Song.drop_collection()
+
+        songs = [Song.objects.create(title="song %d" % i) for i in range(3)]
+        items = [PlaylistItem(song=song) for song in songs]
+        playlist = Playlist.objects.create(items=items)
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            playlist = Playlist.objects.first().select_related()
+            songs = [item.song for item in playlist.items]
+
+            self.assertEqual(q, 2)
+
 if __name__ == '__main__':
     unittest.main()
-

tests/test_django.py

@@ -2,11 +2,11 @@ import sys
 sys.path[0:0] = [""]
 import unittest
 from nose.plugins.skip import SkipTest
+
 from mongoengine import *
-
-
 from mongoengine.django.shortcuts import get_document_or_404
 
+import django
 from django.http import Http404
 from django.template import Context, Template
 from django.conf import settings
@@ -19,6 +19,13 @@ settings.configure(
     AUTHENTICATION_BACKENDS = ('mongoengine.django.auth.MongoEngineBackend',)
 )
 
+try:
+    # For Django >= 1.7
+    if hasattr(django, 'setup'):
+        django.setup()
+except RuntimeError:
+    pass
+
 try:
     from django.contrib.auth import authenticate, get_user_model
     from mongoengine.django.auth import User
@@ -30,8 +37,8 @@ try:
     DJ15 = True
 except Exception:
     DJ15 = False
-from django.contrib.sessions.tests import SessionTestsMixin
 from mongoengine.django.sessions import SessionStore, MongoSession
+from mongoengine.django.tests import MongoTestCase
 from datetime import tzinfo, timedelta
 ZERO = timedelta(0)
 
@@ -165,8 +172,13 @@ class QuerySetTest(unittest.TestCase):
         """Ensure that a queryset and filters work as expected
         """
 
+        class LimitCountQuerySet(QuerySet):
+            def count(self, with_limit_and_skip=True):
+                return super(LimitCountQuerySet, self).count(with_limit_and_skip)
+
         class Note(Document):
-            text = StringField()
+            meta = dict(queryset_class=LimitCountQuerySet)
+            name = StringField()
 
         Note.drop_collection()
 
@@ -216,13 +228,13 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(t.render(c), "10")
 
 
-class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase):
+class _BaseMongoDBSessionTest(unittest.TestCase):
     backend = SessionStore
 
     def setUp(self):
         connect(db='mongoenginetest')
         MongoSession.drop_collection()
-        super(MongoDBSessionTest, self).setUp()
+        super(_BaseMongoDBSessionTest, self).setUp()
 
     def assertIn(self, first, second, msg=None):
         self.assertTrue(first in second, msg)
@@ -249,6 +261,21 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase):
         self.assertTrue('test_expire' in session, 'Session has expired before it is expected')
 
 
+try:
+    # SessionTestsMixin isn't available for import on django > 1.8a1
+    from django.contrib.sessions.tests import SessionTestsMixin
+
+    class _MongoDBSessionTest(SessionTestsMixin):
+        pass
+
+    class MongoDBSessionTest(_BaseMongoDBSessionTest):
+        pass
+
+except ImportError:
+    class MongoDBSessionTest(_BaseMongoDBSessionTest):
+        pass
+
+
 class MongoAuthTest(unittest.TestCase):
     user_data = {
         'username': 'user',
@@ -293,5 +320,11 @@ class MongoAuthTest(unittest.TestCase):
         db_user = User.objects.get(username='user')
         self.assertEqual(user.id, db_user.id)
 
+
+class MongoTestCaseTest(MongoTestCase):
+    def test_mongo_test_case(self):
+        self.db.dummy_collection.insert({'collection': 'will be dropped'})
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_signals.py

@@ -37,7 +37,8 @@ class SignalTests(unittest.TestCase):
 
             @classmethod
             def post_init(cls, sender, document, **kwargs):
-                signal_output.append('post_init signal, %s' % document)
+                signal_output.append('post_init signal, %s, document._created = %s' % (document, document._created))
+
 
             @classmethod
             def pre_save(cls, sender, document, **kwargs):
@@ -54,7 +55,9 @@ class SignalTests(unittest.TestCase):
 
             @classmethod
             def post_save(cls, sender, document, **kwargs):
+                dirty_keys = document._delta()[0].keys() + document._delta()[1].keys()
                 signal_output.append('post_save signal, %s' % document)
+                signal_output.append('post_save dirty keys, %s' % dirty_keys)
                 if 'created' in kwargs:
                     if kwargs['created']:
                         signal_output.append('Is created')
@@ -191,10 +194,16 @@ class SignalTests(unittest.TestCase):
             a1 = self.Author(name='Bill Shakespeare')
             self.Author.objects.insert([a1], load_bulk=False)
 
+        def load_existing_author():
+            a  = self.Author(name='Bill Shakespeare')
+            a.save()
+            self.get_signal_output(lambda: None) # eliminate signal output
+            a1 = self.Author.objects(name='Bill Shakespeare')[0]
+        
         self.assertEqual(self.get_signal_output(create_author), [
             "pre_init signal, Author",
             "{'name': 'Bill Shakespeare'}",
-            "post_init signal, Bill Shakespeare",
+            "post_init signal, Bill Shakespeare, document._created = True",
         ])
 
         a1 = self.Author(name='Bill Shakespeare')
@@ -203,6 +212,7 @@ class SignalTests(unittest.TestCase):
             "pre_save_post_validation signal, Bill Shakespeare",
             "Is created",
             "post_save signal, Bill Shakespeare",
+            "post_save dirty keys, ['name']",
             "Is created"
         ])
 
@@ -213,6 +223,7 @@ class SignalTests(unittest.TestCase):
             "pre_save_post_validation signal, William Shakespeare",
             "Is updated",
             "post_save signal, William Shakespeare",
+            "post_save dirty keys, ['name']",
             "Is updated"
         ])
 
@@ -221,12 +232,22 @@ class SignalTests(unittest.TestCase):
             'post_delete signal, William Shakespeare',
         ])
 
+        signal_output = self.get_signal_output(load_existing_author)
+        # test signal_output lines separately, because of random ObjectID after object load
+        self.assertEqual(signal_output[0],
+            "pre_init signal, Author",
+        )
+        self.assertEqual(signal_output[2],
+            "post_init signal, Bill Shakespeare, document._created = False",
+        )
+
+
         signal_output = self.get_signal_output(bulk_create_author_with_load)
 
         # The output of this signal is not entirely deterministic. The reloaded
         # object will have an object ID. Hence, we only check part of the output
-        self.assertEqual(signal_output[3],
-            "pre_bulk_insert signal, [<Author: Bill Shakespeare>]")
+        self.assertEqual(signal_output[3], "pre_bulk_insert signal, [<Author: Bill Shakespeare>]"
+        )
         self.assertEqual(signal_output[-2:],
             ["post_bulk_insert signal, [<Author: Bill Shakespeare>]",
              "Is loaded",])
@@ -234,7 +255,7 @@ class SignalTests(unittest.TestCase):
         self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [
             "pre_init signal, Author",
             "{'name': 'Bill Shakespeare'}",
-            "post_init signal, Bill Shakespeare",
+            "post_init signal, Bill Shakespeare, document._created = True",
             "pre_bulk_insert signal, [<Author: Bill Shakespeare>]",
             "post_bulk_insert signal, [<Author: Bill Shakespeare>]",
             "Not loaded",