Update changelog location

Conflicts:
	tests/test_document.py

.travis.yml

@@ -1,15 +1,27 @@
 # http://travis-ci.org/#!/MongoEngine/mongoengine
 language: python
+services: mongodb
 python:
-    - 2.5
-    - 2.6
-    - 2.7
-    - 3.1
-    - 3.2
+    - "2.5"
+    - "2.6"
+    - "2.7"
+    - "3.2"
+    - "3.3"
+env:
+  - PYMONGO=dev
+  - PYMONGO=2.5
+  - PYMONGO=2.4.2
 install:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
+    - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
+    - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
     - python setup.py install
 script:
-    - python setup.py test
+    - python setup.py test
+notifications:
+  irc: "irc.freenode.org#mongoengine"
+branches:
+  only:
+    - master
+    - "0.8"

AUTHORS

@@ -106,7 +106,7 @@ that much better:
  * Adam Reeve
  * Anthony Nemitz
  * deignacio
- * shaunduncan
+ * Shaun Duncan
  * Meir Kriheli
  * Andrey Fedoseev
  * aparajita
@@ -122,3 +122,20 @@ that much better:
  * Sergey Nikitin
  * psychogenic
  * Stefan Wójcik
+ * dimonb
+ * Garry Polley
+ * Adrian Scott
+ * Peter Teichman
+ * Jakub Kot
+ * Jorge Bastida
+ * Aleksandr Sorokoumov
+ * Yohan Graterol
+ * bool-dev
+ * Russ Weeks
+ * Paul Swartz
+ * Sundar Raman
+ * Benoit Louy
+ * lraucy
+ * hellysmile
+ * Jaepil Jeong
+ * Daniil Sharou

CONTRIBUTING.rst

@@ -0,0 +1,61 @@
+Contributing to MongoEngine
+===========================
+
+MongoEngine has a large `community
+<https://raw.github.com/MongoEngine/mongoengine/master/AUTHORS>`_ and
+contributions are always encouraged. Contributions can be as simple as
+minor tweaks to the documentation. Please read these guidelines before
+sending a pull request.
+
+Bugfixes and New Features
+-------------------------
+
+Before starting to write code, look for existing `tickets
+<https://github.com/MongoEngine/mongoengine/issues?state=open>`_ or `create one
+<https://github.com/MongoEngine/mongoengine/issues>`_ for your specific
+issue or feature request. That way you avoid working on something
+that might not be of interest or that has already been addressed.  If in doubt
+post to the `user group <http://groups.google.com/group/mongoengine-users>`
+
+Supported Interpreters
+----------------------
+
+PyMongo supports CPython 2.5 and newer. Language
+features not supported by all interpreters can not be used.
+Please also ensure that your code is properly converted by
+`2to3 <http://docs.python.org/library/2to3.html>`_ for Python 3 support.
+
+Style Guide
+-----------
+
+MongoEngine aims to follow `PEP8 <http://www.python.org/dev/peps/pep-0008/>`_
+including 4 space indents and 79 character line limits.
+
+Testing
+-------
+
+All tests are run on `Travis <http://travis-ci.org/MongoEngine/mongoengine>`_
+and any pull requests are automatically tested by Travis. Any pull requests
+without tests will take longer to be integrated and might be refused.
+
+General Guidelines
+------------------
+
+- Avoid backward breaking changes if at all possible.
+- Write inline documentation for new classes and methods.
+- Write tests and make sure they pass (make sure you have a mongod
+  running on the default port, then execute ``python setup.py test``
+  from the cmd line to run the test suite).
+- Add yourself to AUTHORS.rst :)
+
+Documentation
+-------------
+
+To contribute to the `API documentation
+<http://docs.mongoengine.org/en/latest/apireference.html>`_
+just make your changes to the inline documentation of the appropriate
+`source code <https://github.com/MongoEngine/mongoengine>`_ or `rst file
+<https://github.com/MongoEngine/mongoengine/tree/master/docs>`_ in a
+branch and submit a `pull request <https://help.github.com/articles/using-pull-requests>`_.
+You might also use the github `Edit <https://github.com/blog/844-forking-with-the-edit-button>`_
+button.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2012 See AUTHORS
+Copyright (c) 2009 See AUTHORS
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.rst

@@ -14,7 +14,7 @@ About
 MongoEngine is a Python Object-Document Mapper for working with MongoDB.
 Documentation available at http://mongoengine-odm.rtfd.org - there is currently
 a `tutorial <http://readthedocs.org/docs/mongoengine-odm/en/latest/tutorial.html>`_, a `user guide
-<http://readthedocs.org/docs/mongoengine-odm/en/latest/userguide.html>`_ and an `API reference
+<https://mongoengine-odm.readthedocs.org/en/latest/guide/index.html>`_ and an `API reference
 <http://readthedocs.org/docs/mongoengine-odm/en/latest/apireference.html>`_.
 
 Installation
@@ -63,11 +63,6 @@ Some simple examples of what MongoEngine code looks like::
     ...         print 'Link:', post.url
     ...     print
     ...
-    === Using MongoEngine ===
-    See the tutorial
-
-    === MongoEngine Docs ===
-    Link: hmarr.com/mongoengine
 
     >>> len(BlogPost.objects)
     2
@@ -85,7 +80,7 @@ Some simple examples of what MongoEngine code looks like::
 Tests
 =====
 To run the test suite, ensure you are running a local instance of MongoDB on
-the standard port, and run ``python setup.py test``.
+the standard port, and run: ``python setup.py test``.
 
 Community
 =========
@@ -93,11 +88,8 @@ Community
   <http://groups.google.com/group/mongoengine-users>`_
 - `MongoEngine Developers mailing list
   <http://groups.google.com/group/mongoengine-dev>`_
-- `#mongoengine IRC channel <irc://irc.freenode.net/mongoengine>`_
+- `#mongoengine IRC channel <http://webchat.freenode.net/?channels=mongoengine>`_
 
 Contributing
 ============
-The source is available on `GitHub <http://github.com/MongoEngine/mongoengine>`_ - to
-contribute to the project, fork it on GitHub and send a pull request, all
-contributions and suggestions are welcome!
-
+We welcome contributions! see  the`Contribution guidelines <https://github.com/MongoEngine/mongoengine/blob/master/CONTRIBUTING.rst>`_

docs/changelog.rst

@@ -2,8 +2,79 @@
 Changelog
 =========
 
-Changes in 0.7.X
+Changes in 0.7.10
 =================
+- Fix UnicodeEncodeError for dbref (#278)
+- Allow construction using positional parameters (#268)
+- Updated EmailField length to support long domains (#243)
+- Added 64-bit integer support (#251)
+- Added Django sessions TTL support (#224)
+- Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240)
+- Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242)
+- Added "id" back to _data dictionary (#255)
+- Only mark a field as changed if the value has changed (#258)
+- Explicitly check for Document instances when dereferencing (#261)
+- Fixed order_by chaining issue (#265)
+- Added dereference support for tuples (#250)
+- Resolve field name to db field name when using distinct(#260, #264, #269)
+- Added kwargs to doc.save to help interop with django (#223, #270)
+- Fixed cloning querysets in PY3
+- Int fields no longer unset in save when changed to 0 (#272)
+- Fixed ReferenceField query chaining bug fixed (#254)
+
+Changes in 0.7.9
+================
+- Better fix handling for old style _types
+- Embedded SequenceFields follow collection naming convention
+
+Changes in 0.7.8
+================
+- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166)
+- Fix query chaining with .order_by() (MongoEngine/mongoengine#176)
+- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183)
+- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187)
+- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192)
+- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193)
+
+Changes in 0.7.7
+================
+- Fix handling for old style _types
+
+Changes in 0.7.6
+================
+- Unicode fix for repr (MongoEngine/mongoengine#133)
+- Allow updates with match operators (MongoEngine/mongoengine#144)
+- Updated URLField - now can have a override the regex (MongoEngine/mongoengine#136)
+- Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573)
+- Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138)
+
+Changes in 0.7.5
+================
+- ReferenceFields with dbref=False use ObjectId instead of strings (MongoEngine/mongoengine#134)
+  See ticket for upgrade notes (https://github.com/MongoEngine/mongoengine/issues/134)
+
+Changes in 0.7.4
+================
+- Fixed index inheritance issues - firmed up testcases (MongoEngine/mongoengine#123) (MongoEngine/mongoengine#125)
+
+Changes in 0.7.3
+================
+- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (MongoEngine/mongoengine#119)
+
+Changes in 0.7.2
+================
+- Update index spec generation so its not destructive (MongoEngine/mongoengine#113)
+
+Changes in 0.7.1
+=================
+- Fixed index spec inheritance (MongoEngine/mongoengine#111)
+
+Changes in 0.7.0
+=================
+- Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107)
+- Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104)
+- Fixed Q object merge edge case (MongoEngine/mongoengine#109)
+- Fixed reloading on sharded documents (hmarr/mongoengine#569)
 - Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62)
 - Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92)
 - Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88)

docs/django.rst

@@ -10,6 +10,16 @@ In your **settings.py** file, ignore the standard database settings (unless you
 also plan to use the ORM in your project), and instead call
 :func:`~mongoengine.connect` somewhere in the settings module.
 
+.. note ::
+   If you are not using another Database backend make sure you add  a dummy
+   backend, by adding the following to ``settings.py``::
+
+        DATABASES = {
+            'default': {
+                'ENGINE': 'django.db.backends.dummy'
+            }
+        }
+
 Authentication
 ==============
 MongoEngine includes a Django authentication backend, which uses MongoDB. The
@@ -45,6 +55,9 @@ into you settings module::
 
     SESSION_ENGINE = 'mongoengine.django.sessions'
 
+Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports  `mongodb TTL
+<http://docs.mongodb.org/manual/tutorial/expire-data/>`_.
+
 .. versionadded:: 0.2.1
 
 Storage

docs/guide/defining-documents.rst

@@ -344,6 +344,10 @@ Its value can take any of the following constants:
    their :file:`models.py` in the :const:`INSTALLED_APPS` tuple.
 
 
+.. warning::
+   Signals are not triggered when doing cascading updates / deletes - if this
+   is required you must manually handle the update / delete.
+
 Generic reference fields
 ''''''''''''''''''''''''
 A second kind of reference field also exists,
@@ -465,7 +469,7 @@ If a dictionary is passed then the following options are available:
     Whether the index should be sparse.
 
 :attr:`unique` (Default: False)
-    Whether the index should be sparse.
+    Whether the index should be unique.
 
 .. note ::
 

docs/guide/signals.rst

@@ -50,4 +50,11 @@ Example usage::
     signals.post_save.connect(Author.post_save, sender=Author)
 
 
+ReferenceFields and signals
+---------------------------
+
+Currently `reverse_delete_rules` do not trigger signals on the other part of
+the relationship.  If this is required you must manually handled the
+reverse deletion.
+
 .. _blinker: http://pypi.python.org/pypi/blinker

docs/index.rst

@@ -34,10 +34,10 @@ To get help with using MongoEngine, use the `MongoEngine Users mailing list
 Contributing
 ------------
 
-The source is available on `GitHub <http://github.com/hmarr/mongoengine>`_ and
+The source is available on `GitHub <http://github.com/MongoEngine/mongoengine>`_ and
 contributions are always encouraged. Contributions can be as simple as
 minor tweaks to this documentation. To contribute, fork the project on
-`GitHub <http://github.com/hmarr/mongoengine>`_ and send a
+`GitHub <http://github.com/MongoEngine/mongoengine>`_ and send a
 pull request.
 
 Also, you can join the developers' `mailing list

docs/upgrade.rst

@@ -61,6 +61,13 @@ stored in rather than as string representations.  Your code may need to be
 updated to handle native types rather than strings keys for the results of
 item frequency queries.
 
+BinaryFields
+------------
+
+Binary fields have been updated so that they are native binary types.  If you
+previously were doing `str` comparisons with binary field values you will have
+to update and wrap the value in a `str`.
+
 0.5 to 0.6
 ==========
 
@@ -176,3 +183,9 @@ Alternatively, you can rename your collections eg ::
         else:
             print "Upgraded collection names"
 
+
+mongodb 1.8 > 2.0 +
+===================
+
+Its been reported that indexes may need to be recreated to the newer version of indexes.  
+To do this drop indexes and call ``ensure_indexes`` on each model.

mongoengine/__init__.py

@@ -12,7 +12,7 @@ from signals import *
 __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
            queryset.__all__ + signals.__all__)
 
-VERSION = (0, '7rc1')
+VERSION = (0, 7, 10)
 
 
 def get_version():

mongoengine/base.py

@@ -11,7 +11,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned
 from queryset import DO_NOTHING
 
 from mongoengine import signals
-from mongoengine.python_support import (PY3, PY25, txt_type,
+from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type,
                                         to_str_keys_recursive)
 
 import pymongo
@@ -53,7 +53,7 @@ class ValidationError(AssertionError):
         self.message = message
 
     def __str__(self):
-        return self.message
+        return txt_type(self.message)
 
     def __repr__(self):
         return '%s(%s,)' % (self.__class__.__name__, self.message)
@@ -121,10 +121,11 @@ class ValidationError(AssertionError):
 def get_document(name):
     doc = _document_registry.get(name, None)
     if not doc:
-        # Possible old style names
-        end = ".%s" % name
+        # Possible old style name
+        single_end = name.split('.')[-1]
+        compound_end = '.%s' % single_end
         possible_match = [k for k in _document_registry.keys()
-                          if k.endswith(end)]
+                          if k.endswith(compound_end) or k == single_end]
         if len(possible_match) == 1:
             doc = _document_registry.get(possible_match.pop(), None)
     if not doc:
@@ -204,8 +205,12 @@ class BaseField(object):
     def __set__(self, instance, value):
         """Descriptor for assigning a value to a field in a document.
         """
-        instance._data[self.name] = value
-        if instance._initialised:
+        changed = False
+        if (self.name not in instance._data or
+           instance._data[self.name] != value):
+            changed = True
+            instance._data[self.name] = value
+        if changed and instance._initialised:
             instance._mark_as_changed(self.name)
 
     def error(self, message="", errors=None, field_name=None):
@@ -235,7 +240,8 @@ class BaseField(object):
         pass
 
     def _validate(self, value):
-        from mongoengine import Document, EmbeddedDocument
+        Document = _import_class('Document')
+        EmbeddedDocument = _import_class('EmbeddedDocument')
         # check choices
         if self.choices:
             is_cls = isinstance(value, (Document, EmbeddedDocument))
@@ -250,7 +256,7 @@ class BaseField(object):
             elif value_to_check not in self.choices:
                 msg = ('Value must be %s of %s' %
                        (err_msg, unicode(self.choices)))
-                self.error()
+                self.error(msg)
 
         # check validation argument
         if self.validation is not None:
@@ -283,7 +289,9 @@ class ComplexBaseField(BaseField):
         if instance is None:
             # Document class being used rather than a document object
             return self
-        from fields import GenericReferenceField, ReferenceField
+
+        ReferenceField = _import_class('ReferenceField')
+        GenericReferenceField = _import_class('GenericReferenceField')
         dereference = self.field is None or isinstance(self.field,
             (GenericReferenceField, ReferenceField))
         if not self._dereference and instance._initialised and dereference:
@@ -310,18 +318,13 @@ class ComplexBaseField(BaseField):
             )
             value._dereferenced = True
             instance._data[self.name] = value
-        return value
 
-    def __set__(self, instance, value):
-        """Descriptor for assigning a value to a field in a document.
-        """
-        instance._data[self.name] = value
-        instance._mark_as_changed(self.name)
+        return value
 
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
         """
-        from mongoengine import Document
+        Document = _import_class('Document')
 
         if isinstance(value, basestring):
             return value
@@ -363,7 +366,7 @@ class ComplexBaseField(BaseField):
     def to_mongo(self, value):
         """Convert a Python type to a MongoDB-compatible type.
         """
-        from mongoengine import Document
+        Document = _import_class("Document")
 
         if isinstance(value, basestring):
             return value
@@ -399,7 +402,7 @@ class ComplexBaseField(BaseField):
                         meta.get('allow_inheritance', ALLOW_INHERITANCE)
                         == False)
                     if allow_inheritance and not self.field:
-                        from fields import GenericReferenceField
+                        GenericReferenceField = _import_class("GenericReferenceField")
                         value_dict[k] = GenericReferenceField().to_mongo(v)
                     else:
                         collection = v._get_collection_name()
@@ -460,7 +463,7 @@ class ComplexBaseField(BaseField):
     @property
     def _dereference(self,):
         if not self.__dereference:
-            from dereference import DeReference
+            DeReference = _import_class("DeReference")
             self.__dereference = DeReference()  # Cached
         return self.__dereference
 
@@ -508,6 +511,10 @@ class DocumentMetaclass(type):
 
         attrs['_is_document'] = attrs.get('_is_document', False)
 
+        # EmbeddedDocuments could have meta data for inheritance
+        if 'meta' in attrs:
+            attrs['_meta'] = attrs.pop('meta')
+
         # Handle document Fields
 
         # Merge all fields from subclasses
@@ -551,8 +558,11 @@ class DocumentMetaclass(type):
 
         # Set _fields and db_field maps
         attrs['_fields'] = doc_fields
-        attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k))
-                                        for k, v in doc_fields.iteritems()])
+        attrs['_fields_ordered'] = tuple(i[1]
+                                         for i in sorted((v.creation_counter, v.name)
+                                                          for v in doc_fields.itervalues()))
+        attrs['_db_field_map'] = dict((k, getattr(v, 'db_field', k))
+                                       for k, v in doc_fields.iteritems())
         attrs['_reverse_db_field_map'] = dict(
                 (v, k) for k, v in attrs['_db_field_map'].iteritems())
 
@@ -571,6 +581,24 @@ class DocumentMetaclass(type):
                 superclasses[base._class_name] = base
                 superclasses.update(base._superclasses)
 
+            if hasattr(base, '_meta'):
+                # Warn if allow_inheritance isn't set and prevent
+                # inheritance of classes where inheritance is set to False
+                allow_inheritance = base._meta.get('allow_inheritance',
+                                                    ALLOW_INHERITANCE)
+                if (not getattr(base, '_is_base_cls', True)
+                    and allow_inheritance is None):
+                    warnings.warn(
+                        "%s uses inheritance, the default for "
+                        "allow_inheritance is changing to off by default. "
+                        "Please add it to the document meta." % name,
+                        FutureWarning
+                    )
+                elif (allow_inheritance == False and
+                      not base._meta.get('abstract')):
+                    raise ValueError('Document %s may not be subclassed' %
+                                      base.__name__)
+
         attrs['_class_name'] = '.'.join(reversed(class_name))
         attrs['_superclasses'] = superclasses
 
@@ -609,17 +637,6 @@ class DocumentMetaclass(type):
                        "field name" % field.name)
                 raise InvalidDocumentError(msg)
 
-        # Merge in exceptions with parent hierarchy
-        exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned)
-        module = attrs.get('__module__')
-        for exc in exceptions_to_merge:
-            name = exc.__name__
-            parents = tuple(getattr(base, name) for base in flattened_bases
-                         if hasattr(base, name)) or (exc,)
-            # Create new exception and set to new_class
-            exception = type(name, parents, {'__module__': module})
-            setattr(new_class, name, exception)
-
         # Add class to the _document_registry
         _document_registry[new_class._class_name] = new_class
 
@@ -745,21 +762,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             if hasattr(base, 'meta'):
                 meta.merge(base.meta)
             elif hasattr(base, '_meta'):
-                # Warn if allow_inheritance isn't set and prevent
-                # inheritance of classes where inheritance is set to False
-                allow_inheritance = base._meta.get('allow_inheritance',
-                                                    ALLOW_INHERITANCE)
-                if not base._is_base_cls and allow_inheritance is None:
-                    warnings.warn(
-                        "%s uses inheritance, the default for "
-                        "allow_inheritance is changing to off by default. "
-                        "Please add it to the document meta." % name,
-                        FutureWarning
-                    )
-                elif (allow_inheritance == False and
-                      not base._meta.get('abstract')):
-                    raise ValueError('Document %s may not be subclassed' %
-                                      base.__name__)
                 meta.merge(base._meta)
 
             # Set collection in the meta if its callable
@@ -823,8 +825,20 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         if not new_class._meta.get('id_field'):
             new_class._meta['id_field'] = 'id'
             new_class._fields['id'] = ObjectIdField(db_field='_id')
+            new_class._fields['id'].name = 'id'
             new_class.id = new_class._fields['id']
 
+        # Merge in exceptions with parent hierarchy
+        exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned)
+        module = attrs.get('__module__')
+        for exc in exceptions_to_merge:
+            name = exc.__name__
+            parents = tuple(getattr(base, name) for base in flattened_bases
+                         if hasattr(base, name)) or (exc,)
+            # Create new exception and set to new_class
+            exception = type(name, parents, {'__module__': module})
+            setattr(new_class, name, exception)
+
         return new_class
 
     @classmethod
@@ -892,7 +906,17 @@ class BaseDocument(object):
     _dynamic_lock = True
     _initialised = False
 
-    def __init__(self, **values):
+    def __init__(self, *args, **values):
+        if args:
+            # Combine positional arguments with named arguments.
+            # We only want named arguments.
+            field = iter(self._fields_ordered)
+            for value in args:
+                name = field.next()
+                if name in values:
+                    raise TypeError("Multiple values for keyword argument '" + name + "'")
+                values[name] = value
+
         signals.pre_init.send(self.__class__, document=self, values=values)
 
         self._data = {}
@@ -936,7 +960,7 @@ class BaseDocument(object):
 
             field = None
             if not hasattr(self, name) and not name.startswith('_'):
-                from fields import DynamicField
+                DynamicField = _import_class("DynamicField")
                 field = DynamicField(db_field=name)
                 field.name = name
                 self._dynamic_fields[name] = field
@@ -949,8 +973,10 @@ class BaseDocument(object):
                 self._data[name] = value
                 if hasattr(self, '_changed_fields'):
                     self._mark_as_changed(name)
+
         if (self._is_document and not self._created and
-            name in self._meta.get('shard_key', tuple())):
+            name in self._meta.get('shard_key', tuple()) and
+            self._data.get(name) != value):
             OperationError = _import_class('OperationError')
             msg = "Shard Keys are immutable. Tried to update %s" % name
             raise OperationError(msg)
@@ -1052,9 +1078,9 @@ class BaseDocument(object):
         # class if unavailable
         class_name = son.get('_cls', cls._class_name)
         data = dict(("%s" % key, value) for key, value in son.items())
-        if PY25:
-            # PY25 cannot handle unicode keys passed to class constructor
-            # example: cls(**data)
+        if not UNICODE_KWARGS:
+            # python 2.6.4 and lower cannot handle unicode keys
+            # passed to class constructor example: cls(**data)
             to_str_keys_recursive(data)
 
         if '_types' in data:
@@ -1109,10 +1135,27 @@ class BaseDocument(object):
             key not in self._changed_fields):
             self._changed_fields.append(key)
 
+    def _clear_changed_fields(self):
+        self._changed_fields = []
+        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+        for field_name, field in self._fields.iteritems():
+            if (isinstance(field, ComplexBaseField) and
+                isinstance(field.field, EmbeddedDocumentField)):
+                field_value = getattr(self, field_name, None)
+                if field_value:
+                    for idx in (field_value if isinstance(field_value, dict)
+                                            else xrange(len(field_value))):
+                        field_value[idx]._clear_changed_fields()
+            elif isinstance(field, EmbeddedDocumentField):
+                field_value = getattr(self, field_name, None)
+                if field_value:
+                    field_value._clear_changed_fields()
+
     def _get_changed_fields(self, key='', inspected=None):
         """Returns a list of all fields that have explicitly been changed.
         """
-        from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument
+        EmbeddedDocument = _import_class("EmbeddedDocument")
+        DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument")
         _changed_fields = []
         _changed_fields += getattr(self, '_changed_fields', [])
 
@@ -1178,7 +1221,7 @@ class BaseDocument(object):
                 for p in parts:
                     if isinstance(d, DBRef):
                         break
-                    elif p.isdigit():
+                    elif isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
                     elif hasattr(d, 'get'):
                         d = d.get(p)
@@ -1192,7 +1235,7 @@ class BaseDocument(object):
 
         # Determine if any changed items were actually unset.
         for path, value in set_data.items():
-            if value or isinstance(value, bool):
+            if value or isinstance(value, (bool, int)):
                 continue
 
             # If we've set a value that ain't the default value dont unset it.
@@ -1209,7 +1252,7 @@ class BaseDocument(object):
                 parts = path.split('.')
                 db_field_name = parts.pop()
                 for p in parts:
-                    if p.isdigit():
+                    if isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
                     elif (hasattr(d, '__getattribute__') and
                           not isinstance(d, dict)):
@@ -1243,7 +1286,9 @@ class BaseDocument(object):
         geo_indices = []
         inspected.append(cls)
 
-        from fields import EmbeddedDocumentField, GeoPointField
+        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+        GeoPointField = _import_class("GeoPointField")
+
         for field in cls._fields.values():
             if not isinstance(field, (EmbeddedDocumentField, GeoPointField)):
                 continue
@@ -1285,7 +1330,10 @@ class BaseDocument(object):
         return value
 
     def __iter__(self):
-        return iter(self._fields)
+        if 'id' in self._fields and 'id' not in self._fields_ordered:
+            return iter(('id', ) + self._fields_ordered)
+
+        return iter(self._fields_ordered)
 
     def __getitem__(self, name):
         """Dictionary-style field access, return a field's value if present.
@@ -1317,10 +1365,11 @@ class BaseDocument(object):
 
     def __repr__(self):
         try:
-            u = txt_type(self)
+            u = self.__str__()
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = '[Bad Unicode data]'
-        return '<%s: %s>' % (self.__class__.__name__, u)
+        repr_type = type(u)
+        return repr_type('<%s: %s>' % (self.__class__.__name__, u))
 
     def __str__(self):
         if hasattr(self, '__unicode__'):
@@ -1328,7 +1377,7 @@ class BaseDocument(object):
                 return self.__unicode__()
             else:
                 return unicode(self).encode('utf-8')
-        return '%s object' % self.__class__.__name__
+        return txt_type('%s object' % self.__class__.__name__)
 
     def __eq__(self, other):
         if isinstance(other, self.__class__) and hasattr(other, 'id'):
@@ -1477,14 +1526,30 @@ def _import_class(cls_name):
     """Cached mechanism for imports"""
     if cls_name in _class_registry:
         return _class_registry.get(cls_name)
-    if cls_name == 'Document':
-        from mongoengine.document import Document as cls
-    elif cls_name == 'EmbeddedDocument':
-        from mongoengine.document import EmbeddedDocument as cls
-    elif cls_name == 'DictField':
-        from mongoengine.fields import DictField as cls
-    elif cls_name == 'OperationError':
-        from queryset import OperationError as cls
 
-    _class_registry[cls_name] = cls
-    return cls
+    doc_classes = ['Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument']
+    field_classes = ['DictField', 'DynamicField', 'EmbeddedDocumentField',
+                     'GenericReferenceField', 'GeoPointField',
+                     'ReferenceField']
+    queryset_classes = ['OperationError']
+    deref_classes = ['DeReference']
+
+    if cls_name in doc_classes:
+        from mongoengine import document as module
+        import_classes = doc_classes
+    elif cls_name in field_classes:
+        from mongoengine import fields as module
+        import_classes = field_classes
+    elif cls_name in queryset_classes:
+        from mongoengine import queryset as module
+        import_classes = queryset_classes
+    elif cls_name in deref_classes:
+        from mongoengine import dereference as module
+        import_classes = deref_classes
+    else:
+        raise ValueError('No import set for: ' % cls_name)
+
+    for cls in import_classes:
+        _class_registry[cls] = getattr(module, cls)
+
+    return _class_registry.get(cls_name)

mongoengine/dereference.py

@@ -31,10 +31,10 @@ class DeReference(object):
             items = [i for i in items]
 
         self.max_depth = max_depth
-
         doc_type = None
-        if instance and instance._fields:
-            doc_type = instance._fields[name]
+
+        if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)):
+            doc_type = instance._fields.get(name)
             if hasattr(doc_type, 'field'):
                 doc_type = doc_type.field
 
@@ -84,7 +84,7 @@ class DeReference(object):
         # Recursively find dbreferences
         depth += 1
         for k, item in iterator:
-            if hasattr(item, '_fields'):
+            if isinstance(item, Document):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -115,7 +115,7 @@ class DeReference(object):
         object_map = {}
         for col, dbrefs in self.reference_map.iteritems():
             keys = object_map.keys()
-            refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys]))
+            refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys]))
             if hasattr(col, 'objects'):  # We have a document class for the refs
                 references = col.objects.in_bulk(refs)
                 for key, doc in references.iteritems():
@@ -134,7 +134,7 @@ class DeReference(object):
                         elif doc_type is None:
                             doc = get_document(
                                 ''.join(x.capitalize()
-                                        for x in col.split('_')))._from_son(ref)
+                                    for x in col.split('_')))._from_son(ref)
                         else:
                             doc = doc_type._from_son(ref)
                         object_map[doc.id] = doc
@@ -166,11 +166,12 @@ class DeReference(object):
                 return self.object_map.get(items['_ref'].id, items)
             elif '_types' in items and '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
-                doc._data = self._attach_objects(doc._data, depth, doc, name)
+                doc._data = self._attach_objects(doc._data, depth, doc, None)
                 return doc
 
         if not hasattr(items, 'items'):
             is_list = True
+            as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
         else:
@@ -187,7 +188,7 @@ class DeReference(object):
 
             if k in self.object_map and not is_list:
                 data[k] = self.object_map[k]
-            elif hasattr(v, '_fields'):
+            elif isinstance(v, Document):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -205,7 +206,7 @@ class DeReference(object):
 
         if instance and name:
             if is_list:
-                return BaseList(data, instance, name)
+                return tuple(data) if as_tuple else BaseList(data, instance, name)
             return BaseDict(data, instance, name)
         depth += 1
         return data

mongoengine/django/auth.py

@@ -3,6 +3,8 @@ import datetime
 from mongoengine import *
 
 from django.utils.encoding import smart_str
+from django.contrib.auth.models import _user_get_all_permissions
+from django.contrib.auth.models import _user_has_perm
 from django.contrib.auth.models import AnonymousUser
 from django.utils.translation import ugettext_lazy as _
 
@@ -104,6 +106,25 @@ class User(Document):
         """
         return check_password(raw_password, self.password)
 
+    def get_all_permissions(self, obj=None):
+        return _user_get_all_permissions(self, obj)
+
+    def has_perm(self, perm, obj=None):
+        """
+        Returns True if the user has the specified permission. This method
+        queries all available auth backends, but returns immediately if any
+        backend returns True. Thus, a user who has permission from a single
+        auth backend is assumed to have permission in general. If an object is
+        provided, permissions for this specific object are checked.
+        """
+
+        # Active superusers have all permissions.
+        if self.is_active and self.is_superuser:
+            return True
+
+        # Otherwise we need to check the backends.
+        return _user_has_perm(self, perm, obj)
+
     @classmethod
     def create_user(cls, username, password, email=None):
         """Create (and save) a new user with the given username, password and

mongoengine/django/sessions.py

@@ -15,15 +15,33 @@ MONGOENGINE_SESSION_DB_ALIAS = getattr(
     settings, 'MONGOENGINE_SESSION_DB_ALIAS',
     DEFAULT_CONNECTION_NAME)
 
+# a setting for the name of the collection used to store sessions
+MONGOENGINE_SESSION_COLLECTION = getattr(
+    settings, 'MONGOENGINE_SESSION_COLLECTION',
+    'django_session')
+
+# a setting for whether session data is stored encoded or not
+MONGOENGINE_SESSION_DATA_ENCODE = getattr(
+    settings, 'MONGOENGINE_SESSION_DATA_ENCODE',
+    True)
 
 class MongoSession(Document):
     session_key = fields.StringField(primary_key=True, max_length=40)
-    session_data = fields.StringField()
+    session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \
+                                        else fields.DictField()
     expire_date = fields.DateTimeField()
 
-    meta = {'collection': 'django_session',
-            'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
-            'allow_inheritance': False}
+    meta = {
+        'collection': MONGOENGINE_SESSION_COLLECTION,
+        'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
+        'allow_inheritance': False,
+        'indexes': [
+            {
+                'fields': ['expire_date'],
+                'expireAfterSeconds': settings.SESSION_COOKIE_AGE
+            }
+        ]
+    }
 
 
 class SessionStore(SessionBase):
@@ -34,7 +52,10 @@ class SessionStore(SessionBase):
         try:
             s = MongoSession.objects(session_key=self.session_key,
                                      expire_date__gt=datetime.now())[0]
-            return self.decode(force_unicode(s.session_data))
+            if MONGOENGINE_SESSION_DATA_ENCODE:
+                return self.decode(force_unicode(s.session_data))
+            else:
+                return s.session_data
         except (IndexError, SuspiciousOperation):
             self.create()
             return {}
@@ -57,7 +78,10 @@ class SessionStore(SessionBase):
         if self.session_key is None:
             self._session_key = self._get_new_session_key()
         s = MongoSession(session_key=self.session_key)
-        s.session_data = self.encode(self._get_session(no_load=must_create))
+        if MONGOENGINE_SESSION_DATA_ENCODE:
+            s.session_data = self.encode(self._get_session(no_load=must_create))
+        else:
+            s.session_data = self._get_session(no_load=must_create)
         s.expire_date = self.get_expiry_date()
         try:
             s.save(force_insert=must_create, safe=True)

mongoengine/document.py

@@ -25,6 +25,14 @@ class EmbeddedDocument(BaseDocument):
     collection.  :class:`~mongoengine.EmbeddedDocument`\ s should be used as
     fields on :class:`~mongoengine.Document`\ s through the
     :class:`~mongoengine.EmbeddedDocumentField` field type.
+
+    A :class:`~mongoengine.EmbeddedDocument` subclass may be itself subclassed,
+    to create a specialised version of the embedded document that will be
+    stored in the same collection. To facilitate this behaviour, `_cls` and
+    `_types` fields are added to documents (hidden though the MongoEngine
+    interface though). To disable this behaviour and remove the dependence on
+    the presence of `_cls` and `_types`, set :attr:`allow_inheritance` to
+    ``False`` in the :attr:`meta` dictionary.
     """
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
@@ -156,7 +164,7 @@ class Document(BaseDocument):
 
     def save(self, safe=True, force_insert=False, validate=True,
              write_options=None,  cascade=None, cascade_kwargs=None,
-             _refs=None):
+             _refs=None, **kwargs):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -261,7 +269,7 @@ class Document(BaseDocument):
         if id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
-        self._changed_fields = []
+        self._clear_changed_fields()
         self._created = False
         signals.post_save.send(self.__class__, document=self, created=created)
         return self
@@ -295,6 +303,16 @@ class Document(BaseDocument):
                 ref.save(**kwargs)
                 ref._changed_fields = []
 
+    @property
+    def _object_key(self):
+        """Dict to identify object in collection
+        """
+        select_dict = {'pk': self.pk}
+        shard_key = self.__class__._meta.get('shard_key', tuple())
+        for k in shard_key:
+            select_dict[k] = getattr(self, k)
+        return select_dict
+
     def update(self, **kwargs):
         """Performs an update on the :class:`~mongoengine.Document`
         A convenience wrapper to :meth:`~mongoengine.QuerySet.update`.
@@ -306,11 +324,7 @@ class Document(BaseDocument):
             raise OperationError('attempt to update a document not yet saved')
 
         # Need to add shard key to query, or you get an error
-        select_dict = {'pk': self.pk}
-        shard_key = self.__class__._meta.get('shard_key', tuple())
-        for k in shard_key:
-            select_dict[k] = getattr(self, k)
-        return self.__class__.objects(**select_dict).update_one(**kwargs)
+        return self.__class__.objects(**self._object_key).update_one(**kwargs)
 
     def delete(self, safe=False):
         """Delete the :class:`~mongoengine.Document` from the database. This
@@ -321,7 +335,7 @@ class Document(BaseDocument):
         signals.pre_delete.send(self.__class__, document=self)
 
         try:
-            self.__class__.objects(pk=self.pk).delete(safe=safe)
+            self.__class__.objects(**self._object_key).delete(safe=safe)
         except pymongo.errors.OperationFailure, err:
             message = u'Could not delete document (%s)' % err.message
             raise OperationError(message)
@@ -347,7 +361,12 @@ class Document(BaseDocument):
         id_field = self._meta['id_field']
         obj = self.__class__.objects(
                 **{id_field: self[id_field]}
-              ).first().select_related(max_depth=max_depth)
+              ).limit(1).select_related(max_depth=max_depth)
+        if obj:
+            obj = obj[0]
+        else:
+            msg = "Reloaded document has been deleted"
+            raise OperationError(msg)
         for field in self._fields:
             setattr(self, field, self._reload(field, obj[field]))
         if self._dynamic:

mongoengine/fields.py

@@ -1,10 +1,12 @@
 import datetime
 import decimal
+import itertools
 import re
 import time
+import urllib2
+import urlparse
 import uuid
 import warnings
-import itertools
 from operator import itemgetter
 
 import gridfs
@@ -25,7 +27,7 @@ except ImportError:
     Image = None
     ImageOps = None
 
-__all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
+__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField',
            'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField',
            'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField',
            'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField',
@@ -101,25 +103,30 @@ class URLField(StringField):
     .. versionadded:: 0.3
     """
 
-    URL_REGEX = re.compile(
-        r'^https?://'
-        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|'
-        r'localhost|'
-        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
-        r'(?::\d+)?'
-        r'(?:/?|[/?]\S+)$', re.IGNORECASE
-    )
+    _URL_REGEX = re.compile(
+        r'^(?:http|ftp)s?://' # http:// or https://
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
+        r'localhost|' #localhost...
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
+        r'(?::\d+)?' # optional port
+        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
 
-    def __init__(self, verify_exists=False, **kwargs):
+    def __init__(self, verify_exists=False, url_regex=None, **kwargs):
         self.verify_exists = verify_exists
+        self.url_regex = url_regex or self._URL_REGEX
         super(URLField, self).__init__(**kwargs)
 
     def validate(self, value):
-        if not URLField.URL_REGEX.match(value):
+        if not self.url_regex.match(value):
             self.error('Invalid URL: %s' % value)
+            return
 
         if self.verify_exists:
-            import urllib2
+            warnings.warn(
+                "The URLField verify_exists argument has intractable security "
+                "and performance issues. Accordingly, it has been deprecated.",
+            DeprecationWarning
+            )
             try:
                 request = urllib2.Request(value)
                 urllib2.urlopen(request)
@@ -136,16 +143,17 @@ class EmailField(StringField):
     EMAIL_REGEX = re.compile(
         r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"  # dot-atom
         r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'  # quoted-string
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE  # domain
+        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE  # domain
     )
 
     def validate(self, value):
         if not EmailField.EMAIL_REGEX.match(value):
             self.error('Invalid Mail-address: %s' % value)
+        super(EmailField, self).validate(value)
 
 
 class IntField(BaseField):
-    """An integer field.
+    """An 32-bit integer field.
     """
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
@@ -178,6 +186,40 @@ class IntField(BaseField):
         return int(value)
 
 
+class LongField(BaseField):
+    """An 64-bit integer field.
+    """
+
+    def __init__(self, min_value=None, max_value=None, **kwargs):
+        self.min_value, self.max_value = min_value, max_value
+        super(LongField, self).__init__(**kwargs)
+
+    def to_python(self, value):
+        try:
+            value = long(value)
+        except ValueError:
+            pass
+        return value
+
+    def validate(self, value):
+        try:
+            value = long(value)
+        except:
+            self.error('%s could not be converted to long' % value)
+
+        if self.min_value is not None and value < self.min_value:
+            self.error('Long value is too small')
+
+        if self.max_value is not None and value > self.max_value:
+            self.error('Long value is too large')
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+
+        return long(value)
+
+
 class FloatField(BaseField):
     """An floating point number field.
     """
@@ -709,6 +751,10 @@ class ReferenceField(BaseField):
 
         Bar.register_delete_rule(Foo, 'bar', NULLIFY)
 
+    .. note ::
+        `reverse_delete_rules` do not trigger pre / post delete signals to be
+        triggered.
+
     .. versionchanged:: 0.5 added `reverse_delete_rule`
     """
 
@@ -766,7 +812,7 @@ class ReferenceField(BaseField):
     def to_mongo(self, document):
         if isinstance(document, DBRef):
             if not self.dbref:
-                return "%s" % DBRef.id
+                return document.id
             return document
         elif not self.dbref and isinstance(document, basestring):
             return document
@@ -788,7 +834,7 @@ class ReferenceField(BaseField):
             collection = self.document_type._get_collection_name()
             return DBRef(collection, id_)
 
-        return "%s" % id_
+        return id_
 
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
@@ -1326,7 +1372,7 @@ class SequenceField(IntField):
 
     .. versionadded:: 0.5
     """
-    def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs):
+    def __init__(self, collection_name=None, db_alias=None, sequence_name=None, *args, **kwargs):
         self.collection_name = collection_name or 'mongoengine.counters'
         self.db_alias = db_alias or DEFAULT_CONNECTION_NAME
         self.sequence_name = sequence_name
@@ -1336,7 +1382,7 @@ class SequenceField(IntField):
         """
         Generate and Increment the counter
         """
-        sequence_name = self.sequence_name or self.owner_document._get_collection_name()
+        sequence_name = self.get_sequence_name()
         sequence_id = "%s.%s" % (sequence_name, self.name)
         collection = get_db(alias=self.db_alias)[self.collection_name]
         counter = collection.find_and_modify(query={"_id": sequence_id},
@@ -1345,6 +1391,16 @@ class SequenceField(IntField):
                                              upsert=True)
         return counter['next']
 
+    def get_sequence_name(self):
+        if self.sequence_name:
+            return self.sequence_name
+        owner = self.owner_document
+        if issubclass(owner, Document):
+            return owner._get_collection_name()
+        else:
+            return ''.join('_%s' % c if c.isupper() else c
+                            for c in owner._class_name).strip('_').lower()
+
     def __get__(self, instance, owner):
 
         if instance is None:

mongoengine/python_support.py

@@ -4,6 +4,7 @@ import sys
 
 PY3 = sys.version_info[0] == 3
 PY25 = sys.version_info[:2] == (2, 5)
+UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264
 
 if PY3:
     import codecs

mongoengine/queryset.py

@@ -7,10 +7,11 @@ import operator
 from collections import defaultdict
 from functools import partial
 
-from mongoengine.python_support import product, reduce
+from mongoengine.python_support import product, reduce, PY3
 
 import pymongo
 from bson.code import Code
+from bson.son import SON
 
 from mongoengine import signals
 
@@ -218,7 +219,7 @@ class QNode(object):
     def _combine(self, other, operation):
         """Combine this node with another node into a QCombination object.
         """
-        if other.empty:
+        if getattr(other, 'empty', True):
             return self
 
         if self.empty:
@@ -353,6 +354,8 @@ class QuerySet(object):
         self._slave_okay = False
         self._iter = False
         self._scalar = []
+        self._as_pymongo = False
+        self._as_pymongo_coerce = False
 
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -364,6 +367,10 @@ class QuerySet(object):
         self._skip = None
         self._hint = -1  # Using -1 as None is a valid value for hint
 
+    def __deepcopy__(self, memo):
+        """Essential for chained queries with ReferenceFields involved"""
+        return self.clone()
+
     def clone(self):
         """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`
 
@@ -372,8 +379,8 @@ class QuerySet(object):
         c = self.__class__(self._document, self._collection_obj)
 
         copy_props = ('_initial_query', '_query_obj', '_where_clause',
-                    '_loaded_fields', '_ordering', '_snapshot',
-                    '_timeout', '_limit', '_skip', '_slave_okay', '_hint')
+                      '_loaded_fields', '_ordering', '_snapshot', '_timeout',
+                      '_limit', '_skip', '_slave_okay', '_hint')
 
         for prop in copy_props:
             val = getattr(self, prop)
@@ -390,7 +397,7 @@ class QuerySet(object):
         return self._mongo_query
 
     def ensure_index(self, key_or_list, drop_dups=False, background=False,
-        **kwargs):
+                     **kwargs):
         """Ensure that the given indexes are in place.
 
         :param key_or_list: a single index key or a list of index keys (to
@@ -398,12 +405,13 @@ class QuerySet(object):
             or a **-** to determine the index ordering
         """
         index_spec = QuerySet._build_index_spec(self._document, key_or_list)
-        self._collection.ensure_index(
-            index_spec['fields'],
-            drop_dups=drop_dups,
-            background=background,
-            sparse=index_spec.get('sparse', False),
-            unique=index_spec.get('unique', False))
+        index_spec = index_spec.copy()
+        fields = index_spec.pop('fields')
+        index_spec['drop_dups'] = drop_dups
+        index_spec['background'] = background
+        index_spec.update(kwargs)
+
+        self._collection.ensure_index(fields, **index_spec)
         return self
 
     def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query):
@@ -472,12 +480,14 @@ class QuerySet(object):
 
         # Ensure document-defined indexes are created
         if self._document._meta['index_specs']:
-            for spec in self._document._meta['index_specs']:
-                types_indexed = types_indexed or includes_types(spec['fields'])
+            index_spec = self._document._meta['index_specs']
+            for spec in index_spec:
+                spec = spec.copy()
+                fields = spec.pop('fields')
+                types_indexed = types_indexed or includes_types(fields)
                 opts = index_opts.copy()
-                opts['unique'] = spec.get('unique', False)
-                opts['sparse'] = spec.get('sparse', False)
-                self._collection.ensure_index(spec['fields'],
+                opts.update(spec)
+                self._collection.ensure_index(fields,
                     background=background, **opts)
 
         # If _types is being used (for polymorphism), it needs an index,
@@ -498,8 +508,10 @@ class QuerySet(object):
         """
         if isinstance(spec, basestring):
             spec = {'fields': [spec]}
-        if isinstance(spec, (list, tuple)):
-            spec = {'fields': spec}
+        elif isinstance(spec, (list, tuple)):
+            spec = {'fields': list(spec)}
+        elif isinstance(spec, dict):
+            spec = dict(spec)
 
         index_list = []
         direction = None
@@ -510,6 +522,10 @@ class QuerySet(object):
         use_types = allow_inheritance and not spec.get('sparse', False)
 
         for key in spec['fields']:
+            # If inherited spec continue
+            if isinstance(key, (list, tuple)):
+                continue
+
             # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from *
             direction = pymongo.ASCENDING
             if key.startswith("-"):
@@ -599,11 +615,13 @@ class QuerySet(object):
             if self._where_clause:
                 self._cursor_obj.where(self._where_clause)
 
-            # apply default ordering
             if self._ordering:
+                # Apply query ordering
                 self._cursor_obj.sort(self._ordering)
             elif self._document._meta['ordering']:
+                # Otherwise, apply the ordering from the document model
                 self.order_by(*self._document._meta['ordering'])
+                self._cursor_obj.sort(self._ordering)
 
             if self._limit is not None:
                 self._cursor_obj.limit(self._limit - (self._skip or 0))
@@ -691,7 +709,7 @@ class QuerySet(object):
 
         mongo_query = {}
         merge_query = defaultdict(list)
-        for key, value in query.items():
+        for key, value in sorted(query.items()):
             if key == "__raw__":
                 mongo_query.update(value)
                 continue
@@ -795,7 +813,6 @@ class QuerySet(object):
                     mongo_query['$and'].append(value)
                 else:
                     mongo_query['$and'] = value
-
         return mongo_query
 
     def get(self, *q_objs, **query):
@@ -918,7 +935,7 @@ class QuerySet(object):
             if not isinstance(doc, self._document):
                 msg = "Some documents inserted aren't instances of %s" % str(self._document)
                 raise OperationError(msg)
-            if doc.pk:
+            if doc.pk and not doc._created:
                 msg = "Some documents have ObjectIds use doc.update() instead"
                 raise OperationError(msg)
             raw.append(doc.to_mongo())
@@ -977,6 +994,9 @@ class QuerySet(object):
             for doc in docs:
                 doc_map[doc['_id']] = self._get_scalar(
                         self._document._from_son(doc))
+        elif self._as_pymongo:
+            for doc in docs:
+                doc_map[doc['_id']] = self._get_as_pymongo(doc)
         else:
             for doc in docs:
                 doc_map[doc['_id']] = self._document._from_son(doc)
@@ -993,6 +1013,9 @@ class QuerySet(object):
             if self._scalar:
                 return self._get_scalar(self._document._from_son(
                         self._cursor.next()))
+            if self._as_pymongo:
+                return self._get_as_pymongo(self._cursor.next())
+
             return self._document._from_son(self._cursor.next())
         except StopIteration, e:
             self.rewind()
@@ -1175,6 +1198,8 @@ class QuerySet(object):
             if self._scalar:
                 return self._get_scalar(self._document._from_son(
                         self._cursor[key]))
+            if self._as_pymongo:
+                return self._get_as_pymongo(self._cursor.next())
             return self._document._from_son(self._cursor[key])
         raise AttributeError
 
@@ -1187,8 +1212,11 @@ class QuerySet(object):
         .. versionchanged:: 0.5 - Fixed handling references
         .. versionchanged:: 0.6 - Improved db_field refrence handling
         """
-        return self._dereference(self._cursor.distinct(field), 1,
-                                 name=field, instance=self._document)
+        try:
+            field = self._fields_to_dbfields([field]).pop()
+        finally:
+            return self._dereference(self._cursor.distinct(field), 1,
+                                     name=field, instance=self._document)
 
     def only(self, *fields):
         """Load only a subset of this document's fields. ::
@@ -1293,7 +1321,8 @@ class QuerySet(object):
             key_list.append((key, direction))
 
         self._ordering = key_list
-        self._cursor.sort(key_list)
+        if self._cursor_obj:
+            self._cursor_obj.sort(key_list)
         return self
 
     def explain(self, format=False):
@@ -1343,6 +1372,12 @@ class QuerySet(object):
         """
         doc = self._document
 
+        # Handle deletes where skips or limits have been applied
+        if self._skip or self._limit:
+            for doc in self:
+                doc.delete()
+            return
+
         delete_rules = doc._meta.get('delete_rules') or {}
         # Check for DENY rules before actually deleting/nullifying any other
         # references
@@ -1380,6 +1415,8 @@ class QuerySet(object):
         """
         operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all',
                      'pull', 'pull_all', 'add_to_set']
+        match_operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod',
+                           'all', 'size', 'exists', 'not']
 
         mongo_update = {}
         for key, value in update.items():
@@ -1403,6 +1440,10 @@ class QuerySet(object):
                 elif op == 'add_to_set':
                     op = op.replace('_to_set', 'ToSet')
 
+            match = None
+            if parts[-1] in match_operators:
+                match = parts.pop()
+
             if _doc_cls:
                 # Switch field names to proper names [set in Field(name='foo')]
                 fields = QuerySet._lookup_field(_doc_cls, parts)
@@ -1436,16 +1477,22 @@ class QuerySet(object):
                     elif field.required or value is not None:
                         value = field.prepare_query_value(op, value)
 
+            if match:
+                match = '$' + match
+                value = {match: value}
+
             key = '.'.join(parts)
 
             if not op:
-                raise InvalidQueryError("Updates must supply an operation eg: set__FIELD=value")
+                raise InvalidQueryError("Updates must supply an operation "
+                                        "eg: set__FIELD=value")
 
             if 'pull' in op and '.' in key:
                 # Dot operators don't work on pull operations
                 # it uses nested dict syntax
                 if op == 'pullAll':
-                    raise InvalidQueryError("pullAll operations only support a single field depth")
+                    raise InvalidQueryError("pullAll operations only support "
+                                            "a single field depth")
 
                 parts.reverse()
                 for key in parts:
@@ -1553,6 +1600,48 @@ class QuerySet(object):
 
         return tuple(data)
 
+    def _get_as_pymongo(self, row):
+        # Extract which fields paths we should follow if .fields(...) was
+        # used. If not, handle all fields.
+        if not getattr(self, '__as_pymongo_fields', None):
+            self.__as_pymongo_fields = []
+            for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']):
+                self.__as_pymongo_fields.append(field)
+                while '.' in field:
+                    field, _ = field.rsplit('.', 1)
+                    self.__as_pymongo_fields.append(field)
+
+        all_fields = not self.__as_pymongo_fields
+
+        def clean(data, path=None):
+            path = path or ''
+
+            if isinstance(data, dict):
+                new_data = {}
+                for key, value in data.iteritems():
+                    new_path = '%s.%s' % (path, key) if path else key
+                    if all_fields or new_path in self.__as_pymongo_fields:
+                        new_data[key] = clean(value, path=new_path)
+                data = new_data
+            elif isinstance(data, list):
+                data = [clean(d, path=path) for d in data]
+            else:
+                if self._as_pymongo_coerce:
+                    # If we need to coerce types, we need to determine the
+                    # type of this field and use the corresponding .to_python(...)
+                    from mongoengine.fields import EmbeddedDocumentField
+                    obj = self._document
+                    for chunk in path.split('.'):
+                        obj = getattr(obj, chunk, None)
+                        if obj is None:
+                            break
+                        elif isinstance(obj, EmbeddedDocumentField):
+                            obj = obj.document_type
+                    if obj and data is not None:
+                        data = obj.to_python(data)
+            return data
+        return clean(row)
+
     def scalar(self, *fields):
         """Instead of returning Document instances, return either a specific
         value or a tuple of values in order.
@@ -1575,6 +1664,16 @@ class QuerySet(object):
         """An alias for scalar"""
         return self.scalar(*fields)
 
+    def as_pymongo(self, coerce_types=False):
+        """Instead of returning Document instances, return raw values from
+        pymongo.
+
+        :param coerce_type: Field types (if applicable) would be use to coerce types.
+        """
+        self._as_pymongo = True
+        self._as_pymongo_coerce = coerce_types
+        return self
+
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where
         *fieldname* is the Python name of a field, *fieldname* will be

python-mongoengine.spec

@@ -5,7 +5,7 @@
 %define srcname mongoengine
 
 Name:           python-%{srcname}
-Version:        0.7rc1
+Version:        0.7.10
 Release:        1%{?dist}
 Summary:        A Python Document-Object Mapper for working with MongoDB
 
@@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT
 # %{python_sitearch}/*
 
 %changelog
-* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html
+* See: http://docs.mongoengine.org/en/latest/changelog.html

setup.py

@@ -8,8 +8,8 @@ try:
 except ImportError:
     pass
 
-DESCRIPTION = "A Python Document-Object Mapper for working with MongoDB"
-
+DESCRIPTION = """MongoEngine is a Python Object-Document
+Mapper for working with MongoDB."""
 LONG_DESCRIPTION = None
 try:
     LONG_DESCRIPTION = open('README.rst').read()
@@ -58,7 +58,7 @@ if sys.version_info[0] == 3:
         extra_opts['packages'].append("tests")
         extra_opts['package_data'] = {"tests": ["mongoengine.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL']
     extra_opts['packages'] = find_packages(exclude=('tests',))
 
 setup(name='mongoengine',
@@ -68,6 +68,7 @@ setup(name='mongoengine',
       maintainer="Ross Lawley",
       maintainer_email="ross.lawley@{nospam}gmail.com",
       url='http://mongoengine.org/',
+      download_url='https://github.com/MongoEngine/mongoengine/tarball/master',
       license='MIT',
       include_package_data=True,
       description=DESCRIPTION,

tests/test_all_warnings.py

@@ -53,7 +53,7 @@ class TestWarnings(unittest.TestCase):
         p2.parent = p1
         p2.save(cascade=False)
 
-        self.assertEqual(len(self.warning_list), 1)
+        self.assertTrue(len(self.warning_list) > 0)
         warning = self.warning_list[0]
         self.assertEqual(FutureWarning, warning["category"])
         self.assertTrue("ReferenceFields will default to using ObjectId"
@@ -76,7 +76,9 @@ class TestWarnings(unittest.TestCase):
         p2.parent.name = "Poppa Wilson"
         p2.save()
 
-        self.assertEqual(len(self.warning_list), 1)
+        self.assertTrue(len(self.warning_list) > 0)
+        if len(self.warning_list) > 1:
+            print self.warning_list
         warning = self.warning_list[0]
         self.assertEqual(FutureWarning, warning["category"])
         self.assertTrue("Cascading saves will default to off in 0.8"

tests/test_dereference.py

@@ -1,7 +1,8 @@
+# -*- coding: utf-8 -*-
 from __future__ import with_statement
 import unittest
 
-from bson import DBRef
+from bson import DBRef, ObjectId
 
 from mongoengine import *
 from mongoengine.connection import get_db
@@ -42,6 +43,12 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             self.assertEqual(q, 1)
 
+            len(group_obj._data['members'])
+            self.assertEqual(q, 1)
+
+            len(group_obj.members)
+            self.assertEqual(q, 2)
+
             [m for m in group_obj.members]
             self.assertEqual(q, 2)
 
@@ -84,6 +91,7 @@ class FieldTest(unittest.TestCase):
 
         group = Group(members=User.objects)
         group.save()
+        group.reload()  # Confirm reload works
 
         with query_counter() as q:
             self.assertEqual(q, 0)
@@ -178,8 +186,9 @@ class FieldTest(unittest.TestCase):
 
         # Migrate the data
         for g in Group.objects():
-            g.author = g.author
-            g.members = g.members
+            # Explicitly mark as changed so resets
+            g._mark_as_changed('author')
+            g._mark_as_changed('members')
             g.save()
 
         group = Group.objects.first()
@@ -187,8 +196,8 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(group.members, [user])
 
         raw_data = Group._get_collection().find_one()
-        self.assertTrue(isinstance(raw_data['author'], basestring))
-        self.assertTrue(isinstance(raw_data['members'][0], basestring))
+        self.assertTrue(isinstance(raw_data['author'], ObjectId))
+        self.assertTrue(isinstance(raw_data['members'][0], ObjectId))
 
     def test_recursive_reference(self):
         """Ensure that ReferenceFields can reference their own documents.
@@ -990,3 +999,57 @@ class FieldTest(unittest.TestCase):
         msg = Message.objects.get(id=1)
         self.assertEqual(0, msg.comments[0].id)
         self.assertEqual(1, msg.comments[1].id)
+
+    def test_tuples_as_tuples(self):
+        """
+        Ensure that tuples remain tuples when they are
+        inside a ComplexBaseField
+        """
+        from mongoengine.base import BaseField
+        class EnumField(BaseField):
+            def __init__(self, **kwargs):
+                super(EnumField,self).__init__(**kwargs)
+
+            def to_mongo(self, value):
+                return value
+
+            def to_python(self, value):
+                return tuple(value)
+
+        class TestDoc(Document):
+            items = ListField(EnumField())
+
+        TestDoc.drop_collection()
+        tuples = [(100, 'Testing')]
+        doc = TestDoc()
+        doc.items = tuples
+        doc.save()
+        x = TestDoc.objects().get()
+        self.assertTrue(x is not None)
+        self.assertTrue(len(x.items) == 1)
+        self.assertTrue(tuple(x.items[0]) in tuples)
+        self.assertTrue(x.items[0] in tuples)
+
+    def test_non_ascii_pk(self):
+        """
+        Ensure that dbref conversion to string does not fail when
+        non-ascii characters are used in primary key
+        """
+        class Brand(Document):
+            title = StringField(max_length=255, primary_key=True)
+
+        class BrandGroup(Document):
+            title = StringField(max_length=255, primary_key=True)
+            brands = ListField(ReferenceField("Brand", dbref=True))
+
+        Brand.drop_collection()
+        BrandGroup.drop_collection()
+
+        brand1 = Brand(title="Moschino").save()
+        brand2 = Brand(title=u"Денис Симачёв").save()
+
+        BrandGroup(title="top_brands", brands=[brand1, brand2]).save()
+        brand_groups = BrandGroup.objects().all()
+
+        self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands]))
+

tests/test_document.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import with_statement
 import bson
 import os
@@ -7,6 +8,7 @@ import sys
 import unittest
 import uuid
 import warnings
+import operator
 
 from nose.plugins.skip import SkipTest
 from datetime import datetime
@@ -14,9 +16,9 @@ from datetime import datetime
 from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest
 
 from mongoengine import *
-from mongoengine.base import NotRegistered, InvalidDocumentError
+from mongoengine.base import NotRegistered, InvalidDocumentError, get_document
 from mongoengine.queryset import InvalidQueryError
-from mongoengine.connection import get_db
+from mongoengine.connection import get_db, get_connection
 
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png')
 
@@ -85,6 +87,22 @@ class DocumentTest(unittest.TestCase):
         # Ensure Document isn't treated like an actual document
         self.assertFalse(hasattr(Document, '_fields'))
 
+    def test_repr(self):
+        """Ensure that unicode representation works
+        """
+        class Article(Document):
+            title = StringField()
+
+            def __unicode__(self):
+                return self.title
+
+        Article.drop_collection()
+
+        Article(title=u'привет мир').save()
+
+        self.assertEqual('<Article: привет мир>', repr(Article.objects.first()))
+        self.assertEqual('[<Article: привет мир>]', repr(Article.objects.all()))
+
     def test_collection_naming(self):
         """Ensure that a collection with a specified name may be used.
         """
@@ -338,7 +356,6 @@ class DocumentTest(unittest.TestCase):
                 meta = {'allow_inheritance': False}
         self.assertRaises(ValueError, create_employee_class)
 
-
     def test_allow_inheritance_abstract_document(self):
         """Ensure that abstract documents can set inheritance rules and that
         _cls and _types will not be used.
@@ -366,6 +383,31 @@ class DocumentTest(unittest.TestCase):
 
         Animal.drop_collection()
 
+    def test_allow_inheritance_embedded_document(self):
+
+        # Test the same for embedded documents
+        class Comment(EmbeddedDocument):
+            content = StringField()
+            meta = {'allow_inheritance': False}
+
+        def create_special_comment():
+            class SpecialComment(Comment):
+                pass
+
+        self.assertRaises(ValueError, create_special_comment)
+
+        comment = Comment(content='test')
+        self.assertFalse('_cls' in comment.to_mongo())
+        self.assertFalse('_types' in comment.to_mongo())
+
+        class Comment(EmbeddedDocument):
+            content = StringField()
+            meta = {'allow_inheritance': True}
+
+        comment = Comment(content='test')
+        self.assertTrue('_cls' in comment.to_mongo())
+        self.assertTrue('_types' in comment.to_mongo())
+
     def test_document_inheritance(self):
         """Ensure mutliple inheritance of abstract docs works
         """
@@ -396,6 +438,9 @@ class DocumentTest(unittest.TestCase):
                 'indexes': ['name']
             }
 
+        self.assertEqual(Animal._meta['index_specs'],
+                         [{'fields': [('_types', 1), ('name', 1)]}])
+
         Animal.drop_collection()
 
         dog = Animal(name='dog')
@@ -408,7 +453,8 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info)
+        self.assertEqual([[('_id', 1)], [('_types', 1), ('name', 1)]],
+                         sorted(info, key=operator.itemgetter(0)))
 
         # Turn off inheritance
         class Animal(Document):
@@ -417,6 +463,9 @@ class DocumentTest(unittest.TestCase):
                 'allow_inheritance': False,
                 'indexes': ['name']
             }
+
+        self.assertEqual(Animal._meta['index_specs'],
+                         [{'fields': [('name', 1)]}])
         collection.update({}, {"$unset": {"_types": 1, "_cls": 1}},  multi=True)
 
         # Confirm extra data is removed
@@ -426,7 +475,8 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info)
+        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]],
+            sorted(info, key=operator.itemgetter(0)))
 
         info = collection.index_information()
         indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])]
@@ -435,14 +485,16 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)]], info)
+        self.assertEqual([[(u'_id', 1)]],
+            sorted(info, key=operator.itemgetter(0)))
 
         # Recreate indexes
         dog = Animal.objects.first()
         dog.save()
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info)
+        self.assertEqual([[(u'_id', 1)], [(u'name', 1),]],
+            sorted(info, key=operator.itemgetter(0)))
 
         Animal.drop_collection()
 
@@ -634,6 +686,12 @@ class DocumentTest(unittest.TestCase):
                 'allow_inheritance': True
             }
 
+        self.assertEqual(BlogPost._meta['index_specs'],
+                         [{'fields': [('_types', 1), ('addDate', -1)]},
+                          {'fields': [('tags', 1)]},
+                          {'fields': [('_types', 1), ('category', 1),
+                                      ('addDate', -1)]}])
+
         BlogPost.drop_collection()
 
         info = BlogPost.objects._collection.index_information()
@@ -657,6 +715,13 @@ class DocumentTest(unittest.TestCase):
             title = StringField()
             meta = {'indexes': ['title']}
 
+        self.assertEqual(ExtendedBlogPost._meta['index_specs'],
+                         [{'fields': [('_types', 1), ('addDate', -1)]},
+                          {'fields': [('tags', 1)]},
+                          {'fields': [('_types', 1), ('category', 1),
+                                      ('addDate', -1)]},
+                          {'fields': [('_types', 1), ('title', 1)]}])
+
         BlogPost.drop_collection()
 
         list(ExtendedBlogPost.objects)
@@ -669,6 +734,46 @@ class DocumentTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_inherited_index(self):
+        """Ensure index specs are inhertited correctly"""
+
+        class A(Document):
+            title = StringField()
+            meta = {
+                'indexes': [
+                        {
+                        'fields': ('title',),
+                        },
+                ],
+                'allow_inheritance': True,
+                }
+
+        class B(A):
+            description = StringField()
+
+        self.assertEqual(A._meta['index_specs'], B._meta['index_specs'])
+        self.assertEqual([{'fields': [('_types', 1), ('title', 1)]}],
+                         A._meta['index_specs'])
+
+    def test_build_index_spec_is_not_destructive(self):
+
+        class MyDoc(Document):
+            keywords = StringField()
+
+            meta = {
+                'indexes': ['keywords'],
+                'allow_inheritance': False
+            }
+
+        self.assertEqual(MyDoc._meta['index_specs'],
+                         [{'fields': [('keywords', 1)]}])
+
+        # Force index creation
+        MyDoc.objects._ensure_indexes()
+
+        self.assertEqual(MyDoc._meta['index_specs'],
+                        [{'fields': [('keywords', 1)]}])
+
     def test_db_field_load(self):
         """Ensure we load data correctly
         """
@@ -729,6 +834,9 @@ class DocumentTest(unittest.TestCase):
                 'allow_inheritance': False
             }
 
+        self.assertEqual([{'fields': [('rank.title', 1)]}],
+                        Person._meta['index_specs'])
+
         Person.drop_collection()
 
         # Indexes are lazy so use list() to perform query
@@ -747,6 +855,10 @@ class DocumentTest(unittest.TestCase):
                     '*location.point',
                 ],
             }
+
+        self.assertEqual([{'fields': [('location.point', '2d')]}],
+                        Place._meta['index_specs'])
+
         Place.drop_collection()
 
         info = Place.objects._collection.index_information()
@@ -772,6 +884,10 @@ class DocumentTest(unittest.TestCase):
                 ],
             }
 
+        self.assertEqual([{'fields': [('addDate', -1)], 'unique': True,
+                          'sparse': True, 'types': False}],
+                        BlogPost._meta['index_specs'])
+
         BlogPost.drop_collection()
 
         info = BlogPost.objects._collection.index_information()
@@ -813,7 +929,7 @@ class DocumentTest(unittest.TestCase):
 
         self.assertEqual(1, Person.objects.count())
         info = Person.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1'])
+        self.assertEqual(sorted(info.keys()), ['_id_', '_types_1_name_1', '_types_1_user_guid_1'])
         Person.drop_collection()
 
     def test_disable_index_creation(self):
@@ -857,7 +973,7 @@ class DocumentTest(unittest.TestCase):
         BlogPost.drop_collection()
 
         info = BlogPost.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_'])
+        self.assertEqual(sorted(info.keys()), [ '_id_', '_types_1_date.yr_-1'])
         BlogPost.drop_collection()
 
     def test_list_embedded_document_index(self):
@@ -880,7 +996,8 @@ class DocumentTest(unittest.TestCase):
 
         info = BlogPost.objects._collection.index_information()
         # we don't use _types in with list fields by default
-        self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1'])
+        self.assertEqual(sorted(info.keys()),
+                         ['_id_', '_types_1', 'tags.tag_1'])
 
         post1 = BlogPost(title="Embedded Indexes tests in place",
                         tags=[Tag(name="about"), Tag(name="time")]
@@ -897,7 +1014,7 @@ class DocumentTest(unittest.TestCase):
             recursive_obj = EmbeddedDocumentField(RecursiveObject)
 
         info = RecursiveDocument.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_id_', '_types_1'])
+        self.assertEqual(sorted(info.keys()), ['_id_', '_types_1'])
 
     def test_geo_indexes_recursion(self):
 
@@ -1102,6 +1219,32 @@ class DocumentTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_ttl_indexes(self):
+
+        class Log(Document):
+            created = DateTimeField(default=datetime.now)
+            meta = {
+                'indexes': [
+                    {'fields': ['created'], 'expireAfterSeconds': 3600}
+                ]
+            }
+
+        Log.drop_collection()
+
+        if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3:
+            raise SkipTest('pymongo needs to be 2.3 or higher for this test')
+
+        connection = get_connection()
+        version_array = connection.server_info()['versionArray']
+        if version_array[0] < 2 and version_array[1] < 2:
+            raise SkipTest('MongoDB needs to be 2.2 or higher for this test')
+
+        # Indexes are lazy so use list() to perform query
+        list(Log.objects)
+        info = Log.objects._collection.index_information()
+        self.assertEqual(3600,
+                info['_types_1_created_1']['expireAfterSeconds'])
+
     def test_unique_and_indexes(self):
         """Ensure that 'unique' constraints aren't overridden by
         meta.indexes.
@@ -1199,7 +1342,6 @@ class DocumentTest(unittest.TestCase):
 
         User.drop_collection()
 
-
     def test_document_not_registered(self):
 
         class Place(Document):
@@ -1224,6 +1366,19 @@ class DocumentTest(unittest.TestCase):
             print Place.objects.all()
         self.assertRaises(NotRegistered, query_without_importing_nice_place)
 
+    def test_document_registry_regressions(self):
+
+        class Location(Document):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        class Area(Location):
+            location = ReferenceField('Location', dbref=True)
+
+        Location.drop_collection()
+
+        self.assertEquals(Area, get_document("Area"))
+        self.assertEquals(Area, get_document("Location.Area"))
 
     def test_creation(self):
         """Ensure that document may be created using keyword arguments.
@@ -1232,6 +1387,28 @@ class DocumentTest(unittest.TestCase):
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 30)
 
+    def test_positional_creation(self):
+        """Ensure that document may be created using positional arguments.
+        """
+        person = self.Person("Test User", 42)
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 42)
+
+    def test_mixed_creation(self):
+        """Ensure that document may be created using mixed arguments.
+        """
+        person = self.Person("Test User", age=42)
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 42)
+
+    def test_bad_mixed_creation(self):
+        """Ensure that document gives correct error when duplicating arguments
+        """
+        def construct_bad_instance():
+            return self.Person("Test User", 42, name="Bad User")
+
+        self.assertRaises(TypeError, construct_bad_instance)
+
     def test_to_dbref(self):
         """Ensure that you can get a dbref of a document"""
         person = self.Person(name="Test User", age=30)
@@ -1258,6 +1435,17 @@ class DocumentTest(unittest.TestCase):
         self.assertEqual(person.name, "Mr Test User")
         self.assertEqual(person.age, 21)
 
+    def test_reload_sharded(self):
+        class Animal(Document):
+            superphylum = StringField()
+            meta = {'shard_key': ('superphylum',)}
+
+        Animal.drop_collection()
+        doc = Animal(superphylum = 'Deuterostomia')
+        doc.save()
+        doc.reload()
+        Animal.drop_collection()
+
     def test_reload_referencing(self):
         """Ensures reloading updates weakrefs correctly
         """
@@ -1363,8 +1551,10 @@ class DocumentTest(unittest.TestCase):
         doc.validate()
         keys = doc._data.keys()
         self.assertEqual(2, len(keys))
-        self.assertTrue(None in keys)
         self.assertTrue('e' in keys)
+        # Ensure that the _id field has the right id
+        self.assertTrue('id' in keys)
+        self.assertEqual(doc._data.get('id'), doc.id)
 
     def test_save(self):
         """Ensure that a document may be saved in the database.
@@ -2559,7 +2749,7 @@ class DocumentTest(unittest.TestCase):
 
         Person.drop_collection()
 
-        self.assertEqual(Person._fields.keys(), ['name', 'id'])
+        self.assertEqual(sorted(Person._fields.keys()), ['id', 'name'])
 
         Person(name="Rozza").save()
 
@@ -3202,6 +3392,60 @@ class DocumentTest(unittest.TestCase):
                                         }
                                     ) ]), "1,2")
 
+    def test_data_contains_id_field(self):
+        """Ensure that asking for _data returns 'id'
+        """
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        Person(name="Harry Potter").save()
+
+        person = Person.objects.first()
+        self.assertTrue('id' in person._data.keys())
+        self.assertEqual(person._data.get('id'), person.id)
+
+    def test_complex_nesting_document_and_embedded_document(self):
+
+        class Macro(EmbeddedDocument):
+            value = DynamicField(default="UNDEFINED")
+
+        class Parameter(EmbeddedDocument):
+            macros = MapField(EmbeddedDocumentField(Macro))
+
+            def expand(self):
+                self.macros["test"] = Macro()
+
+        class Node(Document):
+            parameters = MapField(EmbeddedDocumentField(Parameter))
+
+            def expand(self):
+                self.flattened_parameter = {}
+                for parameter_name, parameter in self.parameters.iteritems():
+                    parameter.expand()
+
+        class System(Document):
+            name = StringField(required=True)
+            nodes = MapField(ReferenceField(Node, dbref=False))
+
+            def save(self, *args, **kwargs):
+                for node_name, node in self.nodes.iteritems():
+                    node.expand()
+                    node.save(*args, **kwargs)
+                super(System, self).save(*args, **kwargs)
+
+        System.drop_collection()
+        Node.drop_collection()
+
+        system = System(name="system")
+        system.nodes["node"] = Node()
+        system.save()
+        system.nodes["node"].parameters["param"] = Parameter()
+        system.save()
+
+        system = System.objects.first()
+        self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value)
+
 
 class ValidatorErrorTest(unittest.TestCase):
 
@@ -3251,8 +3495,8 @@ class ValidatorErrorTest(unittest.TestCase):
         try:
             User().validate()
         except ValidationError, e:
-            expected_error_message = """ValidationError(Field is required: ['username', 'name'])"""
-            self.assertEqual(e.message, expected_error_message)
+            expected_error_message = """ValidationError(Field is required"""
+            self.assertTrue(expected_error_message in e.message)
             self.assertEqual(e.to_dict(), {
                 'username': 'Field is required',
                 'name': 'Field is required'})
@@ -3355,6 +3599,5 @@ class ValidatorErrorTest(unittest.TestCase):
 
         self.assertRaises(OperationError, change_shard_key)
 
-
 if __name__ == '__main__':
     unittest.main()

tests/test_fields.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import with_statement
 import datetime
 import os
@@ -7,7 +8,7 @@ import tempfile
 
 from decimal import Decimal
 
-from bson import Binary, DBRef
+from bson import Binary, DBRef, ObjectId
 import gridfs
 
 from nose.plugins.skip import SkipTest
@@ -143,6 +144,17 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(1, TestDocument.objects(int_fld__ne=None).count())
         self.assertEqual(1, TestDocument.objects(float_fld__ne=None).count())
 
+    def test_long_ne_operator(self):
+        class TestDocument(Document):
+            long_fld = LongField()
+
+        TestDocument.drop_collection()
+
+        TestDocument(long_fld=None).save()
+        TestDocument(long_fld=1).save()
+
+        self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count())
+
     def test_object_id_validation(self):
         """Ensure that invalid values cannot be assigned to string fields.
         """
@@ -216,6 +228,23 @@ class FieldTest(unittest.TestCase):
         person.age = 'ten'
         self.assertRaises(ValidationError, person.validate)
 
+    def test_long_validation(self):
+        """Ensure that invalid values cannot be assigned to long fields.
+        """
+        class TestDocument(Document):
+            value = LongField(min_value=0, max_value=110)
+
+        doc = TestDocument()
+        doc.value = 50
+        doc.validate()
+
+        doc.value = -1
+        self.assertRaises(ValidationError, doc.validate)
+        doc.age = 120
+        self.assertRaises(ValidationError, doc.validate)
+        doc.age = 'ten'
+        self.assertRaises(ValidationError, doc.validate)
+
     def test_float_validation(self):
         """Ensure that invalid values cannot be assigned to float fields.
         """
@@ -964,6 +993,24 @@ class FieldTest(unittest.TestCase):
         doc = self.db.test.find_one()
         self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2)
 
+    def test_mapfield_numerical_index(self):
+        """Ensure that MapField accept numeric strings as indexes."""
+        class Embedded(EmbeddedDocument):
+            name = StringField()
+
+        class Test(Document):
+            my_map = MapField(EmbeddedDocumentField(Embedded))
+
+        Test.drop_collection()
+
+        test = Test()
+        test.my_map['1'] = Embedded(name='test')
+        test.save()
+        test.my_map['1'].name = 'test updated'
+        test.save()
+
+        Test.drop_collection()
+
     def test_map_field_lookup(self):
         """Ensure MapField lookups succeed on Fields without a lookup method"""
 
@@ -1104,7 +1151,17 @@ class FieldTest(unittest.TestCase):
         p = Person.objects.get(name="Ross")
         self.assertEqual(p.parent, p1)
 
-    def test_str_reference_fields(self):
+    def test_dbref_to_mongo(self):
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self', dbref=False)
+
+        p1 = Person._from_son({'name': "Yakxxx",
+                               'parent': "50a234ea469ac1eda42d347d"})
+        mongoed = p1.to_mongo()
+        self.assertTrue(isinstance(mongoed['parent'], ObjectId))
+
+    def test_objectid_reference_fields(self):
 
         class Person(Document):
             name = StringField()
@@ -1117,7 +1174,7 @@ class FieldTest(unittest.TestCase):
 
         col = Person._get_collection()
         data = col.find_one({'name': 'Ross'})
-        self.assertEqual(data['parent'], "%s" % p1.pk)
+        self.assertEqual(data['parent'], p1.pk)
 
         p = Person.objects.get(name="Ross")
         self.assertEqual(p.parent, p1)
@@ -2174,6 +2231,28 @@ class FieldTest(unittest.TestCase):
         c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'})
         self.assertEqual(c['next'], 10)
 
+    def test_embedded_sequence_field(self):
+        class Comment(EmbeddedDocument):
+            id = SequenceField()
+            content = StringField(required=True)
+
+        class Post(Document):
+            title = StringField(required=True)
+            comments = ListField(EmbeddedDocumentField(Comment))
+
+        self.db['mongoengine.counters'].drop()
+        Post.drop_collection()
+
+        Post(title="MongoEngine",
+             comments=[Comment(content="NoSQL Rocks"),
+                       Comment(content="MongoEngine Rocks")]).save()
+
+        c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'})
+        self.assertEqual(c['next'], 2)
+        post = Post.objects.first()
+        self.assertEqual(1, post.comments[0].id)
+        self.assertEqual(2, post.comments[1].id)
+
     def test_generic_embedded_document(self):
         class Car(EmbeddedDocument):
             name = StringField()
@@ -2292,12 +2371,38 @@ class FieldTest(unittest.TestCase):
             self.assertTrue(1 in error_dict['comments'])
             self.assertTrue('content' in error_dict['comments'][1])
             self.assertEqual(error_dict['comments'][1]['content'],
-                              u'Field is required')
-
+                             u'Field is required')
 
         post.comments[1].content = 'here we go'
         post.validate()
 
+    def test_email_field(self):
+        class User(Document):
+            email = EmailField()
+
+        user = User(email="ross@example.com")
+        self.assertTrue(user.validate() is None)
+
+        user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
+                           "ucictfqpdkK9iS1zeFw8sg7s7cwAF7suIfUfeyueLpfosjn3"
+                           "aJIazqqWkm7.net"))
+        self.assertTrue(user.validate() is None)
+
+        user = User(email='me@localhost')
+        self.assertRaises(ValidationError, user.validate)
+
+    def test_email_field_honors_regex(self):
+        class User(Document):
+            email = EmailField(regex=r'\w+@example.com')
+
+        # Fails regex validation
+        user = User(email='me@foo.com')
+        self.assertRaises(ValidationError, user.validate)
+
+        # Passes regex validation
+        user = User(email='me@example.com')
+        self.assertTrue(user.validate() is None)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_queryset.py

@@ -47,7 +47,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50),
                          {'age': {'$gt': 20, '$lt': 50}})
         self.assertEqual(QuerySet._transform_query(age=20, age__gt=50),
-                         {'age': 20})
+                         {'$and': [{'age': {'$gt': 50}}, {'age': 20}]})
         self.assertEqual(QuerySet._transform_query(friend__age__gte=30),
                          {'friend.age': {'$gte': 30}})
         self.assertEqual(QuerySet._transform_query(name__exists=True),
@@ -230,6 +230,35 @@ class QuerySetTest(unittest.TestCase):
 
         Blog.drop_collection()
 
+    def test_chaining(self):
+        class A(Document):
+            s = StringField()
+
+        class B(Document):
+            ref = ReferenceField(A)
+            boolfield = BooleanField(default=False)
+
+        A.drop_collection()
+        B.drop_collection()
+
+        a1 = A(s="test1").save()
+        a2 = A(s="test2").save()
+
+        B(ref=a1, boolfield=True).save()
+
+        # Works
+        q1 = B.objects.filter(ref__in=[a1, a2], ref=a1)._query
+
+        # Doesn't work
+        q2 = B.objects.filter(ref__in=[a1, a2])
+        q2 = q2.filter(ref=a1)._query
+        self.assertEqual(q1, q2)
+
+        a_objects = A.objects(s='test1')
+        query = B.objects(ref__in=a_objects)
+        query = query.filter(boolfield=True)
+        self.assertEquals(query.count(), 1)
+
     def test_update_write_options(self):
         """Test that passing write_options works"""
 
@@ -414,6 +443,30 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(post.comments[0].by, 'joe')
         self.assertEqual(post.comments[0].votes.score, 4)
 
+    def test_updates_can_have_match_operators(self):
+
+        class Post(Document):
+            title = StringField(required=True)
+            tags = ListField(StringField())
+            comments = ListField(EmbeddedDocumentField("Comment"))
+
+        class Comment(EmbeddedDocument):
+            content = StringField()
+            name = StringField(max_length=120)
+            vote = IntField()
+
+        Post.drop_collection()
+
+        comm1 = Comment(content="very funny indeed", name="John S", vote=1)
+        comm2 = Comment(content="kind of funny", name="Mark P", vote=0)
+
+        Post(title='Fun with MongoEngine', tags=['mongodb', 'mongoengine'],
+             comments=[comm1, comm2]).save()
+
+        Post.objects().update_one(pull__comments__vote__lt=1)
+
+        self.assertEqual(1, len(Post.objects.first().comments))
+
     def test_mapfield_update(self):
         """Ensure that the MapField can be updated."""
         class Member(EmbeddedDocument):
@@ -543,6 +596,10 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(OperationError, throw_operation_error)
 
+        # Test can insert new doc
+        new_post = Blog(title="code", id=ObjectId())
+        Blog.objects.insert(new_post)
+
         # test handles other classes being inserted
         def throw_operation_error_wrong_doc():
             class Author(Document):
@@ -900,6 +957,11 @@ class QuerySetTest(unittest.TestCase):
                              {'attachments.views.extracted': 'no'}]}
         self.assertEqual(expected, raw_query)
 
+    def assertSequence(self, qs, expected):
+        self.assertEqual(len(qs), len(expected))
+        for i in range(len(qs)):
+            self.assertEqual(qs[i], expected[i])
+
     def test_ordering(self):
         """Ensure default ordering is applied and can be overridden.
         """
@@ -913,10 +975,10 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-        blog_post_1 = BlogPost(title="Blog Post #1",
-                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
         blog_post_2 = BlogPost(title="Blog Post #2",
                                published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_1 = BlogPost(title="Blog Post #1",
+                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
         blog_post_3 = BlogPost(title="Blog Post #3",
                                published_date=datetime(2010, 1, 7, 0, 0 ,0))
 
@@ -926,14 +988,13 @@ class QuerySetTest(unittest.TestCase):
 
         # get the "first" BlogPost using default ordering
         # from BlogPost.meta.ordering
-        latest_post = BlogPost.objects.first()
-        self.assertEqual(latest_post.title, "Blog Post #3")
+        expected = [blog_post_3, blog_post_2, blog_post_1]
+        self.assertSequence(BlogPost.objects.all(), expected)
 
         # override default ordering, order BlogPosts by "published_date"
-        first_post = BlogPost.objects.order_by("+published_date").first()
-        self.assertEqual(first_post.title, "Blog Post #1")
-
-        BlogPost.drop_collection()
+        qs = BlogPost.objects.order_by("+published_date")
+        expected = [blog_post_1, blog_post_2, blog_post_3]
+        self.assertSequence(qs, expected)
 
     def test_only(self):
         """Ensure that QuerySet.only only returns the requested fields.
@@ -1347,6 +1408,21 @@ class QuerySetTest(unittest.TestCase):
         query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query
         self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1})
 
+    def test_q_merge_queries_edge_case(self):
+
+        class User(Document):
+            email = EmailField(required=False)
+            name = StringField()
+
+        User.drop_collection()
+        pk = ObjectId()
+        User(email='example@example.com', pk=pk).save()
+
+        self.assertEqual(1, User.objects.filter(
+                                Q(email='example@example.com') |
+                                Q(name='John Doe')
+                                ).limit(2).filter(pk=pk).count())
+
     def test_exec_js_query(self):
         """Ensure that queries are properly formed for use in exec_js.
         """
@@ -1486,7 +1562,8 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(1, BlogPost.objects.count())
 
     def test_reverse_delete_rule_cascade_self_referencing(self):
-        """Ensure self-referencing CASCADE deletes do not result in infinite loop
+        """Ensure self-referencing CASCADE deletes do not result in infinite
+        loop
         """
         class Category(Document):
             name = StringField()
@@ -1592,6 +1669,40 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(post.authors, [me])
         self.assertEqual(another.authors, [])
 
+    def test_delete_with_limits(self):
+
+        class Log(Document):
+            pass
+
+        Log.drop_collection()
+
+        for i in xrange(10):
+            Log().save()
+
+        Log.objects()[3:5].delete()
+        self.assertEqual(8, Log.objects.count())
+
+    def test_delete_with_limit_handles_delete_rules(self):
+        """Ensure cascading deletion of referring documents from the database.
+        """
+        class BlogPost(Document):
+            content = StringField()
+            author = ReferenceField(self.Person, reverse_delete_rule=CASCADE)
+        BlogPost.drop_collection()
+
+        me = self.Person(name='Test User')
+        me.save()
+        someoneelse = self.Person(name='Some-one Else')
+        someoneelse.save()
+
+        BlogPost(content='Watching TV', author=me).save()
+        BlogPost(content='Chilling out', author=me).save()
+        BlogPost(content='Pro Testing', author=someoneelse).save()
+
+        self.assertEqual(3, BlogPost.objects.count())
+        self.Person.objects()[:1].delete()
+        self.assertEqual(1, BlogPost.objects.count())
+
     def test_update(self):
         """Ensure that atomic updates work properly.
         """
@@ -1819,8 +1930,8 @@ class QuerySetTest(unittest.TestCase):
     def test_order_by(self):
         """Ensure that QuerySets may be ordered.
         """
-        self.Person(name="User A", age=20).save()
         self.Person(name="User B", age=40).save()
+        self.Person(name="User A", age=20).save()
         self.Person(name="User C", age=30).save()
 
         names = [p.name for p in self.Person.objects.order_by('-age')]
@@ -1835,6 +1946,93 @@ class QuerySetTest(unittest.TestCase):
         ages = [p.age for p in self.Person.objects.order_by('-name')]
         self.assertEqual(ages, [30, 40, 20])
 
+    def test_order_by_optional(self):
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField(required=False)
+
+        BlogPost.drop_collection()
+
+        blog_post_3 = BlogPost(title="Blog Post #3",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_2 = BlogPost(title="Blog Post #2",
+                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
+        blog_post_4 = BlogPost(title="Blog Post #4",
+                               published_date=datetime(2010, 1, 7, 0, 0 ,0))
+        blog_post_1 = BlogPost(title="Blog Post #1", published_date=None)
+
+        blog_post_3.save()
+        blog_post_1.save()
+        blog_post_4.save()
+        blog_post_2.save()
+
+        expected = [blog_post_1, blog_post_2, blog_post_3, blog_post_4]
+        self.assertSequence(BlogPost.objects.order_by('published_date'),
+                            expected)
+        self.assertSequence(BlogPost.objects.order_by('+published_date'),
+                            expected)
+
+        expected.reverse()
+        self.assertSequence(BlogPost.objects.order_by('-published_date'),
+                            expected)
+
+    def test_order_by_list(self):
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField(required=False)
+
+        BlogPost.drop_collection()
+
+        blog_post_1 = BlogPost(title="A",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_2 = BlogPost(title="B",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_3 = BlogPost(title="C",
+                               published_date=datetime(2010, 1, 7, 0, 0 ,0))
+
+        blog_post_2.save()
+        blog_post_3.save()
+        blog_post_1.save()
+
+        qs = BlogPost.objects.order_by('published_date', 'title')
+        expected = [blog_post_1, blog_post_2, blog_post_3]
+        self.assertSequence(qs, expected)
+
+        qs = BlogPost.objects.order_by('-published_date', '-title')
+        expected.reverse()
+        self.assertSequence(qs, expected)
+
+    def test_order_by_chaining(self):
+        """Ensure that an order_by query chains properly and allows .only()
+        """
+        self.Person(name="User B", age=40).save()
+        self.Person(name="User A", age=20).save()
+        self.Person(name="User C", age=30).save()
+
+        only_age = self.Person.objects.order_by('-age').only('age')
+
+        names = [p.name for p in only_age]
+        ages = [p.age for p in only_age]
+
+        # The .only('age') clause should mean that all names are None
+        self.assertEqual(names, [None, None, None])
+        self.assertEqual(ages, [40, 30, 20])
+
+        qs = self.Person.objects.all().order_by('-age')
+        qs = qs.limit(10)
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
+        qs = self.Person.objects.all().limit(10)
+        qs = qs.order_by('-age')
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
+        qs = self.Person.objects.all().skip(0)
+        qs = qs.order_by('-age')
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
     def test_confirm_order_by_reference_wont_work(self):
         """Ordering by reference is not possible.  Use map / reduce.. or
         denormalise"""
@@ -2363,6 +2561,25 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(Foo.objects.distinct("bar"), [bar])
 
+    def test_distinct_handles_db_field(self):
+        """Ensure that distinct resolves field name to db_field as expected.
+        """
+        class Product(Document):
+            product_id = IntField(db_field='pid')
+
+        Product.drop_collection()
+
+        Product(product_id=1).save()
+        Product(product_id=2).save()
+        Product(product_id=1).save()
+
+        self.assertEqual(set(Product.objects.distinct('product_id')),
+                         set([1, 2]))
+        self.assertEqual(set(Product.objects.distinct('pid')),
+                         set([1, 2]))
+
+        Product.drop_collection()
+
     def test_custom_manager(self):
         """Ensure that custom QuerySetManager instances work as expected.
         """
@@ -2519,30 +2736,30 @@ class QuerySetTest(unittest.TestCase):
         """Ensure that index_types will, when disabled, prevent _types
         being added to all indices.
         """
-        class BlogPost(Document):
+        class BloggPost(Document):
             date = DateTimeField()
             meta = {'index_types': False,
                     'indexes': ['-date']}
 
         # Indexes are lazy so use list() to perform query
-        list(BlogPost.objects)
-        info = BlogPost.objects._collection.index_information()
+        list(BloggPost.objects)
+        info = BloggPost.objects._collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
         self.assertTrue([('_types', 1)] not in info)
         self.assertTrue([('date', -1)] in info)
 
-        BlogPost.drop_collection()
+        BloggPost.drop_collection()
 
-        class BlogPost(Document):
+        class BloggPost(Document):
             title = StringField()
             meta = {'allow_inheritance': False}
 
         # _types is not used on objects where allow_inheritance is False
-        list(BlogPost.objects)
-        info = BlogPost.objects._collection.index_information()
+        list(BloggPost.objects)
+        info = BloggPost.objects._collection.index_information()
         self.assertFalse([('_types', 1)] in info.values())
 
-        BlogPost.drop_collection()
+        BloggPost.drop_collection()
 
     def test_types_index_with_pk(self):
 
@@ -3593,6 +3810,38 @@ class QueryFieldListTest(unittest.TestCase):
         ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"}))
         self.assertEqual([b1], ak)
 
+    def test_as_pymongo(self):
+
+        from decimal import Decimal
+
+        class User(Document):
+            id = ObjectIdField('_id')
+            name = StringField()
+            age = IntField()
+            price = DecimalField()
+
+        User.drop_collection()
+        User(name="Bob Dole", age=89, price=Decimal('1.11')).save()
+        User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
+
+        users = User.objects.only('name', 'price').as_pymongo()
+        results = list(users)
+        self.assertTrue(isinstance(results[0], dict))
+        self.assertTrue(isinstance(results[1], dict))
+        self.assertEqual(results[0]['name'], 'Bob Dole')
+        self.assertEqual(results[0]['price'], '1.11')
+        self.assertEqual(results[1]['name'], 'Barack Obama')
+        self.assertEqual(results[1]['price'], '2.22')
+
+        # Test coerce_types
+        users = User.objects.only('name', 'price').as_pymongo(coerce_types=True)
+        results = list(users)
+        self.assertTrue(isinstance(results[0], dict))
+        self.assertTrue(isinstance(results[1], dict))
+        self.assertEqual(results[0]['name'], 'Bob Dole')
+        self.assertEqual(results[0]['price'], Decimal('1.11'))
+        self.assertEqual(results[1]['name'], 'Barack Obama')
+        self.assertEqual(results[1]['price'], Decimal('2.22'))
 
 if __name__ == '__main__':
     unittest.main()