Update changelog location

Conflicts:
	tests/test_document.py

.travis.yml

@@ -2,20 +2,18 @@
 language: python
 services: mongodb
 python:
-    - 2.5
-    - 2.6
-    - 2.7
-    - 3.1
-    - 3.2
+    - "2.5"
+    - "2.6"
+    - "2.7"
+    - "3.2"
+    - "3.3"
 env:
   - PYMONGO=dev
-  - PYMONGO=2.3
-  - PYMONGO=2.2
+  - PYMONGO=2.5
+  - PYMONGO=2.4.2
 install:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
     - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
     - python setup.py install
@@ -26,4 +24,4 @@ notifications:
 branches:
   only:
     - master
-    - 0.8
+    - "0.8"

AUTHORS

@@ -128,3 +128,14 @@ that much better:
  * Peter Teichman
  * Jakub Kot
  * Jorge Bastida
+ * Aleksandr Sorokoumov
+ * Yohan Graterol
+ * bool-dev
+ * Russ Weeks
+ * Paul Swartz
+ * Sundar Raman
+ * Benoit Louy
+ * lraucy
+ * hellysmile
+ * Jaepil Jeong
+ * Daniil Sharou

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2012 See AUTHORS
+Copyright (c) 2009 See AUTHORS
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

docs/changelog.rst

@@ -2,6 +2,26 @@
 Changelog
 =========
 
+Changes in 0.7.10
+=================
+- Fix UnicodeEncodeError for dbref (#278)
+- Allow construction using positional parameters (#268)
+- Updated EmailField length to support long domains (#243)
+- Added 64-bit integer support (#251)
+- Added Django sessions TTL support (#224)
+- Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240)
+- Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242)
+- Added "id" back to _data dictionary (#255)
+- Only mark a field as changed if the value has changed (#258)
+- Explicitly check for Document instances when dereferencing (#261)
+- Fixed order_by chaining issue (#265)
+- Added dereference support for tuples (#250)
+- Resolve field name to db field name when using distinct(#260, #264, #269)
+- Added kwargs to doc.save to help interop with django (#223, #270)
+- Fixed cloning querysets in PY3
+- Int fields no longer unset in save when changed to 0 (#272)
+- Fixed ReferenceField query chaining bug fixed (#254)
+
 Changes in 0.7.9
 ================
 - Better fix handling for old style _types

docs/django.rst

@@ -10,6 +10,16 @@ In your **settings.py** file, ignore the standard database settings (unless you
 also plan to use the ORM in your project), and instead call
 :func:`~mongoengine.connect` somewhere in the settings module.
 
+.. note ::
+   If you are not using another Database backend make sure you add  a dummy
+   backend, by adding the following to ``settings.py``::
+
+        DATABASES = {
+            'default': {
+                'ENGINE': 'django.db.backends.dummy'
+            }
+        }
+
 Authentication
 ==============
 MongoEngine includes a Django authentication backend, which uses MongoDB. The
@@ -45,6 +55,9 @@ into you settings module::
 
     SESSION_ENGINE = 'mongoengine.django.sessions'
 
+Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports  `mongodb TTL
+<http://docs.mongodb.org/manual/tutorial/expire-data/>`_.
+
 .. versionadded:: 0.2.1
 
 Storage

docs/upgrade.rst

@@ -183,3 +183,9 @@ Alternatively, you can rename your collections eg ::
         else:
             print "Upgraded collection names"
 
+
+mongodb 1.8 > 2.0 +
+===================
+
+Its been reported that indexes may need to be recreated to the newer version of indexes.  
+To do this drop indexes and call ``ensure_indexes`` on each model.

mongoengine/__init__.py

@@ -12,7 +12,7 @@ from signals import *
 __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
            queryset.__all__ + signals.__all__)
 
-VERSION = (0, 7, 9)
+VERSION = (0, 7, 10)
 
 
 def get_version():

mongoengine/base.py

@@ -205,8 +205,12 @@ class BaseField(object):
     def __set__(self, instance, value):
         """Descriptor for assigning a value to a field in a document.
         """
-        instance._data[self.name] = value
-        if instance._initialised:
+        changed = False
+        if (self.name not in instance._data or
+           instance._data[self.name] != value):
+            changed = True
+            instance._data[self.name] = value
+        if changed and instance._initialised:
             instance._mark_as_changed(self.name)
 
     def error(self, message="", errors=None, field_name=None):
@@ -252,7 +256,7 @@ class BaseField(object):
             elif value_to_check not in self.choices:
                 msg = ('Value must be %s of %s' %
                        (err_msg, unicode(self.choices)))
-                self.error()
+                self.error(msg)
 
         # check validation argument
         if self.validation is not None:
@@ -317,12 +321,6 @@ class ComplexBaseField(BaseField):
 
         return value
 
-    def __set__(self, instance, value):
-        """Descriptor for assigning a value to a field in a document.
-        """
-        instance._data[self.name] = value
-        instance._mark_as_changed(self.name)
-
     def to_python(self, value):
         """Convert a MongoDB-compatible type to a Python type.
         """
@@ -560,8 +558,11 @@ class DocumentMetaclass(type):
 
         # Set _fields and db_field maps
         attrs['_fields'] = doc_fields
-        attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k))
-                                        for k, v in doc_fields.iteritems()])
+        attrs['_fields_ordered'] = tuple(i[1]
+                                         for i in sorted((v.creation_counter, v.name)
+                                                          for v in doc_fields.itervalues()))
+        attrs['_db_field_map'] = dict((k, getattr(v, 'db_field', k))
+                                       for k, v in doc_fields.iteritems())
         attrs['_reverse_db_field_map'] = dict(
                 (v, k) for k, v in attrs['_db_field_map'].iteritems())
 
@@ -824,6 +825,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         if not new_class._meta.get('id_field'):
             new_class._meta['id_field'] = 'id'
             new_class._fields['id'] = ObjectIdField(db_field='_id')
+            new_class._fields['id'].name = 'id'
             new_class.id = new_class._fields['id']
 
         # Merge in exceptions with parent hierarchy
@@ -904,7 +906,17 @@ class BaseDocument(object):
     _dynamic_lock = True
     _initialised = False
 
-    def __init__(self, **values):
+    def __init__(self, *args, **values):
+        if args:
+            # Combine positional arguments with named arguments.
+            # We only want named arguments.
+            field = iter(self._fields_ordered)
+            for value in args:
+                name = field.next()
+                if name in values:
+                    raise TypeError("Multiple values for keyword argument '" + name + "'")
+                values[name] = value
+
         signals.pre_init.send(self.__class__, document=self, values=values)
 
         self._data = {}
@@ -1123,6 +1135,22 @@ class BaseDocument(object):
             key not in self._changed_fields):
             self._changed_fields.append(key)
 
+    def _clear_changed_fields(self):
+        self._changed_fields = []
+        EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
+        for field_name, field in self._fields.iteritems():
+            if (isinstance(field, ComplexBaseField) and
+                isinstance(field.field, EmbeddedDocumentField)):
+                field_value = getattr(self, field_name, None)
+                if field_value:
+                    for idx in (field_value if isinstance(field_value, dict)
+                                            else xrange(len(field_value))):
+                        field_value[idx]._clear_changed_fields()
+            elif isinstance(field, EmbeddedDocumentField):
+                field_value = getattr(self, field_name, None)
+                if field_value:
+                    field_value._clear_changed_fields()
+
     def _get_changed_fields(self, key='', inspected=None):
         """Returns a list of all fields that have explicitly been changed.
         """
@@ -1193,7 +1221,7 @@ class BaseDocument(object):
                 for p in parts:
                     if isinstance(d, DBRef):
                         break
-                    elif p.isdigit():
+                    elif isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
                     elif hasattr(d, 'get'):
                         d = d.get(p)
@@ -1207,7 +1235,7 @@ class BaseDocument(object):
 
         # Determine if any changed items were actually unset.
         for path, value in set_data.items():
-            if value or isinstance(value, bool):
+            if value or isinstance(value, (bool, int)):
                 continue
 
             # If we've set a value that ain't the default value dont unset it.
@@ -1224,7 +1252,7 @@ class BaseDocument(object):
                 parts = path.split('.')
                 db_field_name = parts.pop()
                 for p in parts:
-                    if p.isdigit():
+                    if isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
                     elif (hasattr(d, '__getattribute__') and
                           not isinstance(d, dict)):
@@ -1302,7 +1330,10 @@ class BaseDocument(object):
         return value
 
     def __iter__(self):
-        return iter(self._fields)
+        if 'id' in self._fields and 'id' not in self._fields_ordered:
+            return iter(('id', ) + self._fields_ordered)
+
+        return iter(self._fields_ordered)
 
     def __getitem__(self, name):
         """Dictionary-style field access, return a field's value if present.

mongoengine/dereference.py

@@ -33,7 +33,7 @@ class DeReference(object):
         self.max_depth = max_depth
         doc_type = None
 
-        if instance and instance._fields:
+        if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)):
             doc_type = instance._fields.get(name)
             if hasattr(doc_type, 'field'):
                 doc_type = doc_type.field
@@ -84,7 +84,7 @@ class DeReference(object):
         # Recursively find dbreferences
         depth += 1
         for k, item in iterator:
-            if hasattr(item, '_fields'):
+            if isinstance(item, Document):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -115,7 +115,7 @@ class DeReference(object):
         object_map = {}
         for col, dbrefs in self.reference_map.iteritems():
             keys = object_map.keys()
-            refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys]))
+            refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys]))
             if hasattr(col, 'objects'):  # We have a document class for the refs
                 references = col.objects.in_bulk(refs)
                 for key, doc in references.iteritems():
@@ -171,6 +171,7 @@ class DeReference(object):
 
         if not hasattr(items, 'items'):
             is_list = True
+            as_tuple = isinstance(items, tuple)
             iterator = enumerate(items)
             data = []
         else:
@@ -187,7 +188,7 @@ class DeReference(object):
 
             if k in self.object_map and not is_list:
                 data[k] = self.object_map[k]
-            elif hasattr(v, '_fields'):
+            elif isinstance(v, Document):
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
@@ -205,7 +206,7 @@ class DeReference(object):
 
         if instance and name:
             if is_list:
-                return BaseList(data, instance, name)
+                return tuple(data) if as_tuple else BaseList(data, instance, name)
             return BaseDict(data, instance, name)
         depth += 1
         return data

mongoengine/django/sessions.py

@@ -31,9 +31,17 @@ class MongoSession(Document):
                                         else fields.DictField()
     expire_date = fields.DateTimeField()
 
-    meta = {'collection': MONGOENGINE_SESSION_COLLECTION,
-            'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
-            'allow_inheritance': False}
+    meta = {
+        'collection': MONGOENGINE_SESSION_COLLECTION,
+        'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
+        'allow_inheritance': False,
+        'indexes': [
+            {
+                'fields': ['expire_date'],
+                'expireAfterSeconds': settings.SESSION_COOKIE_AGE
+            }
+        ]
+    }
 
 
 class SessionStore(SessionBase):

mongoengine/document.py

@@ -164,7 +164,7 @@ class Document(BaseDocument):
 
     def save(self, safe=True, force_insert=False, validate=True,
              write_options=None,  cascade=None, cascade_kwargs=None,
-             _refs=None):
+             _refs=None, **kwargs):
         """Save the :class:`~mongoengine.Document` to the database. If the
         document already exists, it will be updated, otherwise it will be
         created.
@@ -269,7 +269,7 @@ class Document(BaseDocument):
         if id_field not in self._meta.get('shard_key', []):
             self[id_field] = self._fields[id_field].to_python(object_id)
 
-        self._changed_fields = []
+        self._clear_changed_fields()
         self._created = False
         signals.post_save.send(self.__class__, document=self, created=created)
         return self

mongoengine/fields.py

@@ -27,7 +27,7 @@ except ImportError:
     Image = None
     ImageOps = None
 
-__all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
+__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField',
            'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField',
            'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField',
            'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField',
@@ -143,7 +143,7 @@ class EmailField(StringField):
     EMAIL_REGEX = re.compile(
         r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"  # dot-atom
         r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'  # quoted-string
-        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE  # domain
+        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE  # domain
     )
 
     def validate(self, value):
@@ -153,7 +153,7 @@ class EmailField(StringField):
 
 
 class IntField(BaseField):
-    """An integer field.
+    """An 32-bit integer field.
     """
 
     def __init__(self, min_value=None, max_value=None, **kwargs):
@@ -186,6 +186,40 @@ class IntField(BaseField):
         return int(value)
 
 
+class LongField(BaseField):
+    """An 64-bit integer field.
+    """
+
+    def __init__(self, min_value=None, max_value=None, **kwargs):
+        self.min_value, self.max_value = min_value, max_value
+        super(LongField, self).__init__(**kwargs)
+
+    def to_python(self, value):
+        try:
+            value = long(value)
+        except ValueError:
+            pass
+        return value
+
+    def validate(self, value):
+        try:
+            value = long(value)
+        except:
+            self.error('%s could not be converted to long' % value)
+
+        if self.min_value is not None and value < self.min_value:
+            self.error('Long value is too small')
+
+        if self.max_value is not None and value > self.max_value:
+            self.error('Long value is too large')
+
+    def prepare_query_value(self, op, value):
+        if value is None:
+            return value
+
+        return long(value)
+
+
 class FloatField(BaseField):
     """An floating point number field.
     """

mongoengine/queryset.py

@@ -7,10 +7,11 @@ import operator
 from collections import defaultdict
 from functools import partial
 
-from mongoengine.python_support import product, reduce
+from mongoengine.python_support import product, reduce, PY3
 
 import pymongo
 from bson.code import Code
+from bson.son import SON
 
 from mongoengine import signals
 
@@ -366,6 +367,10 @@ class QuerySet(object):
         self._skip = None
         self._hint = -1  # Using -1 as None is a valid value for hint
 
+    def __deepcopy__(self, memo):
+        """Essential for chained queries with ReferenceFields involved"""
+        return self.clone()
+
     def clone(self):
         """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`
 
@@ -374,8 +379,8 @@ class QuerySet(object):
         c = self.__class__(self._document, self._collection_obj)
 
         copy_props = ('_initial_query', '_query_obj', '_where_clause',
-                    '_loaded_fields', '_ordering', '_snapshot',
-                    '_timeout', '_limit', '_skip', '_slave_okay', '_hint')
+                      '_loaded_fields', '_ordering', '_snapshot', '_timeout',
+                      '_limit', '_skip', '_slave_okay', '_hint')
 
         for prop in copy_props:
             val = getattr(self, prop)
@@ -392,7 +397,7 @@ class QuerySet(object):
         return self._mongo_query
 
     def ensure_index(self, key_or_list, drop_dups=False, background=False,
-        **kwargs):
+                     **kwargs):
         """Ensure that the given indexes are in place.
 
         :param key_or_list: a single index key or a list of index keys (to
@@ -704,7 +709,7 @@ class QuerySet(object):
 
         mongo_query = {}
         merge_query = defaultdict(list)
-        for key, value in query.items():
+        for key, value in sorted(query.items()):
             if key == "__raw__":
                 mongo_query.update(value)
                 continue
@@ -808,7 +813,6 @@ class QuerySet(object):
                     mongo_query['$and'].append(value)
                 else:
                     mongo_query['$and'] = value
-
         return mongo_query
 
     def get(self, *q_objs, **query):
@@ -1208,8 +1212,11 @@ class QuerySet(object):
         .. versionchanged:: 0.5 - Fixed handling references
         .. versionchanged:: 0.6 - Improved db_field refrence handling
         """
-        return self._dereference(self._cursor.distinct(field), 1,
-                                 name=field, instance=self._document)
+        try:
+            field = self._fields_to_dbfields([field]).pop()
+        finally:
+            return self._dereference(self._cursor.distinct(field), 1,
+                                     name=field, instance=self._document)
 
     def only(self, *fields):
         """Load only a subset of this document's fields. ::
@@ -1314,7 +1321,8 @@ class QuerySet(object):
             key_list.append((key, direction))
 
         self._ordering = key_list
-
+        if self._cursor_obj:
+            self._cursor_obj.sort(key_list)
         return self
 
     def explain(self, format=False):

python-mongoengine.spec

@@ -5,7 +5,7 @@
 %define srcname mongoengine
 
 Name:           python-%{srcname}
-Version:        0.7.9
+Version:        0.7.10
 Release:        1%{?dist}
 Summary:        A Python Document-Object Mapper for working with MongoDB
 
@@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT
 # %{python_sitearch}/*
 
 %changelog
-* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html
+* See: http://docs.mongoengine.org/en/latest/changelog.html

setup.py

@@ -58,7 +58,7 @@ if sys.version_info[0] == 3:
         extra_opts['packages'].append("tests")
         extra_opts['package_data'] = {"tests": ["mongoengine.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL']
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL']
     extra_opts['packages'] = find_packages(exclude=('tests',))
 
 setup(name='mongoengine',

tests/test_all_warnings.py

@@ -76,7 +76,7 @@ class TestWarnings(unittest.TestCase):
         p2.parent.name = "Poppa Wilson"
         p2.save()
 
-        self.assertEqual(len(self.warning_list), 1)
+        self.assertTrue(len(self.warning_list) > 0)
         if len(self.warning_list) > 1:
             print self.warning_list
         warning = self.warning_list[0]

tests/test_dereference.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import with_statement
 import unittest
 
@@ -185,8 +186,9 @@ class FieldTest(unittest.TestCase):
 
         # Migrate the data
         for g in Group.objects():
-            g.author = g.author
-            g.members = g.members
+            # Explicitly mark as changed so resets
+            g._mark_as_changed('author')
+            g._mark_as_changed('members')
             g.save()
 
         group = Group.objects.first()
@@ -997,3 +999,57 @@ class FieldTest(unittest.TestCase):
         msg = Message.objects.get(id=1)
         self.assertEqual(0, msg.comments[0].id)
         self.assertEqual(1, msg.comments[1].id)
+
+    def test_tuples_as_tuples(self):
+        """
+        Ensure that tuples remain tuples when they are
+        inside a ComplexBaseField
+        """
+        from mongoengine.base import BaseField
+        class EnumField(BaseField):
+            def __init__(self, **kwargs):
+                super(EnumField,self).__init__(**kwargs)
+
+            def to_mongo(self, value):
+                return value
+
+            def to_python(self, value):
+                return tuple(value)
+
+        class TestDoc(Document):
+            items = ListField(EnumField())
+
+        TestDoc.drop_collection()
+        tuples = [(100, 'Testing')]
+        doc = TestDoc()
+        doc.items = tuples
+        doc.save()
+        x = TestDoc.objects().get()
+        self.assertTrue(x is not None)
+        self.assertTrue(len(x.items) == 1)
+        self.assertTrue(tuple(x.items[0]) in tuples)
+        self.assertTrue(x.items[0] in tuples)
+
+    def test_non_ascii_pk(self):
+        """
+        Ensure that dbref conversion to string does not fail when
+        non-ascii characters are used in primary key
+        """
+        class Brand(Document):
+            title = StringField(max_length=255, primary_key=True)
+
+        class BrandGroup(Document):
+            title = StringField(max_length=255, primary_key=True)
+            brands = ListField(ReferenceField("Brand", dbref=True))
+
+        Brand.drop_collection()
+        BrandGroup.drop_collection()
+
+        brand1 = Brand(title="Moschino").save()
+        brand2 = Brand(title=u"Денис Симачёв").save()
+
+        BrandGroup(title="top_brands", brands=[brand1, brand2]).save()
+        brand_groups = BrandGroup.objects().all()
+
+        self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands]))
+

tests/test_document.py

@@ -8,6 +8,7 @@ import sys
 import unittest
 import uuid
 import warnings
+import operator
 
 from nose.plugins.skip import SkipTest
 from datetime import datetime
@@ -452,7 +453,8 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info)
+        self.assertEqual([[('_id', 1)], [('_types', 1), ('name', 1)]],
+                         sorted(info, key=operator.itemgetter(0)))
 
         # Turn off inheritance
         class Animal(Document):
@@ -473,7 +475,8 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info)
+        self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]],
+            sorted(info, key=operator.itemgetter(0)))
 
         info = collection.index_information()
         indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])]
@@ -482,14 +485,16 @@ class DocumentTest(unittest.TestCase):
 
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)]], info)
+        self.assertEqual([[(u'_id', 1)]],
+            sorted(info, key=operator.itemgetter(0)))
 
         # Recreate indexes
         dog = Animal.objects.first()
         dog.save()
         info = collection.index_information()
         info = [value['key'] for key, value in info.iteritems()]
-        self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info)
+        self.assertEqual([[(u'_id', 1)], [(u'name', 1),]],
+            sorted(info, key=operator.itemgetter(0)))
 
         Animal.drop_collection()
 
@@ -924,7 +929,7 @@ class DocumentTest(unittest.TestCase):
 
         self.assertEqual(1, Person.objects.count())
         info = Person.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1'])
+        self.assertEqual(sorted(info.keys()), ['_id_', '_types_1_name_1', '_types_1_user_guid_1'])
         Person.drop_collection()
 
     def test_disable_index_creation(self):
@@ -968,7 +973,7 @@ class DocumentTest(unittest.TestCase):
         BlogPost.drop_collection()
 
         info = BlogPost.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_'])
+        self.assertEqual(sorted(info.keys()), [ '_id_', '_types_1_date.yr_-1'])
         BlogPost.drop_collection()
 
     def test_list_embedded_document_index(self):
@@ -991,7 +996,8 @@ class DocumentTest(unittest.TestCase):
 
         info = BlogPost.objects._collection.index_information()
         # we don't use _types in with list fields by default
-        self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1'])
+        self.assertEqual(sorted(info.keys()),
+                         ['_id_', '_types_1', 'tags.tag_1'])
 
         post1 = BlogPost(title="Embedded Indexes tests in place",
                         tags=[Tag(name="about"), Tag(name="time")]
@@ -1008,7 +1014,7 @@ class DocumentTest(unittest.TestCase):
             recursive_obj = EmbeddedDocumentField(RecursiveObject)
 
         info = RecursiveDocument.objects._collection.index_information()
-        self.assertEqual(info.keys(), ['_id_', '_types_1'])
+        self.assertEqual(sorted(info.keys()), ['_id_', '_types_1'])
 
     def test_geo_indexes_recursion(self):
 
@@ -1381,6 +1387,28 @@ class DocumentTest(unittest.TestCase):
         self.assertEqual(person.name, "Test User")
         self.assertEqual(person.age, 30)
 
+    def test_positional_creation(self):
+        """Ensure that document may be created using positional arguments.
+        """
+        person = self.Person("Test User", 42)
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 42)
+
+    def test_mixed_creation(self):
+        """Ensure that document may be created using mixed arguments.
+        """
+        person = self.Person("Test User", age=42)
+        self.assertEqual(person.name, "Test User")
+        self.assertEqual(person.age, 42)
+
+    def test_bad_mixed_creation(self):
+        """Ensure that document gives correct error when duplicating arguments
+        """
+        def construct_bad_instance():
+            return self.Person("Test User", 42, name="Bad User")
+
+        self.assertRaises(TypeError, construct_bad_instance)
+
     def test_to_dbref(self):
         """Ensure that you can get a dbref of a document"""
         person = self.Person(name="Test User", age=30)
@@ -1523,8 +1551,10 @@ class DocumentTest(unittest.TestCase):
         doc.validate()
         keys = doc._data.keys()
         self.assertEqual(2, len(keys))
-        self.assertTrue(None in keys)
         self.assertTrue('e' in keys)
+        # Ensure that the _id field has the right id
+        self.assertTrue('id' in keys)
+        self.assertEqual(doc._data.get('id'), doc.id)
 
     def test_save(self):
         """Ensure that a document may be saved in the database.
@@ -2719,7 +2749,7 @@ class DocumentTest(unittest.TestCase):
 
         Person.drop_collection()
 
-        self.assertEqual(Person._fields.keys(), ['name', 'id'])
+        self.assertEqual(sorted(Person._fields.keys()), ['id', 'name'])
 
         Person(name="Rozza").save()
 
@@ -3362,6 +3392,60 @@ class DocumentTest(unittest.TestCase):
                                         }
                                     ) ]), "1,2")
 
+    def test_data_contains_id_field(self):
+        """Ensure that asking for _data returns 'id'
+        """
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        Person(name="Harry Potter").save()
+
+        person = Person.objects.first()
+        self.assertTrue('id' in person._data.keys())
+        self.assertEqual(person._data.get('id'), person.id)
+
+    def test_complex_nesting_document_and_embedded_document(self):
+
+        class Macro(EmbeddedDocument):
+            value = DynamicField(default="UNDEFINED")
+
+        class Parameter(EmbeddedDocument):
+            macros = MapField(EmbeddedDocumentField(Macro))
+
+            def expand(self):
+                self.macros["test"] = Macro()
+
+        class Node(Document):
+            parameters = MapField(EmbeddedDocumentField(Parameter))
+
+            def expand(self):
+                self.flattened_parameter = {}
+                for parameter_name, parameter in self.parameters.iteritems():
+                    parameter.expand()
+
+        class System(Document):
+            name = StringField(required=True)
+            nodes = MapField(ReferenceField(Node, dbref=False))
+
+            def save(self, *args, **kwargs):
+                for node_name, node in self.nodes.iteritems():
+                    node.expand()
+                    node.save(*args, **kwargs)
+                super(System, self).save(*args, **kwargs)
+
+        System.drop_collection()
+        Node.drop_collection()
+
+        system = System(name="system")
+        system.nodes["node"] = Node()
+        system.save()
+        system.nodes["node"].parameters["param"] = Parameter()
+        system.save()
+
+        system = System.objects.first()
+        self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value)
+
 
 class ValidatorErrorTest(unittest.TestCase):
 
@@ -3411,8 +3495,8 @@ class ValidatorErrorTest(unittest.TestCase):
         try:
             User().validate()
         except ValidationError, e:
-            expected_error_message = """ValidationError(Field is required: ['username', 'name'])"""
-            self.assertEqual(e.message, expected_error_message)
+            expected_error_message = """ValidationError(Field is required"""
+            self.assertTrue(expected_error_message in e.message)
             self.assertEqual(e.to_dict(), {
                 'username': 'Field is required',
                 'name': 'Field is required'})
@@ -3515,6 +3599,5 @@ class ValidatorErrorTest(unittest.TestCase):
 
         self.assertRaises(OperationError, change_shard_key)
 
-
 if __name__ == '__main__':
     unittest.main()

tests/test_fields.py

@@ -144,6 +144,17 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(1, TestDocument.objects(int_fld__ne=None).count())
         self.assertEqual(1, TestDocument.objects(float_fld__ne=None).count())
 
+    def test_long_ne_operator(self):
+        class TestDocument(Document):
+            long_fld = LongField()
+
+        TestDocument.drop_collection()
+
+        TestDocument(long_fld=None).save()
+        TestDocument(long_fld=1).save()
+
+        self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count())
+
     def test_object_id_validation(self):
         """Ensure that invalid values cannot be assigned to string fields.
         """
@@ -217,6 +228,23 @@ class FieldTest(unittest.TestCase):
         person.age = 'ten'
         self.assertRaises(ValidationError, person.validate)
 
+    def test_long_validation(self):
+        """Ensure that invalid values cannot be assigned to long fields.
+        """
+        class TestDocument(Document):
+            value = LongField(min_value=0, max_value=110)
+
+        doc = TestDocument()
+        doc.value = 50
+        doc.validate()
+
+        doc.value = -1
+        self.assertRaises(ValidationError, doc.validate)
+        doc.age = 120
+        self.assertRaises(ValidationError, doc.validate)
+        doc.age = 'ten'
+        self.assertRaises(ValidationError, doc.validate)
+
     def test_float_validation(self):
         """Ensure that invalid values cannot be assigned to float fields.
         """
@@ -965,6 +993,24 @@ class FieldTest(unittest.TestCase):
         doc = self.db.test.find_one()
         self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2)
 
+    def test_mapfield_numerical_index(self):
+        """Ensure that MapField accept numeric strings as indexes."""
+        class Embedded(EmbeddedDocument):
+            name = StringField()
+
+        class Test(Document):
+            my_map = MapField(EmbeddedDocumentField(Embedded))
+
+        Test.drop_collection()
+
+        test = Test()
+        test.my_map['1'] = Embedded(name='test')
+        test.save()
+        test.my_map['1'].name = 'test updated'
+        test.save()
+
+        Test.drop_collection()
+
     def test_map_field_lookup(self):
         """Ensure MapField lookups succeed on Fields without a lookup method"""
 
@@ -2325,12 +2371,26 @@ class FieldTest(unittest.TestCase):
             self.assertTrue(1 in error_dict['comments'])
             self.assertTrue('content' in error_dict['comments'][1])
             self.assertEqual(error_dict['comments'][1]['content'],
-                              u'Field is required')
-
+                             u'Field is required')
 
         post.comments[1].content = 'here we go'
         post.validate()
 
+    def test_email_field(self):
+        class User(Document):
+            email = EmailField()
+
+        user = User(email="ross@example.com")
+        self.assertTrue(user.validate() is None)
+
+        user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
+                           "ucictfqpdkK9iS1zeFw8sg7s7cwAF7suIfUfeyueLpfosjn3"
+                           "aJIazqqWkm7.net"))
+        self.assertTrue(user.validate() is None)
+
+        user = User(email='me@localhost')
+        self.assertRaises(ValidationError, user.validate)
+
     def test_email_field_honors_regex(self):
         class User(Document):
             email = EmailField(regex=r'\w+@example.com')

tests/test_queryset.py

@@ -47,7 +47,7 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50),
                          {'age': {'$gt': 20, '$lt': 50}})
         self.assertEqual(QuerySet._transform_query(age=20, age__gt=50),
-                         {'age': 20})
+                         {'$and': [{'age': {'$gt': 50}}, {'age': 20}]})
         self.assertEqual(QuerySet._transform_query(friend__age__gte=30),
                          {'friend.age': {'$gte': 30}})
         self.assertEqual(QuerySet._transform_query(name__exists=True),
@@ -232,28 +232,33 @@ class QuerySetTest(unittest.TestCase):
 
     def test_chaining(self):
         class A(Document):
-            pass
+            s = StringField()
 
         class B(Document):
-            a = ReferenceField(A)
+            ref = ReferenceField(A)
+            boolfield = BooleanField(default=False)
 
         A.drop_collection()
         B.drop_collection()
 
-        a1 = A().save()
-        a2 = A().save()
+        a1 = A(s="test1").save()
+        a2 = A(s="test2").save()
 
-        B(a=a1).save()
+        B(ref=a1, boolfield=True).save()
 
         # Works
-        q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query
+        q1 = B.objects.filter(ref__in=[a1, a2], ref=a1)._query
 
         # Doesn't work
-        q2 = B.objects.filter(a__in=[a1, a2])
-        q2 = q2.filter(a=a1)._query
-
+        q2 = B.objects.filter(ref__in=[a1, a2])
+        q2 = q2.filter(ref=a1)._query
         self.assertEqual(q1, q2)
 
+        a_objects = A.objects(s='test1')
+        query = B.objects(ref__in=a_objects)
+        query = query.filter(boolfield=True)
+        self.assertEquals(query.count(), 1)
+
     def test_update_write_options(self):
         """Test that passing write_options works"""
 
@@ -952,6 +957,11 @@ class QuerySetTest(unittest.TestCase):
                              {'attachments.views.extracted': 'no'}]}
         self.assertEqual(expected, raw_query)
 
+    def assertSequence(self, qs, expected):
+        self.assertEqual(len(qs), len(expected))
+        for i in range(len(qs)):
+            self.assertEqual(qs[i], expected[i])
+
     def test_ordering(self):
         """Ensure default ordering is applied and can be overridden.
         """
@@ -965,10 +975,10 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
-        blog_post_1 = BlogPost(title="Blog Post #1",
-                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
         blog_post_2 = BlogPost(title="Blog Post #2",
                                published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_1 = BlogPost(title="Blog Post #1",
+                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
         blog_post_3 = BlogPost(title="Blog Post #3",
                                published_date=datetime(2010, 1, 7, 0, 0 ,0))
 
@@ -978,14 +988,13 @@ class QuerySetTest(unittest.TestCase):
 
         # get the "first" BlogPost using default ordering
         # from BlogPost.meta.ordering
-        latest_post = BlogPost.objects.first()
-        self.assertEqual(latest_post.title, "Blog Post #3")
+        expected = [blog_post_3, blog_post_2, blog_post_1]
+        self.assertSequence(BlogPost.objects.all(), expected)
 
         # override default ordering, order BlogPosts by "published_date"
-        first_post = BlogPost.objects.order_by("+published_date").first()
-        self.assertEqual(first_post.title, "Blog Post #1")
-
-        BlogPost.drop_collection()
+        qs = BlogPost.objects.order_by("+published_date")
+        expected = [blog_post_1, blog_post_2, blog_post_3]
+        self.assertSequence(qs, expected)
 
     def test_only(self):
         """Ensure that QuerySet.only only returns the requested fields.
@@ -1921,8 +1930,8 @@ class QuerySetTest(unittest.TestCase):
     def test_order_by(self):
         """Ensure that QuerySets may be ordered.
         """
-        self.Person(name="User A", age=20).save()
         self.Person(name="User B", age=40).save()
+        self.Person(name="User A", age=20).save()
         self.Person(name="User C", age=30).save()
 
         names = [p.name for p in self.Person.objects.order_by('-age')]
@@ -1937,11 +1946,67 @@ class QuerySetTest(unittest.TestCase):
         ages = [p.age for p in self.Person.objects.order_by('-name')]
         self.assertEqual(ages, [30, 40, 20])
 
+    def test_order_by_optional(self):
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField(required=False)
+
+        BlogPost.drop_collection()
+
+        blog_post_3 = BlogPost(title="Blog Post #3",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_2 = BlogPost(title="Blog Post #2",
+                               published_date=datetime(2010, 1, 5, 0, 0 ,0))
+        blog_post_4 = BlogPost(title="Blog Post #4",
+                               published_date=datetime(2010, 1, 7, 0, 0 ,0))
+        blog_post_1 = BlogPost(title="Blog Post #1", published_date=None)
+
+        blog_post_3.save()
+        blog_post_1.save()
+        blog_post_4.save()
+        blog_post_2.save()
+
+        expected = [blog_post_1, blog_post_2, blog_post_3, blog_post_4]
+        self.assertSequence(BlogPost.objects.order_by('published_date'),
+                            expected)
+        self.assertSequence(BlogPost.objects.order_by('+published_date'),
+                            expected)
+
+        expected.reverse()
+        self.assertSequence(BlogPost.objects.order_by('-published_date'),
+                            expected)
+
+    def test_order_by_list(self):
+        class BlogPost(Document):
+            title = StringField()
+            published_date = DateTimeField(required=False)
+
+        BlogPost.drop_collection()
+
+        blog_post_1 = BlogPost(title="A",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_2 = BlogPost(title="B",
+                               published_date=datetime(2010, 1, 6, 0, 0 ,0))
+        blog_post_3 = BlogPost(title="C",
+                               published_date=datetime(2010, 1, 7, 0, 0 ,0))
+
+        blog_post_2.save()
+        blog_post_3.save()
+        blog_post_1.save()
+
+        qs = BlogPost.objects.order_by('published_date', 'title')
+        expected = [blog_post_1, blog_post_2, blog_post_3]
+        self.assertSequence(qs, expected)
+
+        qs = BlogPost.objects.order_by('-published_date', '-title')
+        expected.reverse()
+        self.assertSequence(qs, expected)
+
     def test_order_by_chaining(self):
         """Ensure that an order_by query chains properly and allows .only()
         """
-        self.Person(name="User A", age=20).save()
         self.Person(name="User B", age=40).save()
+        self.Person(name="User A", age=20).save()
         self.Person(name="User C", age=30).save()
 
         only_age = self.Person.objects.order_by('-age').only('age')
@@ -1953,6 +2018,21 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(names, [None, None, None])
         self.assertEqual(ages, [40, 30, 20])
 
+        qs = self.Person.objects.all().order_by('-age')
+        qs = qs.limit(10)
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
+        qs = self.Person.objects.all().limit(10)
+        qs = qs.order_by('-age')
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
+        qs = self.Person.objects.all().skip(0)
+        qs = qs.order_by('-age')
+        ages = [p.age for p in qs]
+        self.assertEqual(ages, [40, 30, 20])
+
     def test_confirm_order_by_reference_wont_work(self):
         """Ordering by reference is not possible.  Use map / reduce.. or
         denormalise"""
@@ -2481,6 +2561,25 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertEqual(Foo.objects.distinct("bar"), [bar])
 
+    def test_distinct_handles_db_field(self):
+        """Ensure that distinct resolves field name to db_field as expected.
+        """
+        class Product(Document):
+            product_id = IntField(db_field='pid')
+
+        Product.drop_collection()
+
+        Product(product_id=1).save()
+        Product(product_id=2).save()
+        Product(product_id=1).save()
+
+        self.assertEqual(set(Product.objects.distinct('product_id')),
+                         set([1, 2]))
+        self.assertEqual(set(Product.objects.distinct('pid')),
+                         set([1, 2]))
+
+        Product.drop_collection()
+
     def test_custom_manager(self):
         """Ensure that custom QuerySetManager instances work as expected.
         """