From 198ccc028a9d24e7879db583f8a263a061255742 Mon Sep 17 00:00:00 2001
From: Greg Turner <assembla@gregturner.com>
Date: Fri, 6 Aug 2010 20:29:09 +1000
Subject: [PATCH 01/32] made list queries work with regexes (e.g. istartswith)

---
 mongoengine/fields.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 127f029f..73bc7562 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -319,8 +319,8 @@ class ListField(BaseField):
 
     def prepare_query_value(self, op, value):
         if op in ('set', 'unset'):
-            return [self.field.to_mongo(v) for v in value]
-        return self.field.to_mongo(value)
+            return [self.field.prepare_query_value(op, v) for v in value]
+        return self.field.prepare_query_value(op, value)
 
     def lookup_member(self, member_name):
         return self.field.lookup_member(member_name)

From 809fe44b43decc6f271d3eedd1f637db4149f24c Mon Sep 17 00:00:00 2001
From: Greg Turner <assembla@gregturner.com>
Date: Thu, 12 Aug 2010 15:14:20 +1000
Subject: [PATCH 02/32] Added a __raw__ parameter for passing query
 dictionaries directly to pymongo

---
 mongoengine/queryset.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 069ab113..b2460217 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -318,6 +318,11 @@ class QuerySet(object):
 
         mongo_query = {}
         for key, value in query.items():
+
+            if key == "__raw__":
+                mongo_query.update(value)
+                return mongo_query
+
             parts = key.split('__')
             # Check for an operator and transform to mongo-style if there is
             op = None

From 6373e20696533630979327ed0183ea2cbbc6cc48 Mon Sep 17 00:00:00 2001
From: Greg Turner <assembla@gregturner.com>
Date: Fri, 13 Aug 2010 22:28:26 +1000
Subject: [PATCH 03/32] Better error reporting on a validation error for a
 list.

---
 mongoengine/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 73bc7562..866dc7e3 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -315,7 +315,7 @@ class ListField(BaseField):
         try:
             [self.field.validate(item) for item in value]
         except Exception, err:
-            raise ValidationError('Invalid ListField item (%s)' % str(err))
+            raise ValidationError('Invalid ListField item (%s)' % str(item))
 
     def prepare_query_value(self, op, value):
         if op in ('set', 'unset'):

From d274576b4739d77803cc4adfda5e069be865d909 Mon Sep 17 00:00:00 2001
From: Greg Turner <assembla@gregturner.com>
Date: Fri, 13 Aug 2010 22:30:36 +1000
Subject: [PATCH 04/32] Fixed premature return for query gen

---
 mongoengine/queryset.py | 57 ++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index b2460217..3f9b7c5c 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -321,40 +321,39 @@ class QuerySet(object):
 
             if key == "__raw__":
                 mongo_query.update(value)
-                return mongo_query
+            else:
+                parts = key.split('__')
+                # Check for an operator and transform to mongo-style if there is
+                op = None
+                if parts[-1] in operators + match_operators:
+                    op = parts.pop()
 
-            parts = key.split('__')
-            # Check for an operator and transform to mongo-style if there is
-            op = None
-            if parts[-1] in operators + match_operators:
-                op = parts.pop()
+                if _doc_cls:
+                    # Switch field names to proper names [set in Field(name='foo')]
+                    fields = QuerySet._lookup_field(_doc_cls, parts)
+                    parts = [field.db_field for field in fields]
 
-            if _doc_cls:
-                # Switch field names to proper names [set in Field(name='foo')]
-                fields = QuerySet._lookup_field(_doc_cls, parts)
-                parts = [field.db_field for field in fields]
+                    # Convert value to proper value
+                    field = fields[-1]
+                    singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte']
+                    singular_ops += match_operators
+                    if op in singular_ops:
+                        value = field.prepare_query_value(op, value)
+                    elif op in ('in', 'nin', 'all'):
+                        # 'in', 'nin' and 'all' require a list of values
+                        value = [field.prepare_query_value(op, v) for v in value]
 
-                # Convert value to proper value
-                field = fields[-1]
-                singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte']
-                singular_ops += match_operators
-                if op in singular_ops:
-                    value = field.prepare_query_value(op, value)
-                elif op in ('in', 'nin', 'all'):
-                    # 'in', 'nin' and 'all' require a list of values
-                    value = [field.prepare_query_value(op, v) for v in value]
+                    if field.__class__.__name__ == 'GenericReferenceField':
+                        parts.append('_ref')
 
-                if field.__class__.__name__ == 'GenericReferenceField':
-                    parts.append('_ref')
+                if op and op not in match_operators:
+                    value = {'$' + op: value}
 
-            if op and op not in match_operators:
-                value = {'$' + op: value}
-
-            key = '.'.join(parts)
-            if op is None or key not in mongo_query:
-                mongo_query[key] = value
-            elif key in mongo_query and isinstance(mongo_query[key], dict):
-                mongo_query[key].update(value)
+                key = '.'.join(parts)
+                if op is None or key not in mongo_query:
+                    mongo_query[key] = value
+                elif key in mongo_query and isinstance(mongo_query[key], dict):
+                    mongo_query[key].update(value)
 
         return mongo_query
 

From b7e84031e310561469b99d62c99dd354e4fa6fe5 Mon Sep 17 00:00:00 2001
From: Greg Turner <assembla@gregturner.com>
Date: Thu, 16 Sep 2010 14:37:18 +1000
Subject: [PATCH 05/32] Escape strings for regex query.

---
 mongoengine/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 866dc7e3..79045898 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -64,7 +64,7 @@ class StringField(BaseField):
                 regex = r'%s$'
             elif op == 'exact':
                 regex = r'^%s$'
-            value = re.compile(regex % value, flags)
+            value = re.compile(regex % re.escape(value), flags)
         return value
 
 

From 2c8f00410301d3a184e03cf012a0737533478be4 Mon Sep 17 00:00:00 2001
From: sib <sib@narf.local>
Date: Thu, 30 Sep 2010 02:53:44 -0300
Subject: [PATCH 06/32] added update operator for addToSet

---
 mongoengine/queryset.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 069ab113..4d0f113c 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -662,7 +662,7 @@ class QuerySet(object):
         """Transform an update spec from Django-style format to Mongo format.
         """
         operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull',
-                     'pull_all']
+                     'pull_all', 'add_to_set']
 
         mongo_update = {}
         for key, value in update.items():
@@ -680,7 +680,9 @@ class QuerySet(object):
                     op = 'inc'
                     if value > 0:
                         value = -value
-
+                elif op == 'add_to_set':
+                    op = op.replace('_to_set', 'ToSet')
+                    
             if _doc_cls:
                 # Switch field names to proper names [set in Field(name='foo')]
                 fields = QuerySet._lookup_field(_doc_cls, parts)
@@ -688,7 +690,7 @@ class QuerySet(object):
 
                 # Convert value to proper value
                 field = fields[-1]
-                if op in (None, 'set', 'unset', 'push', 'pull'):
+                if op in (None, 'set', 'unset', 'push', 'pull', 'addToSet'):
                     value = field.prepare_query_value(op, value)
                 elif op in ('pushAll', 'pullAll'):
                     value = [field.prepare_query_value(op, v) for v in value]

From 72c7a010ff6d782d850c245deee1857f87b9a1f4 Mon Sep 17 00:00:00 2001
From: sib <sib@narf.local>
Date: Thu, 30 Sep 2010 03:05:15 -0300
Subject: [PATCH 07/32] added unit test for addToSet

---
 tests/queryset.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/queryset.py b/tests/queryset.py
index 51f92993..10825d07 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -664,6 +664,11 @@ class QuerySetTest(unittest.TestCase):
         post.reload()
         self.assertTrue('db' in post.tags and 'nosql' in post.tags)
 
+        BlogPost.objects.update_one(add_to_set__tags='unique')
+        BlogPost.objects.update_one(add_to_set__tags='unique')
+        post.reload()
+        self.assertEqual(post.tags.count('unique'), 1)
+        
         BlogPost.drop_collection()
 
     def test_update_pull(self):

From 159923fae237f1c292b1eae901ea2fea6b653763 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 01:48:42 +0100
Subject: [PATCH 08/32] Made lists of recursive reference fields possible

---
 mongoengine/fields.py | 23 ++++++++++++++++++++++-
 tests/fields.py       | 12 +++++++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 87d52fd6..65b397da 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -20,6 +20,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
 
 RECURSIVE_REFERENCE_CONSTANT = 'self'
 
+
 class StringField(BaseField):
     """A unicode string field.
     """
@@ -105,6 +106,7 @@ class URLField(StringField):
                 message = 'This URL appears to be a broken link: %s' % e
                 raise ValidationError(message)
 
+
 class EmailField(StringField):
     """A field that validates input as an E-Mail-Address.
     """
@@ -119,6 +121,7 @@ class EmailField(StringField):
         if not EmailField.EMAIL_REGEX.match(value):
             raise ValidationError('Invalid Mail-address: %s' % value)
 
+
 class IntField(BaseField):
     """An integer field.
     """
@@ -142,6 +145,7 @@ class IntField(BaseField):
         if self.max_value is not None and value > self.max_value:
             raise ValidationError('Integer value is too large')
 
+
 class FloatField(BaseField):
     """An floating point number field.
     """
@@ -197,6 +201,7 @@ class DecimalField(BaseField):
         if self.max_value is not None and value > self.max_value:
             raise ValidationError('Decimal value is too large')
 
+
 class BooleanField(BaseField):
     """A boolean field type.
 
@@ -209,6 +214,7 @@ class BooleanField(BaseField):
     def validate(self, value):
         assert isinstance(value, bool)
 
+
 class DateTimeField(BaseField):
     """A datetime field.
     """
@@ -216,6 +222,7 @@ class DateTimeField(BaseField):
     def validate(self, value):
         assert isinstance(value, datetime.datetime)
 
+
 class EmbeddedDocumentField(BaseField):
     """An embedded document field. Only valid values are subclasses of
     :class:`~mongoengine.EmbeddedDocument`.
@@ -331,6 +338,16 @@ class ListField(BaseField):
     def lookup_member(self, member_name):
         return self.field.lookup_member(member_name)
 
+    def _set_owner_document(self, owner_document):
+        self.field.owner_document = owner_document
+        self._owner_document = owner_document
+
+    def _get_owner_document(self, owner_document):
+        self._owner_document = owner_document
+
+    owner_document = property(_get_owner_document, _set_owner_document)
+
+
 class SortedListField(ListField):
     """A ListField that sorts the contents of its list before writing to
     the database in order to ensure that a sorted list is always
@@ -346,9 +363,11 @@ class SortedListField(ListField):
 
     def to_mongo(self, value):
         if self._ordering is not None:
-            return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering))
+            return sorted([self.field.to_mongo(item) for item in value],
+                          key=itemgetter(self._ordering))
         return sorted([self.field.to_mongo(item) for item in value])
 
+
 class DictField(BaseField):
     """A dictionary field that wraps a standard Python dictionary. This is
     similar to an embedded document, but the structure is not defined.
@@ -442,6 +461,7 @@ class ReferenceField(BaseField):
     def lookup_member(self, member_name):
         return self.document_type._fields.get(member_name)
 
+
 class GenericReferenceField(BaseField):
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
@@ -640,6 +660,7 @@ class FileField(BaseField):
             assert isinstance(value, GridFSProxy)
             assert isinstance(value.grid_id, pymongo.objectid.ObjectId)
 
+
 class GeoPointField(BaseField):
     """A list storing a latitude and longitude.
     """
diff --git a/tests/fields.py b/tests/fields.py
index 8c727196..622016c0 100644
--- a/tests/fields.py
+++ b/tests/fields.py
@@ -394,14 +394,24 @@ class FieldTest(unittest.TestCase):
         class Employee(Document):
             name = StringField()
             boss = ReferenceField('self')
+            friends = ListField(ReferenceField('self'))
 
         bill = Employee(name='Bill Lumbergh')
         bill.save()
-        peter = Employee(name='Peter Gibbons', boss=bill)
+
+        michael = Employee(name='Michael Bolton')
+        michael.save()
+
+        samir = Employee(name='Samir Nagheenanajar')
+        samir.save()
+
+        friends = [michael, samir]
+        peter = Employee(name='Peter Gibbons', boss=bill, friends=friends)
         peter.save()
 
         peter = Employee.objects.with_id(peter.id)
         self.assertEqual(peter.boss, bill)
+        self.assertEqual(peter.friends, friends)
 
     def test_undefined_reference(self):
         """Ensure that ReferenceFields may reference undefined Documents.

From 4012722a8d15bb3adc6d8581f092882b474b2ced Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 15:01:45 +0100
Subject: [PATCH 09/32] QuerySet.item_frequencies works with non-list fields

---
 mongoengine/queryset.py | 28 ++++++++++++++++++++--------
 tests/queryset.py       |  9 ++++++++-
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 73a45299..016430d1 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -916,20 +916,27 @@ class QuerySet(object):
         """
         return self.exec_js(average_func, field)
 
-    def item_frequencies(self, list_field, normalize=False):
-        """Returns a dictionary of all items present in a list field across
+    def item_frequencies(self, field, normalize=False):
+        """Returns a dictionary of all items present in a field across
         the whole queried set of documents, and their corresponding frequency.
         This is useful for generating tag clouds, or searching documents.
 
-        :param list_field: the list field to use
+        If the field is a :class:`~mongoengine.ListField`, the items within
+        each list will be counted individually.
+
+        :param field: the field to use
         :param normalize: normalize the results so they add to 1.0
         """
         freq_func = """
-            function(listField) {
+            function(field) {
                 if (options.normalize) {
                     var total = 0.0;
                     db[collection].find(query).forEach(function(doc) {
-                        total += doc[listField].length;
+                        if (doc[field].constructor == Array) {
+                            total += doc[field].length;
+                        } else {
+                            total++;
+                        }
                     });
                 }
 
@@ -939,14 +946,19 @@ class QuerySet(object):
                     inc /= total;
                 }
                 db[collection].find(query).forEach(function(doc) {
-                    doc[listField].forEach(function(item) {
+                    if (doc[field].constructor == Array) {
+                        doc[field].forEach(function(item) {
+                            frequencies[item] = inc + (frequencies[item] || 0);
+                        });
+                    } else {
+                        var item = doc[field];
                         frequencies[item] = inc + (frequencies[item] || 0);
-                    });
+                    }
                 });
                 return frequencies;
             }
         """
-        return self.exec_js(freq_func, list_field, normalize=normalize)
+        return self.exec_js(freq_func, field, normalize=normalize)
 
     def __repr__(self):
         limit = REPR_OUTPUT_SIZE + 1
diff --git a/tests/queryset.py b/tests/queryset.py
index ab28ff3d..62cf4954 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -973,7 +973,7 @@ class QuerySetTest(unittest.TestCase):
 
         BlogPost(hits=1, tags=['music', 'film', 'actors']).save()
         BlogPost(hits=2, tags=['music']).save()
-        BlogPost(hits=3, tags=['music', 'actors']).save()
+        BlogPost(hits=2, tags=['music', 'actors']).save()
 
         f = BlogPost.objects.item_frequencies('tags')
         f = dict((key, int(val)) for key, val in f.items())
@@ -995,6 +995,13 @@ class QuerySetTest(unittest.TestCase):
         self.assertAlmostEqual(f['actors'], 2.0/6.0)
         self.assertAlmostEqual(f['film'], 1.0/6.0)
 
+        # Check item_frequencies works for non-list fields
+        f = BlogPost.objects.item_frequencies('hits')
+        f = dict((key, int(val)) for key, val in f.items())
+        self.assertEqual(set(['1', '2']), set(f.keys()))
+        self.assertEqual(f['1'], 1)
+        self.assertEqual(f['2'], 2)
+
         BlogPost.drop_collection()
 
     def test_average(self):

From 556eed0151ea845bef1cb761f84cc07cadf4329c Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 15:22:47 +0100
Subject: [PATCH 10/32] QuerySet.distinct respects query. Closes #64.

---
 mongoengine/queryset.py |  2 +-
 tests/queryset.py       | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 016430d1..48936e68 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -649,7 +649,7 @@ class QuerySet(object):
 
         .. versionadded:: 0.4
         """
-        return self._collection.distinct(field)
+        return self._cursor.distinct(field)
 
     def only(self, *fields):
         """Load only a subset of this document's fields. ::
diff --git a/tests/queryset.py b/tests/queryset.py
index 62cf4954..2271c366 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1038,9 +1038,13 @@ class QuerySetTest(unittest.TestCase):
         self.Person(name='Mr Orange', age=20).save()
         self.Person(name='Mr White', age=20).save()
         self.Person(name='Mr Orange', age=30).save()
-        self.assertEqual(self.Person.objects.distinct('name'), 
-                         ['Mr Orange', 'Mr White'])
-        self.assertEqual(self.Person.objects.distinct('age'), [20, 30])
+        self.Person(name='Mr Pink', age=30).save()
+        self.assertEqual(set(self.Person.objects.distinct('name')),
+                         set(['Mr Orange', 'Mr White', 'Mr Pink']))
+        self.assertEqual(set(self.Person.objects.distinct('age')),
+                         set([20, 30]))
+        self.assertEqual(set(self.Person.objects(age=30).distinct('name')),
+                         set(['Mr Orange', 'Mr Pink']))
 
     def test_custom_manager(self):
         """Ensure that custom QuerySetManager instances work as expected.

From 62388cb740deb5be6845225f2294bb94abac30dc Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 21:08:28 +0100
Subject: [PATCH 11/32] Started work on new Q-object implementation

---
 mongoengine/queryset.py | 112 ++++++++++++++++++++++++++++++++++++++++
 tests/queryset.py       |  25 +++++++--
 2 files changed, 133 insertions(+), 4 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 48936e68..ad3c2de1 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -15,6 +15,7 @@ REPR_OUTPUT_SIZE = 20
 class DoesNotExist(Exception):
     pass
 
+
 class MultipleObjectsReturned(Exception):
     pass
 
@@ -26,12 +27,123 @@ class InvalidQueryError(Exception):
 class OperationError(Exception):
     pass
 
+
 class InvalidCollectionError(Exception):
     pass
 
+
 RE_TYPE = type(re.compile(''))
 
 
+class QNodeVisitor(object):
+
+    def visit_combination(self, combination):
+        return combination
+
+    def visit_query(self, query):
+        return query
+
+
+class SimplificationVisitor(QNodeVisitor):
+
+    def visit_combination(self, combination):
+        if combination.operation != combination.AND:
+            return combination
+
+        if any(not isinstance(node, NewQ) for node in combination.children):
+            return combination
+
+        query_ops = set()
+        query = {}
+        for node in combination.children:
+            ops = set(node.query.keys())
+            intersection = ops.intersection(query_ops)
+            if intersection:
+                msg = 'Duplicate query contitions: '
+                raise InvalidQueryError(msg + ', '.join(intersection))
+
+            query_ops.update(ops)
+            query.update(copy.deepcopy(node.query))
+        return NewQ(**query)
+
+
+class QueryCompilerVisitor(QNodeVisitor):
+
+    def __init__(self, document):
+        self.document = document
+
+    def visit_combination(self, combination):
+        if combination.operation == combination.OR:
+            return combination
+        return combination
+
+    def visit_query(self, query):
+        return QuerySet._transform_query(self.document, **query.query)
+
+
+class QNode(object):
+
+    AND = 0
+    OR = 1
+
+    def to_query(self, document):
+        query = self.accept(SimplificationVisitor())
+        query = query.accept(QueryCompilerVisitor(document))
+        return query
+
+    def accept(self, visitor):
+        raise NotImplementedError
+
+    def _combine(self, other, operation):
+        if other.empty:
+            return self
+
+        if self.empty:
+            return other
+
+        return QCombination(operation, [self, other])
+
+    @property
+    def empty(self):
+        return False
+
+    def __or__(self, other):
+        return self._combine(other, self.OR)
+
+    def __and__(self, other):
+        return self._combine(other, self.AND)
+
+
+class QCombination(QNode):
+
+    def __init__(self, operation, children):
+        self.operation = operation
+        self.children = children
+
+    def accept(self, visitor):
+        for i in range(len(self.children)):
+            self.children[i] = self.children[i].accept(visitor)
+
+        return visitor.visit_combination(self)
+
+    @property
+    def empty(self):
+        return not bool(self.query)
+
+
+class NewQ(QNode):
+
+    def __init__(self, **query):
+        self.query = query
+
+    def accept(self, visitor):
+        return visitor.visit_query(self)
+
+    @property
+    def empty(self):
+        return not bool(self.query)
+
+
 class Q(object):
 
     OR = '||'
diff --git a/tests/queryset.py b/tests/queryset.py
index 2271c366..60952513 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -6,7 +6,7 @@ import pymongo
 from datetime import datetime, timedelta
 
 from mongoengine.queryset import (QuerySet, MultipleObjectsReturned,
-                                  DoesNotExist)
+                                  DoesNotExist, NewQ)
 from mongoengine import *
 
 
@@ -53,9 +53,6 @@ class QuerySetTest(unittest.TestCase):
         person2 = self.Person(name="User B", age=30)
         person2.save()
 
-        q1 = Q(name='test')
-        q2 = Q(age__gte=18)
-
         # Find all people in the collection
         people = self.Person.objects
         self.assertEqual(len(people), 2)
@@ -1415,5 +1412,25 @@ class QTest(unittest.TestCase):
         self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1)
 
 
+class NewQTest(unittest.TestCase):
+
+    def test_and_combination(self):
+        class TestDoc(Document):
+            x = IntField()
+
+        # Check than an error is raised when conflicting queries are anded
+        def invalid_combination():
+            query = NewQ(x__lt=7) & NewQ(x__lt=3)
+            query.to_query(TestDoc)
+        self.assertRaises(InvalidQueryError, invalid_combination)
+
+        # Check normal cases work without an error
+        query = NewQ(x__lt=7) & NewQ(x__gt=3)
+
+        q1 = NewQ(x__lt=7)
+        q2 = NewQ(x__gt=3)
+        query = (q1 & q2).to_query(TestDoc)
+        self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}})
+
 if __name__ == '__main__':
     unittest.main()

From a3c46fec0778dd6dba5aa9d693eb9cecac530f0c Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 21:26:26 +0100
Subject: [PATCH 12/32] Compilation of combinations - simple $or now works

---
 mongoengine/queryset.py | 32 +++++++++++++++++++-------------
 tests/queryset.py       | 14 ++++++++++++++
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index ad3c2de1..b3fe29f5 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -43,6 +43,20 @@ class QNodeVisitor(object):
     def visit_query(self, query):
         return query
 
+    def _query_conjunction(self, queries):
+        query_ops = set()
+        combined_query = {}
+        for query in queries:
+            ops = set(query.keys())
+            intersection = ops.intersection(query_ops)
+            if intersection:
+                msg = 'Duplicate query contitions: '
+                raise InvalidQueryError(msg + ', '.join(intersection))
+
+            query_ops.update(ops)
+            combined_query.update(copy.deepcopy(query))
+        return combined_query
+
 
 class SimplificationVisitor(QNodeVisitor):
 
@@ -53,18 +67,8 @@ class SimplificationVisitor(QNodeVisitor):
         if any(not isinstance(node, NewQ) for node in combination.children):
             return combination
 
-        query_ops = set()
-        query = {}
-        for node in combination.children:
-            ops = set(node.query.keys())
-            intersection = ops.intersection(query_ops)
-            if intersection:
-                msg = 'Duplicate query contitions: '
-                raise InvalidQueryError(msg + ', '.join(intersection))
-
-            query_ops.update(ops)
-            query.update(copy.deepcopy(node.query))
-        return NewQ(**query)
+        queries = [node.query for node in combination.children]
+        return NewQ(**self._query_conjunction(queries))
 
 
 class QueryCompilerVisitor(QNodeVisitor):
@@ -74,7 +78,9 @@ class QueryCompilerVisitor(QNodeVisitor):
 
     def visit_combination(self, combination):
         if combination.operation == combination.OR:
-            return combination
+            return {'$or': combination.children}
+        elif combination.operation == combination.AND:
+            return self._query_conjunction(combination.children)
         return combination
 
     def visit_query(self, query):
diff --git a/tests/queryset.py b/tests/queryset.py
index 60952513..6d3114e5 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1432,5 +1432,19 @@ class NewQTest(unittest.TestCase):
         query = (q1 & q2).to_query(TestDoc)
         self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}})
 
+    def test_or_combination(self):
+        class TestDoc(Document):
+            x = IntField()
+
+        q1 = NewQ(x__lt=3)
+        q2 = NewQ(x__gt=7)
+        query = (q1 | q2).to_query(TestDoc)
+        self.assertEqual(query, {
+            '$or': [
+                {'x': {'$lt': 3}},
+                {'x': {'$gt': 7}},
+            ]
+        })
+
 if __name__ == '__main__':
     unittest.main()

From db2f64c290c5469c0e82952cb3c9ef0c5457b0f8 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 23:01:44 +0100
Subject: [PATCH 13/32] Made query-tree code a bit clearer

---
 mongoengine/queryset.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index b3fe29f5..21bd44d0 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -36,18 +36,28 @@ RE_TYPE = type(re.compile(''))
 
 
 class QNodeVisitor(object):
+    """Base visitor class for visiting Q-object nodes in a query tree.
+    """
 
     def visit_combination(self, combination):
+        """Called by QCombination objects.
+        """
         return combination
 
     def visit_query(self, query):
+        """Called by (New)Q objects.
+        """
         return query
 
     def _query_conjunction(self, queries):
+        """Merges two query dicts - effectively &ing them together.
+        """
         query_ops = set()
         combined_query = {}
         for query in queries:
             ops = set(query.keys())
+            # Make sure that the same operation isn't applied more than once
+            # to a single field
             intersection = ops.intersection(query_ops)
             if intersection:
                 msg = 'Duplicate query contitions: '
@@ -59,11 +69,15 @@ class QNodeVisitor(object):
 
 
 class SimplificationVisitor(QNodeVisitor):
+    """Simplifies query trees by combinging unnecessary 'and' connection nodes
+    into a single Q-object.
+    """
 
     def visit_combination(self, combination):
         if combination.operation != combination.AND:
             return combination
 
+        # The simplification only applies to 'simple' queries
         if any(not isinstance(node, NewQ) for node in combination.children):
             return combination
 
@@ -72,6 +86,9 @@ class SimplificationVisitor(QNodeVisitor):
 
 
 class QueryCompilerVisitor(QNodeVisitor):
+    """Compiles the nodes in a query tree to a PyMongo-compatible query
+    dictionary.
+    """
 
     def __init__(self, document):
         self.document = document
@@ -88,6 +105,8 @@ class QueryCompilerVisitor(QNodeVisitor):
 
 
 class QNode(object):
+    """Base class for nodes in query trees.
+    """
 
     AND = 0
     OR = 1
@@ -101,6 +120,8 @@ class QNode(object):
         raise NotImplementedError
 
     def _combine(self, other, operation):
+        """Combine this node with another node into a QCombination object.
+        """
         if other.empty:
             return self
 
@@ -121,6 +142,9 @@ class QNode(object):
 
 
 class QCombination(QNode):
+    """Represents the combination of several conditions by a given logical
+    operator.
+    """
 
     def __init__(self, operation, children):
         self.operation = operation
@@ -138,6 +162,9 @@ class QCombination(QNode):
 
 
 class NewQ(QNode):
+    """A simple query object, used in a query tree to build up more complex
+    query structures.
+    """
 
     def __init__(self, **query):
         self.query = query

From c0f7c4ca2ddabb33a5d7dcfd87dfe9f8059844bc Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 3 Oct 2010 23:22:36 +0100
Subject: [PATCH 14/32] Fixed error in empty property on QCombination

---
 mongoengine/queryset.py |  2 +-
 tests/queryset.py       | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 21bd44d0..2c822c75 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -158,7 +158,7 @@ class QCombination(QNode):
 
     @property
     def empty(self):
-        return not bool(self.query)
+        return not bool(self.children)
 
 
 class NewQ(QNode):
diff --git a/tests/queryset.py b/tests/queryset.py
index 6d3114e5..6a337640 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1417,6 +1417,7 @@ class NewQTest(unittest.TestCase):
     def test_and_combination(self):
         class TestDoc(Document):
             x = IntField()
+            y = StringField()
 
         # Check than an error is raised when conflicting queries are anded
         def invalid_combination():
@@ -1432,6 +1433,15 @@ class NewQTest(unittest.TestCase):
         query = (q1 & q2).to_query(TestDoc)
         self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}})
 
+        # More complex nested example
+        query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString')
+        query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100)
+        mongo_query = {
+            'x': {'$lt': 100, '$gt': -100}, 
+            'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']},
+        }
+        self.assertEqual(query.to_query(TestDoc), mongo_query)
+
     def test_or_combination(self):
         class TestDoc(Document):
             x = IntField()

From 8e651542015ba7de526477d17a5cf661aca949e2 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 00:06:42 +0100
Subject: [PATCH 15/32] Added a tree transformer, got complex ANDs working

---
 mongoengine/queryset.py | 39 +++++++++++++++++++++++++++++++++++++++
 tests/queryset.py       | 14 ++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 2c822c75..9650fe1f 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -85,6 +85,44 @@ class SimplificationVisitor(QNodeVisitor):
         return NewQ(**self._query_conjunction(queries))
 
 
+class QueryTreeTransformerVisitor(QNodeVisitor):
+    """Transforms the query tree in to a form that may be used with MongoDB.
+    """
+
+    def visit_combination(self, combination):
+        if combination.operation == combination.AND:
+            # MongoDB doesn't allow us to have too many $or operations in our
+            # queries, so the aim is to move the ORs up the tree to one
+            # 'master' $or. Firstly, we must find all the necessary parts (part
+            # of an AND combination or just standard Q object), and store them
+            # separately from the OR parts.
+            or_parts = []
+            and_parts = []
+            for node in combination.children:
+                if isinstance(node, QCombination):
+                    if node.operation == node.OR:
+                        # Any of the children in an $or component may cause
+                        # the query to succeed
+                        or_parts += node.children
+                    elif node.operation == node.AND:
+                        and_parts.append(node)
+                elif isinstance(node, NewQ):
+                    and_parts.append(node)
+
+            # Now we combine the parts into a usable query. AND together all of
+            # the necessary parts. Then for each $or part, create a new query
+            # that ANDs the necessary part with the $or part.
+            clauses = []
+            for or_part in or_parts:
+                q_object = reduce(lambda a, b: a & b, and_parts, NewQ())
+                clauses.append(q_object & or_part)
+
+            # Finally, $or the generated clauses in to one query. Each of the
+            # clauses is sufficient for the query to succeed.
+            return reduce(lambda a, b: a | b, clauses, NewQ())
+        return combination
+
+
 class QueryCompilerVisitor(QNodeVisitor):
     """Compiles the nodes in a query tree to a PyMongo-compatible query
     dictionary.
@@ -113,6 +151,7 @@ class QNode(object):
 
     def to_query(self, document):
         query = self.accept(SimplificationVisitor())
+        query = query.accept(QueryTreeTransformerVisitor())
         query = query.accept(QueryCompilerVisitor(document))
         return query
 
diff --git a/tests/queryset.py b/tests/queryset.py
index 6a337640..f83a8d43 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1456,5 +1456,19 @@ class NewQTest(unittest.TestCase):
             ]
         })
 
+    def test_and_or_combination(self):
+        class TestDoc(Document):
+            x = IntField()
+
+        query = NewQ(x__gt=0) | NewQ(x__exists=False)
+        query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000])
+        print query.to_query(TestDoc)
+#        self.assertEqual(query.to_query(TestDoc, {
+#            '$or': [
+#                {'x': {'$lt': 3}},
+#                {'x': {'$gt': 7}},
+#            ]
+#        })
+
 if __name__ == '__main__':
     unittest.main()

From 3fcc0e97895d1f2d2a1bea150f15d1f8aa35b7e9 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 02:10:37 +0100
Subject: [PATCH 16/32] Combining OR nodes works, fixed other Q-object bugs

---
 mongoengine/queryset.py | 97 +++++++++++++++++++++++++++++++----------
 tests/queryset.py       | 62 ++++++++++++++++++++++----
 2 files changed, 126 insertions(+), 33 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 9650fe1f..06b67ab5 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -4,6 +4,7 @@ import pprint
 import pymongo
 import re
 import copy
+import itertools
 
 __all__ = ['queryset_manager', 'Q', 'InvalidQueryError',
            'InvalidCollectionError']
@@ -49,8 +50,22 @@ class QNodeVisitor(object):
         """
         return query
 
+
+class SimplificationVisitor(QNodeVisitor):
+    """Simplifies query trees by combinging unnecessary 'and' connection nodes
+    into a single Q-object.
+    """
+
+    def visit_combination(self, combination):
+        if combination.operation == combination.AND:
+            # The simplification only applies to 'simple' queries
+            if all(isinstance(node, NewQ) for node in combination.children):
+                queries = [node.query for node in combination.children]
+                return NewQ(**self._query_conjunction(queries))
+        return combination
+
     def _query_conjunction(self, queries):
-        """Merges two query dicts - effectively &ing them together.
+        """Merges query dicts - effectively &ing them together.
         """
         query_ops = set()
         combined_query = {}
@@ -68,23 +83,6 @@ class QNodeVisitor(object):
         return combined_query
 
 
-class SimplificationVisitor(QNodeVisitor):
-    """Simplifies query trees by combinging unnecessary 'and' connection nodes
-    into a single Q-object.
-    """
-
-    def visit_combination(self, combination):
-        if combination.operation != combination.AND:
-            return combination
-
-        # The simplification only applies to 'simple' queries
-        if any(not isinstance(node, NewQ) for node in combination.children):
-            return combination
-
-        queries = [node.query for node in combination.children]
-        return NewQ(**self._query_conjunction(queries))
-
-
 class QueryTreeTransformerVisitor(QNodeVisitor):
     """Transforms the query tree in to a form that may be used with MongoDB.
     """
@@ -96,14 +94,14 @@ class QueryTreeTransformerVisitor(QNodeVisitor):
             # 'master' $or. Firstly, we must find all the necessary parts (part
             # of an AND combination or just standard Q object), and store them
             # separately from the OR parts.
-            or_parts = []
+            or_groups = []
             and_parts = []
             for node in combination.children:
                 if isinstance(node, QCombination):
                     if node.operation == node.OR:
                         # Any of the children in an $or component may cause
                         # the query to succeed
-                        or_parts += node.children
+                        or_groups.append(node.children)
                     elif node.operation == node.AND:
                         and_parts.append(node)
                 elif isinstance(node, NewQ):
@@ -113,13 +111,27 @@ class QueryTreeTransformerVisitor(QNodeVisitor):
             # the necessary parts. Then for each $or part, create a new query
             # that ANDs the necessary part with the $or part.
             clauses = []
-            for or_part in or_parts:
+            for or_group in itertools.product(*or_groups):
                 q_object = reduce(lambda a, b: a & b, and_parts, NewQ())
-                clauses.append(q_object & or_part)
+                q_object = reduce(lambda a, b: a & b, or_group, q_object)
+                clauses.append(q_object)
 
             # Finally, $or the generated clauses in to one query. Each of the
             # clauses is sufficient for the query to succeed.
             return reduce(lambda a, b: a | b, clauses, NewQ())
+
+        if combination.operation == combination.OR:
+            children = []
+            # Crush any nested ORs in to this combination as MongoDB doesn't
+            # support nested $or operations
+            for node in combination.children:
+                if (isinstance(node, QCombination) and
+                    node.operation == combination.OR):
+                    children += node.children
+                else:
+                    children.append(node)
+            combination.children = children
+
         return combination
 
 
@@ -135,12 +147,42 @@ class QueryCompilerVisitor(QNodeVisitor):
         if combination.operation == combination.OR:
             return {'$or': combination.children}
         elif combination.operation == combination.AND:
-            return self._query_conjunction(combination.children)
+            return self._mongo_query_conjunction(combination.children)
         return combination
 
     def visit_query(self, query):
         return QuerySet._transform_query(self.document, **query.query)
 
+    def _mongo_query_conjunction(self, queries):
+        """Merges Mongo query dicts - effectively &ing them together.
+        """
+        combined_query = {}
+        for query in queries:
+            for field, ops in query.items():
+                if field not in combined_query:
+                    combined_query[field] = ops
+                else:
+                    # The field is already present in the query the only way
+                    # we can merge is if both the existing value and the new
+                    # value are operation dicts, reject anything else
+                    if (not isinstance(combined_query[field], dict) or
+                        not isinstance(ops, dict)):
+                        message = 'Conflicting values for ' + field
+                        raise InvalidQueryError(message)
+
+                    current_ops = set(combined_query[field].keys())
+                    new_ops = set(ops.keys())
+                    # Make sure that the same operation isn't applied more than
+                    # once to a single field
+                    intersection = current_ops.intersection(new_ops)
+                    if intersection:
+                        msg = 'Duplicate query contitions: '
+                        raise InvalidQueryError(msg + ', '.join(intersection))
+
+                    # Right! We've got two non-overlapping dicts of operations!
+                    combined_query[field].update(copy.deepcopy(ops))
+        return combined_query
+
 
 class QNode(object):
     """Base class for nodes in query trees.
@@ -187,7 +229,14 @@ class QCombination(QNode):
 
     def __init__(self, operation, children):
         self.operation = operation
-        self.children = children
+        self.children = []
+        for node in children:
+            # If the child is a combination of the same type, we can merge its
+            # children directly into this combinations children
+            if isinstance(node, QCombination) and node.operation == operation:
+                self.children += node.children
+            else:
+                self.children.append(node)
 
     def accept(self, visitor):
         for i in range(len(self.children)):
diff --git a/tests/queryset.py b/tests/queryset.py
index f83a8d43..0f8c9a92 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1415,6 +1415,8 @@ class QTest(unittest.TestCase):
 class NewQTest(unittest.TestCase):
 
     def test_and_combination(self):
+        """Ensure that Q-objects correctly AND together.
+        """
         class TestDoc(Document):
             x = IntField()
             y = StringField()
@@ -1443,6 +1445,8 @@ class NewQTest(unittest.TestCase):
         self.assertEqual(query.to_query(TestDoc), mongo_query)
 
     def test_or_combination(self):
+        """Ensure that Q-objects correctly OR together.
+        """
         class TestDoc(Document):
             x = IntField()
 
@@ -1457,18 +1461,58 @@ class NewQTest(unittest.TestCase):
         })
 
     def test_and_or_combination(self):
+        """Ensure that Q-objects handle ANDing ORed components.
+        """
         class TestDoc(Document):
             x = IntField()
+            y = BooleanField()
+
+        query = (NewQ(x__gt=0) | NewQ(x__exists=False))
+        query &= NewQ(x__lt=100)
+        self.assertEqual(query.to_query(TestDoc), {
+            '$or': [
+                {'x': {'$lt': 100, '$gt': 0}},
+                {'x': {'$lt': 100, '$exists': False}},
+            ]
+        })
+
+        q1 = (NewQ(x__gt=0) | NewQ(x__exists=False))
+        q2 = (NewQ(x__lt=100) | NewQ(y=True))
+        query = (q1 & q2).to_query(TestDoc)
+
+        self.assertEqual(['$or'], query.keys())
+        conditions = [
+            {'x': {'$lt': 100, '$gt': 0}},
+            {'x': {'$lt': 100, '$exists': False}},
+            {'x': {'$gt': 0}, 'y': True},
+            {'x': {'$exists': False}, 'y': True},
+        ]
+        self.assertEqual(len(conditions), len(query['$or']))
+        for condition in conditions:
+            self.assertTrue(condition in query['$or'])
+
+    def test_or_and_or_combination(self):
+        """Ensure that Q-objects handle ORing ANDed ORed components. :)
+        """
+        class TestDoc(Document):
+            x = IntField()
+            y = BooleanField()
+
+        q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False)))
+        q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False)))
+        query = (q1 | q2).to_query(TestDoc)
+
+        self.assertEqual(['$or'], query.keys())
+        conditions = [
+            {'x': {'$gt': 0}, 'y': True},
+            {'x': {'$gt': 0}, 'y': {'$exists': False}},
+            {'x': {'$lt': 100}, 'y':False},
+            {'x': {'$lt': 100}, 'y': {'$exists': False}},
+        ]
+        self.assertEqual(len(conditions), len(query['$or']))
+        for condition in conditions:
+            self.assertTrue(condition in query['$or'])
 
-        query = NewQ(x__gt=0) | NewQ(x__exists=False)
-        query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000])
-        print query.to_query(TestDoc)
-#        self.assertEqual(query.to_query(TestDoc, {
-#            '$or': [
-#                {'x': {'$lt': 3}},
-#                {'x': {'$gt': 7}},
-#            ]
-#        })
 
 if __name__ == '__main__':
     unittest.main()

From 76cb851c40c00b11700689b72b022541de3b422a Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 11:41:07 +0100
Subject: [PATCH 17/32] Replaced old Q-object with new, revamped Q-object

---
 mongoengine/queryset.py | 154 +++++++---------------------------------
 tests/queryset.py       |  86 ++++++++--------------
 2 files changed, 52 insertions(+), 188 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 06b67ab5..2eabfb0e 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -59,9 +59,9 @@ class SimplificationVisitor(QNodeVisitor):
     def visit_combination(self, combination):
         if combination.operation == combination.AND:
             # The simplification only applies to 'simple' queries
-            if all(isinstance(node, NewQ) for node in combination.children):
+            if all(isinstance(node, Q) for node in combination.children):
                 queries = [node.query for node in combination.children]
-                return NewQ(**self._query_conjunction(queries))
+                return Q(**self._query_conjunction(queries))
         return combination
 
     def _query_conjunction(self, queries):
@@ -104,7 +104,7 @@ class QueryTreeTransformerVisitor(QNodeVisitor):
                         or_groups.append(node.children)
                     elif node.operation == node.AND:
                         and_parts.append(node)
-                elif isinstance(node, NewQ):
+                elif isinstance(node, Q):
                     and_parts.append(node)
 
             # Now we combine the parts into a usable query. AND together all of
@@ -112,13 +112,13 @@ class QueryTreeTransformerVisitor(QNodeVisitor):
             # that ANDs the necessary part with the $or part.
             clauses = []
             for or_group in itertools.product(*or_groups):
-                q_object = reduce(lambda a, b: a & b, and_parts, NewQ())
+                q_object = reduce(lambda a, b: a & b, and_parts, Q())
                 q_object = reduce(lambda a, b: a & b, or_group, q_object)
                 clauses.append(q_object)
 
             # Finally, $or the generated clauses in to one query. Each of the
             # clauses is sufficient for the query to succeed.
-            return reduce(lambda a, b: a | b, clauses, NewQ())
+            return reduce(lambda a, b: a | b, clauses, Q())
 
         if combination.operation == combination.OR:
             children = []
@@ -249,7 +249,7 @@ class QCombination(QNode):
         return not bool(self.children)
 
 
-class NewQ(QNode):
+class Q(QNode):
     """A simple query object, used in a query tree to build up more complex
     query structures.
     """
@@ -265,124 +265,6 @@ class NewQ(QNode):
         return not bool(self.query)
 
 
-class Q(object):
-
-    OR = '||'
-    AND = '&&'
-    OPERATORS = {
-        'eq': ('((this.%(field)s instanceof Array) && '
-               '  this.%(field)s.indexOf(%(value)s) != -1) ||'
-               ' this.%(field)s == %(value)s'),
-        'ne': 'this.%(field)s != %(value)s',
-        'gt': 'this.%(field)s > %(value)s',
-        'gte': 'this.%(field)s >= %(value)s',
-        'lt': 'this.%(field)s < %(value)s',
-        'lte': 'this.%(field)s <= %(value)s',
-        'lte': 'this.%(field)s <= %(value)s',
-        'in': '%(value)s.indexOf(this.%(field)s) != -1',
-        'nin': '%(value)s.indexOf(this.%(field)s) == -1',
-        'mod': '%(field)s %% %(value)s',
-        'all': ('%(value)s.every(function(a){'
-                'return this.%(field)s.indexOf(a) != -1 })'),
-        'size': 'this.%(field)s.length == %(value)s',
-        'exists': 'this.%(field)s != null',
-        'regex_eq': '%(value)s.test(this.%(field)s)',
-        'regex_ne': '!%(value)s.test(this.%(field)s)',
-    }
-
-    def __init__(self, **query):
-        self.query = [query]
-
-    def _combine(self, other, op):
-        obj = Q()
-        if not other.query[0]:
-            return self
-        if self.query[0]:
-            obj.query = (['('] + copy.deepcopy(self.query) + [op] +
-                         copy.deepcopy(other.query) + [')'])
-        else:
-            obj.query = copy.deepcopy(other.query)
-        return obj
-
-    def __or__(self, other):
-        return self._combine(other, self.OR)
-
-    def __and__(self, other):
-        return self._combine(other, self.AND)
-
-    def as_js(self, document):
-        js = []
-        js_scope = {}
-        for i, item in enumerate(self.query):
-            if isinstance(item, dict):
-                item_query = QuerySet._transform_query(document, **item)
-                # item_query will values will either be a value or a dict
-                js.append(self._item_query_as_js(item_query, js_scope, i))
-            else:
-                js.append(item)
-        return pymongo.code.Code(' '.join(js), js_scope)
-
-    def _item_query_as_js(self, item_query, js_scope, item_num):
-        # item_query will be in one of the following forms
-        #    {'age': 25, 'name': 'Test'}
-        #    {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']}
-        #    {'age': {'$lt': 25, '$gt': 18}}
-        js = []
-        for i, (key, value) in enumerate(item_query.items()):
-            op = 'eq'
-            # Construct a variable name for the value in the JS
-            value_name = 'i%sf%s' % (item_num, i)
-            if isinstance(value, dict):
-                # Multiple operators for this field
-                for j, (op, value) in enumerate(value.items()):
-                    # Create a custom variable name for this operator
-                    op_value_name = '%so%s' % (value_name, j)
-                    # Construct the JS that uses this op
-                    value, operation_js = self._build_op_js(op, key, value,
-                                                            op_value_name)
-                    # Update the js scope with the value for this op
-                    js_scope[op_value_name] = value
-                    js.append(operation_js)
-            else:
-                # Construct the JS for this field
-                value, field_js = self._build_op_js(op, key, value, value_name)
-                js_scope[value_name] = value
-                js.append(field_js)
-        return ' && '.join(js)
-
-    def _build_op_js(self, op, key, value, value_name):
-        """Substitute the values in to the correct chunk of Javascript.
-        """
-        if isinstance(value, RE_TYPE):
-            # Regexes are handled specially
-            if op.strip('$') == 'ne':
-                op_js = Q.OPERATORS['regex_ne']
-            else:
-                op_js = Q.OPERATORS['regex_eq']
-        else:
-            op_js = Q.OPERATORS[op.strip('$')]
-
-        # Comparing two ObjectIds in Javascript doesn't work..
-        if isinstance(value, pymongo.objectid.ObjectId):
-            value = unicode(value)
-
-        # Handle DBRef
-        if isinstance(value, pymongo.dbref.DBRef):
-            op_js = '(this.%(field)s.$id == "%(id)s" &&'\
-                    ' this.%(field)s.$ref == "%(ref)s")' % {
-                        'field': key,
-                        'id': unicode(value.id),
-                        'ref': unicode(value.collection)
-                    }
-            value = None
-
-        # Perform the substitution
-        operation_js = op_js % {
-            'field': key, 
-            'value': value_name
-        }
-        return value, operation_js
-
 class QuerySet(object):
     """A set of results returned from a query. Wraps a MongoDB cursor,
     providing :class:`~mongoengine.Document` objects as the results.
@@ -392,7 +274,9 @@ class QuerySet(object):
         self._document = document
         self._collection_obj = collection
         self._accessed_collection = False
-        self._query = {}
+        self._mongo_query = None
+        self._query_obj = Q()
+        self._initial_query = {}
         self._where_clause = None
         self._loaded_fields = []
         self._ordering = []
@@ -400,11 +284,18 @@ class QuerySet(object):
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
         if document._meta.get('allow_inheritance'):
-            self._query = {'_types': self._document._class_name}
+            self._initial_query = {'_types': self._document._class_name}
         self._cursor_obj = None
         self._limit = None
         self._skip = None
 
+    @property
+    def _query(self):
+        if self._mongo_query is None:
+            self._mongo_query = self._query_obj.to_query(self._document)
+            self._mongo_query.update(self._initial_query)
+        return self._mongo_query
+
     def ensure_index(self, key_or_list, drop_dups=False, background=False,
         **kwargs):
         """Ensure that the given indexes are in place.
@@ -463,10 +354,13 @@ class QuerySet(object):
             objects, only the last one will be used
         :param query: Django-style query keyword arguments
         """
+        #if q_obj:
+            #self._where_clause = q_obj.as_js(self._document)
+        query = Q(**query)
         if q_obj:
-            self._where_clause = q_obj.as_js(self._document)
-        query = QuerySet._transform_query(_doc_cls=self._document, **query)
-        self._query.update(query)
+            query &= q_obj
+        self._query_obj &= query
+        self._mongo_query = None
         return self
 
     def filter(self, *q_objs, **query):
@@ -616,7 +510,7 @@ class QuerySet(object):
                                                   "been implemented" % op)
                 elif op not in match_operators:
                     value = {'$' + op: value}
-            
+
             for i, part in indices:
                 parts.insert(i, part)
             key = '.'.join(parts)
diff --git a/tests/queryset.py b/tests/queryset.py
index 0f8c9a92..c941ecd5 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -6,7 +6,7 @@ import pymongo
 from datetime import datetime, timedelta
 
 from mongoengine.queryset import (QuerySet, MultipleObjectsReturned,
-                                  DoesNotExist, NewQ)
+                                  DoesNotExist)
 from mongoengine import *
 
 
@@ -153,7 +153,8 @@ class QuerySetTest(unittest.TestCase):
 
         # Retrieve the first person from the database
         self.assertRaises(MultipleObjectsReturned, self.Person.objects.get)
-        self.assertRaises(self.Person.MultipleObjectsReturned, self.Person.objects.get)
+        self.assertRaises(self.Person.MultipleObjectsReturned,
+                          self.Person.objects.get)
 
         # Use a query to filter the people found to just person2
         person = self.Person.objects.get(age=30)
@@ -231,7 +232,8 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual(created, False)
         
         # Try retrieving when no objects exists - new doc should be created
-        person, created = self.Person.objects.get_or_create(age=50, defaults={'name': 'User C'})
+        kwargs = dict(age=50, defaults={'name': 'User C'})
+        person, created = self.Person.objects.get_or_create(**kwargs)
         self.assertEqual(created, True)
         
         person = self.Person.objects.get(age=50)
@@ -545,6 +547,7 @@ class QuerySetTest(unittest.TestCase):
 
         obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first()
         self.assertEqual(obj, person)
+        
         obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first()
         self.assertEqual(obj, None)
 
@@ -1343,43 +1346,6 @@ class QuerySetTest(unittest.TestCase):
 
 class QTest(unittest.TestCase):
 
-    def test_or_and(self):
-        """Ensure that Q objects may be combined correctly.
-        """
-        q1 = Q(name='test')
-        q2 = Q(age__gte=18)
-
-        query = ['(', {'name': 'test'}, '||', {'age__gte': 18}, ')']
-        self.assertEqual((q1 | q2).query, query)
-
-        query = ['(', {'name': 'test'}, '&&', {'age__gte': 18}, ')']
-        self.assertEqual((q1 & q2).query, query)
-
-        query = ['(', '(', {'name': 'test'}, '&&', {'age__gte': 18}, ')', '||',
-                 {'name': 'example'}, ')']
-        self.assertEqual((q1 & q2 | Q(name='example')).query, query)
-
-    def test_item_query_as_js(self):
-        """Ensure that the _item_query_as_js utilitiy method works properly.
-        """
-        q = Q()
-        examples = [
-            
-            ({'name': 'test'}, ('((this.name instanceof Array) &&   '
-             'this.name.indexOf(i0f0) != -1) || this.name == i0f0'), 
-             {'i0f0': 'test'}),
-            ({'age': {'$gt': 18}}, 'this.age > i0f0o0', {'i0f0o0': 18}),
-            ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}},
-              ('this.age <= i0f0o0 && this.age > i0f0o1 && '
-               '((this.name instanceof Array) &&   '
-               'this.name.indexOf(i0f1) != -1) || this.name == i0f1'),
-             {'i0f0o0': 65, 'i0f0o1': 18, 'i0f1': 'test'}),
-        ]
-        for item, js, scope in examples:
-            test_scope = {}
-            self.assertEqual(q._item_query_as_js(item, test_scope, 0), js)
-            self.assertEqual(scope, test_scope)
-
     def test_empty_q(self):
         """Ensure that empty Q objects won't hurt.
         """
@@ -1389,11 +1355,15 @@ class QTest(unittest.TestCase):
         q4 = Q(name='test')
         q5 = Q()
 
-        query = ['(', {'age__gte': 18}, '||', {'name': 'test'}, ')']
-        self.assertEqual((q1 | q2 | q3 | q4 | q5).query, query)
+        class Person(Document):
+            name = StringField()
+            age = IntField()
 
-        query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')']
-        self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query)
+        query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]}
+        self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query)
+
+        query = {'age': {'$gte': 18}, 'name': 'test'}
+        self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query)
     
     def test_q_with_dbref(self):
         """Ensure Q objects handle DBRefs correctly"""
@@ -1423,21 +1393,21 @@ class NewQTest(unittest.TestCase):
 
         # Check than an error is raised when conflicting queries are anded
         def invalid_combination():
-            query = NewQ(x__lt=7) & NewQ(x__lt=3)
+            query = Q(x__lt=7) & Q(x__lt=3)
             query.to_query(TestDoc)
         self.assertRaises(InvalidQueryError, invalid_combination)
 
         # Check normal cases work without an error
-        query = NewQ(x__lt=7) & NewQ(x__gt=3)
+        query = Q(x__lt=7) & Q(x__gt=3)
 
-        q1 = NewQ(x__lt=7)
-        q2 = NewQ(x__gt=3)
+        q1 = Q(x__lt=7)
+        q2 = Q(x__gt=3)
         query = (q1 & q2).to_query(TestDoc)
         self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}})
 
         # More complex nested example
-        query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString')
-        query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100)
+        query = Q(x__lt=100) & Q(y__ne='NotMyString')
+        query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100)
         mongo_query = {
             'x': {'$lt': 100, '$gt': -100}, 
             'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']},
@@ -1450,8 +1420,8 @@ class NewQTest(unittest.TestCase):
         class TestDoc(Document):
             x = IntField()
 
-        q1 = NewQ(x__lt=3)
-        q2 = NewQ(x__gt=7)
+        q1 = Q(x__lt=3)
+        q2 = Q(x__gt=7)
         query = (q1 | q2).to_query(TestDoc)
         self.assertEqual(query, {
             '$or': [
@@ -1467,8 +1437,8 @@ class NewQTest(unittest.TestCase):
             x = IntField()
             y = BooleanField()
 
-        query = (NewQ(x__gt=0) | NewQ(x__exists=False))
-        query &= NewQ(x__lt=100)
+        query = (Q(x__gt=0) | Q(x__exists=False))
+        query &= Q(x__lt=100)
         self.assertEqual(query.to_query(TestDoc), {
             '$or': [
                 {'x': {'$lt': 100, '$gt': 0}},
@@ -1476,8 +1446,8 @@ class NewQTest(unittest.TestCase):
             ]
         })
 
-        q1 = (NewQ(x__gt=0) | NewQ(x__exists=False))
-        q2 = (NewQ(x__lt=100) | NewQ(y=True))
+        q1 = (Q(x__gt=0) | Q(x__exists=False))
+        q2 = (Q(x__lt=100) | Q(y=True))
         query = (q1 & q2).to_query(TestDoc)
 
         self.assertEqual(['$or'], query.keys())
@@ -1498,8 +1468,8 @@ class NewQTest(unittest.TestCase):
             x = IntField()
             y = BooleanField()
 
-        q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False)))
-        q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False)))
+        q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False)))
+        q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False)))
         query = (q1 | q2).to_query(TestDoc)
 
         self.assertEqual(['$or'], query.keys())

From b4c54b1b6257a2b5a262eb0c794aa1371f7aa1fb Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 11:41:49 +0100
Subject: [PATCH 18/32] Added support for the $not operator

---
 mongoengine/queryset.py | 130 +++++++++++++++++++++++++++++++++++++++-
 tests/queryset.py       |  16 ++++-
 2 files changed, 142 insertions(+), 4 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 2eabfb0e..e275bb31 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -265,6 +265,124 @@ class Q(QNode):
         return not bool(self.query)
 
 
+class OldQ(object):
+
+    OR = '||'
+    AND = '&&'
+    OPERATORS = {
+        'eq': ('((this.%(field)s instanceof Array) && '
+               '  this.%(field)s.indexOf(%(value)s) != -1) ||'
+               ' this.%(field)s == %(value)s'),
+        'ne': 'this.%(field)s != %(value)s',
+        'gt': 'this.%(field)s > %(value)s',
+        'gte': 'this.%(field)s >= %(value)s',
+        'lt': 'this.%(field)s < %(value)s',
+        'lte': 'this.%(field)s <= %(value)s',
+        'lte': 'this.%(field)s <= %(value)s',
+        'in': '%(value)s.indexOf(this.%(field)s) != -1',
+        'nin': '%(value)s.indexOf(this.%(field)s) == -1',
+        'mod': '%(field)s %% %(value)s',
+        'all': ('%(value)s.every(function(a){'
+                'return this.%(field)s.indexOf(a) != -1 })'),
+        'size': 'this.%(field)s.length == %(value)s',
+        'exists': 'this.%(field)s != null',
+        'regex_eq': '%(value)s.test(this.%(field)s)',
+        'regex_ne': '!%(value)s.test(this.%(field)s)',
+    }
+
+    def __init__(self, **query):
+        self.query = [query]
+
+    def _combine(self, other, op):
+        obj = Q()
+        if not other.query[0]:
+            return self
+        if self.query[0]:
+            obj.query = (['('] + copy.deepcopy(self.query) + [op] +
+                         copy.deepcopy(other.query) + [')'])
+        else:
+            obj.query = copy.deepcopy(other.query)
+        return obj
+
+    def __or__(self, other):
+        return self._combine(other, self.OR)
+
+    def __and__(self, other):
+        return self._combine(other, self.AND)
+
+    def as_js(self, document):
+        js = []
+        js_scope = {}
+        for i, item in enumerate(self.query):
+            if isinstance(item, dict):
+                item_query = QuerySet._transform_query(document, **item)
+                # item_query will values will either be a value or a dict
+                js.append(self._item_query_as_js(item_query, js_scope, i))
+            else:
+                js.append(item)
+        return pymongo.code.Code(' '.join(js), js_scope)
+
+    def _item_query_as_js(self, item_query, js_scope, item_num):
+        # item_query will be in one of the following forms
+        #    {'age': 25, 'name': 'Test'}
+        #    {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']}
+        #    {'age': {'$lt': 25, '$gt': 18}}
+        js = []
+        for i, (key, value) in enumerate(item_query.items()):
+            op = 'eq'
+            # Construct a variable name for the value in the JS
+            value_name = 'i%sf%s' % (item_num, i)
+            if isinstance(value, dict):
+                # Multiple operators for this field
+                for j, (op, value) in enumerate(value.items()):
+                    # Create a custom variable name for this operator
+                    op_value_name = '%so%s' % (value_name, j)
+                    # Construct the JS that uses this op
+                    value, operation_js = self._build_op_js(op, key, value,
+                                                            op_value_name)
+                    # Update the js scope with the value for this op
+                    js_scope[op_value_name] = value
+                    js.append(operation_js)
+            else:
+                # Construct the JS for this field
+                value, field_js = self._build_op_js(op, key, value, value_name)
+                js_scope[value_name] = value
+                js.append(field_js)
+        return ' && '.join(js)
+
+    def _build_op_js(self, op, key, value, value_name):
+        """Substitute the values in to the correct chunk of Javascript.
+        """
+        if isinstance(value, RE_TYPE):
+            # Regexes are handled specially
+            if op.strip('$') == 'ne':
+                op_js = Q.OPERATORS['regex_ne']
+            else:
+                op_js = Q.OPERATORS['regex_eq']
+        else:
+            op_js = Q.OPERATORS[op.strip('$')]
+
+        # Comparing two ObjectIds in Javascript doesn't work..
+        if isinstance(value, pymongo.objectid.ObjectId):
+            value = unicode(value)
+
+        # Handle DBRef
+        if isinstance(value, pymongo.dbref.DBRef):
+            op_js = '(this.%(field)s.$id == "%(id)s" &&'\
+                    ' this.%(field)s.$ref == "%(ref)s")' % {
+                        'field': key,
+                        'id': unicode(value.id),
+                        'ref': unicode(value.collection)
+                    }
+            value = None
+
+        # Perform the substitution
+        operation_js = op_js % {
+            'field': key, 
+            'value': value_name
+        }
+        return value, operation_js
+
 class QuerySet(object):
     """A set of results returned from a query. Wraps a MongoDB cursor,
     providing :class:`~mongoengine.Document` objects as the results.
@@ -462,7 +580,7 @@ class QuerySet(object):
         """Transform a query from Django-style format to Mongo format.
         """
         operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod',
-                     'all', 'size', 'exists']
+                     'all', 'size', 'exists', 'not']
         geo_operators = ['within_distance', 'within_box', 'near']
         match_operators = ['contains', 'icontains', 'startswith', 
                            'istartswith', 'endswith', 'iendswith', 
@@ -478,6 +596,11 @@ class QuerySet(object):
             if parts[-1] in operators + match_operators + geo_operators:
                 op = parts.pop()
 
+            negate = False
+            if parts[-1] == 'not':
+                parts.pop()
+                negate = True
+
             if _doc_cls:
                 # Switch field names to proper names [set in Field(name='foo')]
                 fields = QuerySet._lookup_field(_doc_cls, parts)
@@ -485,7 +608,7 @@ class QuerySet(object):
 
                 # Convert value to proper value
                 field = fields[-1]
-                singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte']
+                singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not']
                 singular_ops += match_operators
                 if op in singular_ops:
                     value = field.prepare_query_value(op, value)
@@ -511,6 +634,9 @@ class QuerySet(object):
                 elif op not in match_operators:
                     value = {'$' + op: value}
 
+            if negate:
+                value = {'$not': value}
+            
             for i, part in indices:
                 parts.insert(i, part)
             key = '.'.join(parts)
diff --git a/tests/queryset.py b/tests/queryset.py
index c941ecd5..32d1902e 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -336,6 +336,18 @@ class QuerySetTest(unittest.TestCase):
         obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first()
         self.assertEqual(obj, person)
 
+    def test_not(self):
+        """Ensure that the __not operator works as expected.
+        """
+        alice = self.Person(name='Alice', age=25)
+        alice.save()
+
+        obj = self.Person.objects(name__iexact='alice').first()
+        self.assertEqual(obj, alice)
+
+        obj = self.Person.objects(name__not__iexact='alice').first()
+        self.assertEqual(obj, None)
+
     def test_filter_chaining(self):
         """Ensure filters can be chained together.
         """
@@ -545,10 +557,10 @@ class QuerySetTest(unittest.TestCase):
         obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first()
         self.assertEqual(obj, person)
 
-        obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first()
+        obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first()
         self.assertEqual(obj, person)
         
-        obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first()
+        obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first()
         self.assertEqual(obj, None)
 
     def test_q_lists(self):

From 4742328b908e57b3d87e2c0752ed8d4b51763e55 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 12:10:29 +0100
Subject: [PATCH 19/32] Delete stale cursor when query is filtered. Closes #62.

---
 mongoengine/queryset.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index e275bb31..6d4ad55e 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -479,6 +479,7 @@ class QuerySet(object):
             query &= q_obj
         self._query_obj &= query
         self._mongo_query = None
+        self._cursor_obj = None
         return self
 
     def filter(self, *q_objs, **query):

From 3acfd907202afa4f2712f6e7fce9c34c2e67c8a0 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 4 Oct 2010 14:58:00 +0100
Subject: [PATCH 20/32] Added some imports for PyMongo 1.9 compatibility.

---
 mongoengine/base.py     | 1 +
 mongoengine/fields.py   | 3 +++
 mongoengine/queryset.py | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/mongoengine/base.py b/mongoengine/base.py
index 0cbd707d..1f7ba1fe 100644
--- a/mongoengine/base.py
+++ b/mongoengine/base.py
@@ -3,6 +3,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned
 
 import sys
 import pymongo
+import pymongo.objectid
 
 
 _document_registry = {}
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 65b397da..8fcb1d62 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -5,6 +5,9 @@ from operator import itemgetter
 
 import re
 import pymongo
+import pymongo.dbref
+import pymongo.son
+import pymongo.binary
 import datetime
 import decimal
 import gridfs
diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 48936e68..99417850 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -2,6 +2,9 @@ from connection import _get_db
 
 import pprint
 import pymongo
+import pymongo.code
+import pymongo.dbref
+import pymongo.objectid
 import re
 import copy
 

From 92471445ec5aede3d6ffb14e6b63026156c6d623 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Tue, 5 Oct 2010 00:46:13 +0100
Subject: [PATCH 21/32] Fix changing databases

Conflicts:

	mongoengine/connection.py
	mongoengine/queryset.py
---
 mongoengine/connection.py | 22 ++++++++++++----------
 mongoengine/queryset.py   | 18 ++++++++++--------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/mongoengine/connection.py b/mongoengine/connection.py
index 94cc6ea1..814fde13 100644
--- a/mongoengine/connection.py
+++ b/mongoengine/connection.py
@@ -4,11 +4,12 @@ import multiprocessing
 __all__ = ['ConnectionError', 'connect']
 
 
-_connection_settings = {
+_connection_defaults = {
     'host': 'localhost',
     'port': 27017,
 }
 _connection = {}
+_connection_settings = _connection_defaults.copy()
 
 _db_name = None
 _db_username = None
@@ -20,25 +21,25 @@ class ConnectionError(Exception):
     pass
 
 
-def _get_connection():
+def _get_connection(reconnect=False):
     global _connection
     identity = get_identity()
     # Connect to the database if not already connected
-    if _connection.get(identity) is None:
+    if _connection.get(identity) is None or reconnect:
         try:
             _connection[identity] = Connection(**_connection_settings)
         except:
             raise ConnectionError('Cannot connect to the database')
     return _connection[identity]
 
-def _get_db():
+def _get_db(reconnect=False):
     global _db, _connection
     identity = get_identity()
     # Connect if not already connected
-    if _connection.get(identity) is None:
-        _connection[identity] = _get_connection()
+    if _connection.get(identity) is None or reconnect:
+        _connection[identity] = _get_connection(reconnect=reconnect)
 
-    if _db.get(identity) is None:
+    if _db.get(identity) is None or reconnect:
         # _db_name will be None if the user hasn't called connect()
         if _db_name is None:
             raise ConnectionError('Not connected to the database')
@@ -61,9 +62,10 @@ def connect(db, username=None, password=None, **kwargs):
     the default port on localhost. If authentication is needed, provide
     username and password arguments as well.
     """
-    global _connection_settings, _db_name, _db_username, _db_password
-    _connection_settings.update(kwargs)
+    global _connection_settings, _db_name, _db_username, _db_password, _db
+    _connection_settings = dict(_connection_defaults, **kwargs)
     _db_name = db
     _db_username = username
     _db_password = password
-    return _get_db()
\ No newline at end of file
+    return _get_db(reconnect=True)
+
diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 99417850..fae2aabf 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -977,7 +977,7 @@ class QuerySetManager(object):
 
     def __init__(self, manager_func=None):
         self._manager_func = manager_func
-        self._collection = None
+        self._collections = {}
 
     def __get__(self, instance, owner):
         """Descriptor for instantiating a new QuerySet object when
@@ -987,8 +987,8 @@ class QuerySetManager(object):
             # Document class being used rather than a document object
             return self
 
-        if self._collection is None:
-            db = _get_db()
+        db = _get_db()
+        if db not in self._collections:
             collection = owner._meta['collection']
 
             # Create collection as a capped collection if specified
@@ -998,10 +998,10 @@ class QuerySetManager(object):
                 max_documents = owner._meta['max_documents']
 
                 if collection in db.collection_names():
-                    self._collection = db[collection]
+                    self._collections[db] = db[collection]
                     # The collection already exists, check if its capped
                     # options match the specified capped options
-                    options = self._collection.options()
+                    options = self._collections[db].options()
                     if options.get('max') != max_documents or \
                        options.get('size') != max_size:
                         msg = ('Cannot create collection "%s" as a capped '
@@ -1012,13 +1012,15 @@ class QuerySetManager(object):
                     opts = {'capped': True, 'size': max_size}
                     if max_documents:
                         opts['max'] = max_documents
-                    self._collection = db.create_collection(collection, **opts)
+                    self._collections[db] = db.create_collection(
+                        collection, **opts
+                    )
             else:
-                self._collection = db[collection]
+                self._collections[db] = db[collection]
 
         # owner is the document that contains the QuerySetManager
         queryset_class = owner._meta['queryset_class'] or QuerySet
-        queryset = queryset_class(owner, self._collection)
+        queryset = queryset_class(owner, self._collections[db])
         if self._manager_func:
             if self._manager_func.func_code.co_argcount == 1:
                 queryset = self._manager_func(queryset)

From 833fa3d94dcb35fd10131894f1d45f57000e0fbe Mon Sep 17 00:00:00 2001
From: Jaime <jaime@joltonline.com>
Date: Wed, 6 Oct 2010 19:56:12 +0100
Subject: [PATCH 22/32] Added note about the use of default parameters

---
 docs/guide/defining-documents.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index 3c276869..a8e9924f 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -66,6 +66,25 @@ arguments can be set on all fields:
 :attr:`default` (Default: None)
     A value to use when no value is set for this field.
 
+    The definion of default parameters follow `the general rules on Python
+    <http://docs.python.org/reference/compound_stmts.html#function-definitions>`__,
+    which means that some care should be taken when dealing with default mutable objects 
+    (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`)::
+
+        class ExampleFirst(Document):
+            # Default an empty list
+            values = ListField(IntField(), default=list)
+
+        class ExampleSecond(Document):
+            # Default a set of values
+            values = ListField(IntField(), default=lambda: [1,2,3])
+
+        class ExampleDangerous(Document):
+            # This can make an .append call to  add values to the default (and all the following objects),
+            # instead to just an object
+            values = ListField(IntField(), default=[1,2,3])
+    
+
 :attr:`unique` (Default: False)
     When True, no documents in the collection will have the same value for this
     field.

From f6661419816157bec89db3f97b4f8d4a95c67897 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 13:23:11 +0100
Subject: [PATCH 23/32] Added test for list of referencefields

---
 tests/fields.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tests/fields.py b/tests/fields.py
index 622016c0..e30f843e 100644
--- a/tests/fields.py
+++ b/tests/fields.py
@@ -189,6 +189,9 @@ class FieldTest(unittest.TestCase):
     def test_list_validation(self):
         """Ensure that a list field only accepts lists with valid elements.
         """
+        class User(Document):
+            pass
+
         class Comment(EmbeddedDocument):
             content = StringField()
 
@@ -196,6 +199,7 @@ class FieldTest(unittest.TestCase):
             content = StringField()
             comments = ListField(EmbeddedDocumentField(Comment))
             tags = ListField(StringField())
+            authors = ListField(ReferenceField(User))
 
         post = BlogPost(content='Went for a walk today...')
         post.validate()
@@ -210,15 +214,21 @@ class FieldTest(unittest.TestCase):
         post.tags = ('fun', 'leisure')
         post.validate()
 
-        comments = [Comment(content='Good for you'), Comment(content='Yay.')]
-        post.comments = comments
-        post.validate()
-
         post.comments = ['a']
         self.assertRaises(ValidationError, post.validate)
         post.comments = 'yay'
         self.assertRaises(ValidationError, post.validate)
 
+        comments = [Comment(content='Good for you'), Comment(content='Yay.')]
+        post.comments = comments
+        post.validate()
+
+        post.authors = [Comment()]
+        self.assertRaises(ValidationError, post.validate)
+
+        post.authors = [User()]
+        post.validate()
+
     def test_sorted_list_sorting(self):
         """Ensure that a sorted list field properly sorts values.
         """

From 3591593ac73dbd0a2275e5335b0ceaace232048d Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 13:55:48 +0100
Subject: [PATCH 24/32] Fixed GenericReferenceField query issue

---
 mongoengine/fields.py   | 2 +-
 mongoengine/queryset.py | 3 ---
 tests/fields.py         | 1 -
 3 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index f65ca15d..9fc50d68 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -509,7 +509,7 @@ class GenericReferenceField(BaseField):
         return {'_cls': document.__class__.__name__, '_ref': ref}
 
     def prepare_query_value(self, op, value):
-        return self.to_mongo(value)['_ref']
+        return self.to_mongo(value)
 
 
 class BinaryField(BaseField):
diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index a3d4c544..06524dcd 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -624,9 +624,6 @@ class QuerySet(object):
                     # 'in', 'nin' and 'all' require a list of values
                     value = [field.prepare_query_value(op, v) for v in value]
 
-                if field.__class__.__name__ == 'GenericReferenceField':
-                    parts.append('_ref')
-
             # if op and op not in match_operators:
             if op:
                 if op in geo_operators:
diff --git a/tests/fields.py b/tests/fields.py
index 1df484bf..e30f843e 100644
--- a/tests/fields.py
+++ b/tests/fields.py
@@ -545,7 +545,6 @@ class FieldTest(unittest.TestCase):
         user.save()
         
         user = User.objects(bookmarks__all=[post_1, link_1]).first()
-        print User.objects(bookmarks__all=[post_1, link_1]).explain()
         
         self.assertEqual(user.bookmarks[0], post_1)
         self.assertEqual(user.bookmarks[1], link_1)

From 26723992e30d1db03c40250c7635374d0cc502d8 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 14:14:05 +0100
Subject: [PATCH 25/32] Combined Q-object tests

---
 tests/queryset.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/queryset.py b/tests/queryset.py
index 32d1902e..38e1cb33 100644
--- a/tests/queryset.py
+++ b/tests/queryset.py
@@ -1393,9 +1393,6 @@ class QTest(unittest.TestCase):
         self.assertEqual(Post.objects.filter(created_user=user).count(), 1)
         self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1)
 
-
-class NewQTest(unittest.TestCase):
-
     def test_and_combination(self):
         """Ensure that Q-objects correctly AND together.
         """

From 012352cf24e55558c9638ff29ad36010f98e34a6 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 14:21:55 +0100
Subject: [PATCH 26/32] Added snapshot and timeout methods to QuerySet

---
 mongoengine/queryset.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 06524dcd..ead659d0 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -401,6 +401,8 @@ class QuerySet(object):
         self._where_clause = None
         self._loaded_fields = []
         self._ordering = []
+        self._snapshot = False
+        self._timeout = True
         
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -534,9 +536,12 @@ class QuerySet(object):
     @property
     def _cursor(self):
         if self._cursor_obj is None:
-            cursor_args = {}
+            cursor_args = {
+                'snapshot': self._snapshot,
+                'timeout': self._timeout,
+            }
             if self._loaded_fields:
-                cursor_args = {'fields': self._loaded_fields}
+                cursor_args['fields'] = self._loaded_fields
             self._cursor_obj = self._collection.find(self._query, 
                                                      **cursor_args)
             # Apply where clauses to cursor
@@ -967,6 +972,20 @@ class QuerySet(object):
             plan = pprint.pformat(plan)
         return plan
 
+    def snapshot(self, enabled):
+        """Enable or disable snapshot mode when querying.
+
+        :param enabled: whether or not snapshot mode is enabled
+        """
+        self._snapshot = enabled
+
+    def timeout(self, enabled):
+        """Enable or disable the default mongod timeout when querying.
+
+        :param enabled: whether or not the timeout is used
+        """
+        self._timeout = enabled
+
     def delete(self, safe=False):
         """Delete the documents matched by the query.
 

From 36993029ad4aa5b3a929e68b626eec083078738b Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 14:22:45 +0100
Subject: [PATCH 27/32] Removed old Q-object implementation

---
 mongoengine/queryset.py | 118 ----------------------------------------
 1 file changed, 118 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index ead659d0..f6296dd2 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -268,124 +268,6 @@ class Q(QNode):
         return not bool(self.query)
 
 
-class OldQ(object):
-
-    OR = '||'
-    AND = '&&'
-    OPERATORS = {
-        'eq': ('((this.%(field)s instanceof Array) && '
-               '  this.%(field)s.indexOf(%(value)s) != -1) ||'
-               ' this.%(field)s == %(value)s'),
-        'ne': 'this.%(field)s != %(value)s',
-        'gt': 'this.%(field)s > %(value)s',
-        'gte': 'this.%(field)s >= %(value)s',
-        'lt': 'this.%(field)s < %(value)s',
-        'lte': 'this.%(field)s <= %(value)s',
-        'lte': 'this.%(field)s <= %(value)s',
-        'in': '%(value)s.indexOf(this.%(field)s) != -1',
-        'nin': '%(value)s.indexOf(this.%(field)s) == -1',
-        'mod': '%(field)s %% %(value)s',
-        'all': ('%(value)s.every(function(a){'
-                'return this.%(field)s.indexOf(a) != -1 })'),
-        'size': 'this.%(field)s.length == %(value)s',
-        'exists': 'this.%(field)s != null',
-        'regex_eq': '%(value)s.test(this.%(field)s)',
-        'regex_ne': '!%(value)s.test(this.%(field)s)',
-    }
-
-    def __init__(self, **query):
-        self.query = [query]
-
-    def _combine(self, other, op):
-        obj = Q()
-        if not other.query[0]:
-            return self
-        if self.query[0]:
-            obj.query = (['('] + copy.deepcopy(self.query) + [op] +
-                         copy.deepcopy(other.query) + [')'])
-        else:
-            obj.query = copy.deepcopy(other.query)
-        return obj
-
-    def __or__(self, other):
-        return self._combine(other, self.OR)
-
-    def __and__(self, other):
-        return self._combine(other, self.AND)
-
-    def as_js(self, document):
-        js = []
-        js_scope = {}
-        for i, item in enumerate(self.query):
-            if isinstance(item, dict):
-                item_query = QuerySet._transform_query(document, **item)
-                # item_query will values will either be a value or a dict
-                js.append(self._item_query_as_js(item_query, js_scope, i))
-            else:
-                js.append(item)
-        return pymongo.code.Code(' '.join(js), js_scope)
-
-    def _item_query_as_js(self, item_query, js_scope, item_num):
-        # item_query will be in one of the following forms
-        #    {'age': 25, 'name': 'Test'}
-        #    {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']}
-        #    {'age': {'$lt': 25, '$gt': 18}}
-        js = []
-        for i, (key, value) in enumerate(item_query.items()):
-            op = 'eq'
-            # Construct a variable name for the value in the JS
-            value_name = 'i%sf%s' % (item_num, i)
-            if isinstance(value, dict):
-                # Multiple operators for this field
-                for j, (op, value) in enumerate(value.items()):
-                    # Create a custom variable name for this operator
-                    op_value_name = '%so%s' % (value_name, j)
-                    # Construct the JS that uses this op
-                    value, operation_js = self._build_op_js(op, key, value,
-                                                            op_value_name)
-                    # Update the js scope with the value for this op
-                    js_scope[op_value_name] = value
-                    js.append(operation_js)
-            else:
-                # Construct the JS for this field
-                value, field_js = self._build_op_js(op, key, value, value_name)
-                js_scope[value_name] = value
-                js.append(field_js)
-        return ' && '.join(js)
-
-    def _build_op_js(self, op, key, value, value_name):
-        """Substitute the values in to the correct chunk of Javascript.
-        """
-        if isinstance(value, RE_TYPE):
-            # Regexes are handled specially
-            if op.strip('$') == 'ne':
-                op_js = Q.OPERATORS['regex_ne']
-            else:
-                op_js = Q.OPERATORS['regex_eq']
-        else:
-            op_js = Q.OPERATORS[op.strip('$')]
-
-        # Comparing two ObjectIds in Javascript doesn't work..
-        if isinstance(value, pymongo.objectid.ObjectId):
-            value = unicode(value)
-
-        # Handle DBRef
-        if isinstance(value, pymongo.dbref.DBRef):
-            op_js = '(this.%(field)s.$id == "%(id)s" &&'\
-                    ' this.%(field)s.$ref == "%(ref)s")' % {
-                        'field': key,
-                        'id': unicode(value.id),
-                        'ref': unicode(value.collection)
-                    }
-            value = None
-
-        # Perform the substitution
-        operation_js = op_js % {
-            'field': key, 
-            'value': value_name
-        }
-        return value, operation_js
-
 class QuerySet(object):
     """A set of results returned from a query. Wraps a MongoDB cursor,
     providing :class:`~mongoengine.Document` objects as the results.

From 6817f3b7ba1b6f25554937179cd112a636140e01 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 15:40:49 +0100
Subject: [PATCH 28/32] Updated docs for v0.4

---
 docs/apireference.rst             |  2 ++
 docs/changelog.rst                | 11 ++++++-
 docs/guide/defining-documents.rst |  6 +++-
 docs/guide/querying.rst           | 51 +++++++++++++++++++++++++++++++
 docs/index.rst                    |  2 +-
 mongoengine/fields.py             |  6 ++++
 mongoengine/queryset.py           |  1 +
 7 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/docs/apireference.rst b/docs/apireference.rst
index 4fff317a..34d4536d 100644
--- a/docs/apireference.rst
+++ b/docs/apireference.rst
@@ -66,3 +66,5 @@ Fields
 .. autoclass:: mongoengine.GenericReferenceField
 
 .. autoclass:: mongoengine.FileField
+
+.. autoclass:: mongoengine.GeoPointField
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 8dd5b00d..64bdf90a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,16 +4,25 @@ Changelog
 
 Changes in v0.4
 ===============
+- New Q-object implementation, which is no longer based on Javascript
 - Added ``SortedListField``
 - Added ``EmailField``
 - Added ``GeoPointField``
 - Added ``exact`` and ``iexact`` match operators to ``QuerySet``
 - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts
-- Fixed bug in Q-objects
+- Added new query operators for Geo queries
+- Added ``not`` query operator
+- Added new update operators: ``pop`` and ``add_to_set``
+- Added ``__raw__`` query parameter
 - Fixed document inheritance primary key issue
+- Added support for querying by array element position
 - Base class can now be defined for ``DictField``
 - Fixed MRO error that occured on document inheritance
+- Added ``QuerySet.distinct``, ``QuerySet.create``, ``QuerySet.snapshot``,
+  ``QuerySet.timeout`` and ``QuerySet.all``
+- Subsequent calls to ``connect()`` now work
 - Introduced ``min_length`` for ``StringField``
+- Fixed multi-process connection issue
 - Other minor fixes
 
 Changes in v0.3
diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
index dc136f27..106d4ec8 100644
--- a/docs/guide/defining-documents.rst
+++ b/docs/guide/defining-documents.rst
@@ -47,11 +47,11 @@ are as follows:
 * :class:`~mongoengine.ReferenceField`
 * :class:`~mongoengine.GenericReferenceField`
 * :class:`~mongoengine.BooleanField`
-* :class:`~mongoengine.GeoLocationField`
 * :class:`~mongoengine.FileField`
 * :class:`~mongoengine.EmailField`
 * :class:`~mongoengine.SortedListField`
 * :class:`~mongoengine.BinaryField`
+* :class:`~mongoengine.GeoPointField`
 
 Field arguments
 ---------------
@@ -298,6 +298,10 @@ or a **-** sign. Note that direction only matters on multi-field indexes. ::
         meta = {
             'indexes': ['title', ('title', '-rating')]
         }
+
+.. note::
+   Geospatial indexes will be automatically created for all 
+   :class:`~mongoengine.GeoPointField`\ s
         
 Ordering
 ========
diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
index bef19bc5..58bf9f63 100644
--- a/docs/guide/querying.rst
+++ b/docs/guide/querying.rst
@@ -53,6 +53,16 @@ lists that contain that item will be matched::
     # 'tags' list
     Page.objects(tags='coding')
 
+Raw queries
+-----------
+It is possible to provide a raw PyMongo query as a query parameter, which will
+be integrated directly into the query. This is done using the ``__raw__``
+keyword argument::
+
+    Page.objects(__raw__={'tags': 'coding'})
+
+.. versionadded:: 0.4
+
 Query operators
 ===============
 Operators other than equality may also be used in queries; just attach the
@@ -68,6 +78,8 @@ Available operators are as follows:
 * ``lte`` -- less than or equal to
 * ``gt`` -- greater than
 * ``gte`` -- greater than or equal to
+* ``not`` -- negate a standard check, may be used before other operators (e.g.
+  ``Q(age__not__mod=5)``)
 * ``in`` -- value is in list (a list of values should be provided)
 * ``nin`` -- value is not in list (a list of values should be provided)
 * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values
@@ -89,6 +101,27 @@ expressions:
 
 .. versionadded:: 0.3
 
+There are a few special operators for performing geographical queries, that
+may used with :class:`~mongoengine.GeoPointField`\ s:
+
+* ``within_distance`` -- provide a list containing a point and a maximum
+  distance (e.g. [(41.342, -87.653), 5])
+* ``within_box`` -- filter documents to those within a given bounding box (e.g.
+  [(35.0, -125.0), (40.0, -100.0)])
+* ``near`` -- order the documents by how close they are to a given point
+
+.. versionadded:: 0.4
+
+Querying by position
+====================
+It is possible to query by position in a list by using a numerical value as a
+query operator. So if you wanted to find all pages whose first tag was ``db``,
+you could use the following query::
+
+    BlogPost.objects(tags__0='db')
+
+.. versionadded:: 0.4
+
 Limiting and skipping results
 =============================
 Just as with traditional ORMs, you may limit the number of results returned, or
@@ -181,6 +214,22 @@ custom manager methods as you like::
     assert len(BlogPost.objects) == 2
     assert len(BlogPost.live_posts) == 1
 
+Custom QuerySets
+================
+Should you want to add custom methods for interacting with or filtering
+documents, extending the :class:`~mongoengine.queryset.QuerySet` class may be
+the way to go. To use a custom :class:`~mongoengine.queryset.QuerySet` class on
+a document, set ``queryset_class`` to the custom class in a
+:class:`~mongoengine.Document`\ s ``meta`` dictionary::
+
+    class AwesomerQuerySet(QuerySet):
+        pass
+
+    class Page(Document):
+        meta = {'queryset_class': AwesomerQuerySet}
+
+.. versionadded:: 0.4
+
 Aggregation
 ===========
 MongoDB provides some aggregation methods out of the box, but there are not as
@@ -402,8 +451,10 @@ that you may use with these methods:
 * ``pop`` -- remove the last item from a list
 * ``push`` -- append a value to a list
 * ``push_all`` -- append several values to a list
+* ``pop`` -- remove the first or last element of a list
 * ``pull`` -- remove a value from a list
 * ``pull_all`` -- remove several values from a list
+* ``add_to_set`` -- add value to a list only if its not in the list already
 
 The syntax for atomic updates is similar to the querying syntax, but the 
 modifier comes before the field, not after it::
diff --git a/docs/index.rst b/docs/index.rst
index a28b344c..ccb7fbe2 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -7,7 +7,7 @@ MongoDB. To install it, simply run
 
 .. code-block:: console
 
-    # easy_install -U mongoengine
+    # pip install -U mongoengine
 
 The source is available on `GitHub <http://github.com/hmarr/mongoengine>`_.
 
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 9fc50d68..62d2ef2f 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -112,6 +112,8 @@ class URLField(StringField):
 
 class EmailField(StringField):
     """A field that validates input as an E-Mail-Address.
+
+    .. versionadded:: 0.4
     """
 
     EMAIL_REGEX = re.compile(
@@ -355,6 +357,8 @@ class SortedListField(ListField):
     """A ListField that sorts the contents of its list before writing to
     the database in order to ensure that a sorted list is always
     retrieved.
+
+    .. versionadded:: 0.4
     """
 
     _ordering = None
@@ -666,6 +670,8 @@ class FileField(BaseField):
 
 class GeoPointField(BaseField):
     """A list storing a latitude and longitude.
+
+    .. versionadded:: 0.4
     """
 
     _geo_index = True
diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index f6296dd2..86f8b1d5 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -590,6 +590,7 @@ class QuerySet(object):
 
     def create(self, **kwargs):
         """Create new object. Returns the saved object instance.
+
         .. versionadded:: 0.4
         """
         doc = self._document(**kwargs)

From 007f116bfaec40205626075ef9050ab35b1753df Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 15:42:31 +0100
Subject: [PATCH 29/32] Increment version to 0.4

---
 docs/changelog.rst      | 1 +
 mongoengine/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 64bdf90a..277d8e96 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -14,6 +14,7 @@ Changes in v0.4
 - Added ``not`` query operator
 - Added new update operators: ``pop`` and ``add_to_set``
 - Added ``__raw__`` query parameter
+- Added support for custom querysets
 - Fixed document inheritance primary key issue
 - Added support for querying by array element position
 - Base class can now be defined for ``DictField``
diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index e01d31ae..6d18ffe7 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -12,7 +12,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
 
 __author__ = 'Harry Marr'
 
-VERSION = (0, 3, 0)
+VERSION = (0, 4, 0)
 
 def get_version():
     version = '%s.%s' % (VERSION[0], VERSION[1])

From dcec61e9b2738f9fc3bd0f09a2cb1bc631438c66 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 16:36:22 +0100
Subject: [PATCH 30/32] Raise AttributeError when necessary on QuerySet[]

---
 mongoengine/queryset.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 86f8b1d5..a8d739d9 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -787,6 +787,7 @@ class QuerySet(object):
         # Integer index provided
         elif isinstance(key, int):
             return self._document._from_son(self._cursor[key])
+        raise AttributeError
     
     def distinct(self, field):
         """Return a list of distinct values for a given field.

From e93c4c87d8953db1c4da24415c0da3c463523ab9 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Sun, 17 Oct 2010 17:41:20 +0100
Subject: [PATCH 31/32] Fixed inheritance collection issue

---
 mongoengine/queryset.py | 15 +++++++--------
 tests/document.py       | 31 +++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index a8d739d9..7422bb8f 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -1146,9 +1146,8 @@ class QuerySetManager(object):
             return self
 
         db = _get_db()
-        if db not in self._collections:
-            collection = owner._meta['collection']
-
+        collection = owner._meta['collection']
+        if (db, collection) not in self._collections:
             # Create collection as a capped collection if specified
             if owner._meta['max_size'] or owner._meta['max_documents']:
                 # Get max document limit and max byte size from meta
@@ -1156,10 +1155,10 @@ class QuerySetManager(object):
                 max_documents = owner._meta['max_documents']
 
                 if collection in db.collection_names():
-                    self._collections[db] = db[collection]
+                    self._collections[(db, collection)] = db[collection]
                     # The collection already exists, check if its capped
                     # options match the specified capped options
-                    options = self._collections[db].options()
+                    options = self._collections[(db, collection)].options()
                     if options.get('max') != max_documents or \
                        options.get('size') != max_size:
                         msg = ('Cannot create collection "%s" as a capped '
@@ -1170,15 +1169,15 @@ class QuerySetManager(object):
                     opts = {'capped': True, 'size': max_size}
                     if max_documents:
                         opts['max'] = max_documents
-                    self._collections[db] = db.create_collection(
+                    self._collections[(db, collection)] = db.create_collection(
                         collection, **opts
                     )
             else:
-                self._collections[db] = db[collection]
+                self._collections[(db, collection)] = db[collection]
 
         # owner is the document that contains the QuerySetManager
         queryset_class = owner._meta['queryset_class'] or QuerySet
-        queryset = queryset_class(owner, self._collections[db])
+        queryset = queryset_class(owner, self._collections[(db, collection)])
         if self._manager_func:
             if self._manager_func.func_code.co_argcount == 1:
                 queryset = self._manager_func(queryset)
diff --git a/tests/document.py b/tests/document.py
index c2e0d6f9..aa901813 100644
--- a/tests/document.py
+++ b/tests/document.py
@@ -200,6 +200,37 @@ class DocumentTest(unittest.TestCase):
         Person.drop_collection()
         self.assertFalse(collection in self.db.collection_names())
 
+    def test_inherited_collections(self):
+        """Ensure that subclassed documents don't override parents' collections.
+        """
+        class Drink(Document):
+            name = StringField()
+
+        class AlcoholicDrink(Drink):
+            meta = {'collection': 'booze'}
+
+        class Drinker(Document):
+            drink = GenericReferenceField()
+
+        Drink.drop_collection()
+        AlcoholicDrink.drop_collection()
+        Drinker.drop_collection()
+
+        red_bull = Drink(name='Red Bull')
+        red_bull.save()
+
+        programmer = Drinker(drink=red_bull)
+        programmer.save()
+
+        beer = AlcoholicDrink(name='Beer')
+        beer.save()
+
+        real_person = Drinker(drink=beer)
+        real_person.save()
+
+        self.assertEqual(Drinker.objects[0].drink.name, red_bull.name)
+        self.assertEqual(Drinker.objects[1].drink.name, beer.name)
+
     def test_capped_collection(self):
         """Ensure that capped collections work properly.
         """

From 0902b957641e09861f0864e09501b174624577f0 Mon Sep 17 00:00:00 2001
From: Harry Marr <harry.marr@gmail.com>
Date: Mon, 18 Oct 2010 00:27:40 +0100
Subject: [PATCH 32/32] Added support for recursive embedded documents

---
 mongoengine/base.py   |  7 +++----
 mongoengine/fields.py | 33 +++++++++++++++++++++------------
 tests/fields.py       | 30 ++++++++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/mongoengine/base.py b/mongoengine/base.py
index 1f7ba1fe..2253e4a2 100644
--- a/mongoengine/base.py
+++ b/mongoengine/base.py
@@ -204,6 +204,9 @@ class DocumentMetaclass(type):
         exc = subclass_exception('MultipleObjectsReturned', base_excs, module)
         new_class.add_to_class('MultipleObjectsReturned', exc)
     
+        global _document_registry
+        _document_registry[name] = new_class
+
         return new_class
         
     def add_to_class(self, name, value):
@@ -216,8 +219,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
     """
 
     def __new__(cls, name, bases, attrs):
-        global _document_registry
-
         super_new = super(TopLevelDocumentMetaclass, cls).__new__
         # Classes defined in this package are abstract and should not have 
         # their own metadata with DB collection, etc.
@@ -322,8 +323,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             new_class._fields['id'] = ObjectIdField(db_field='_id')
             new_class.id = new_class._fields['id']
 
-        _document_registry[name] = new_class
-
         return new_class
 
 
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 62d2ef2f..63107f23 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -233,33 +233,43 @@ class EmbeddedDocumentField(BaseField):
     :class:`~mongoengine.EmbeddedDocument`.
     """
 
-    def __init__(self, document, **kwargs):
-        if not issubclass(document, EmbeddedDocument):
-            raise ValidationError('Invalid embedded document class provided '
-                                  'to an EmbeddedDocumentField')
-        self.document = document
+    def __init__(self, document_type, **kwargs):
+        if not isinstance(document_type, basestring):
+            if not issubclass(document_type, EmbeddedDocument):
+                raise ValidationError('Invalid embedded document class '
+                                      'provided to an EmbeddedDocumentField')
+        self.document_type_obj = document_type
         super(EmbeddedDocumentField, self).__init__(**kwargs)
 
+    @property
+    def document_type(self):
+        if isinstance(self.document_type_obj, basestring):
+            if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT:
+                self.document_type_obj = self.owner_document
+            else:
+                self.document_type_obj = get_document(self.document_type_obj)
+        return self.document_type_obj
+
     def to_python(self, value):
-        if not isinstance(value, self.document):
-            return self.document._from_son(value)
+        if not isinstance(value, self.document_type):
+            return self.document_type._from_son(value)
         return value
 
     def to_mongo(self, value):
-        return self.document.to_mongo(value)
+        return self.document_type.to_mongo(value)
 
     def validate(self, value):
         """Make sure that the document instance is an instance of the
         EmbeddedDocument subclass provided when the document was defined.
         """
         # Using isinstance also works for subclasses of self.document
-        if not isinstance(value, self.document):
+        if not isinstance(value, self.document_type):
             raise ValidationError('Invalid embedded document instance '
                                   'provided to an EmbeddedDocumentField')
-        self.document.validate(value)
+        self.document_type.validate(value)
 
     def lookup_member(self, member_name):
-        return self.document._fields.get(member_name)
+        return self.document_type._fields.get(member_name)
 
     def prepare_query_value(self, op, value):
         return self.to_mongo(value)
@@ -413,7 +423,6 @@ class ReferenceField(BaseField):
                 raise ValidationError('Argument to ReferenceField constructor '
                                       'must be a document class or a string')
         self.document_type_obj = document_type
-        self.document_obj = None
         super(ReferenceField, self).__init__(**kwargs)
 
     @property
diff --git a/tests/fields.py b/tests/fields.py
index e30f843e..208b4643 100644
--- a/tests/fields.py
+++ b/tests/fields.py
@@ -423,6 +423,36 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(peter.boss, bill)
         self.assertEqual(peter.friends, friends)
 
+    def test_recursive_embedding(self):
+        """Ensure that EmbeddedDocumentFields can contain their own documents.
+        """
+        class Tree(Document):
+            name = StringField()
+            children = ListField(EmbeddedDocumentField('TreeNode'))
+
+        class TreeNode(EmbeddedDocument):
+            name = StringField()
+            children = ListField(EmbeddedDocumentField('self'))
+        
+        tree = Tree(name="Tree")
+
+        first_child = TreeNode(name="Child 1")
+        tree.children.append(first_child)
+
+        second_child = TreeNode(name="Child 2")
+        first_child.children.append(second_child)
+        
+        third_child = TreeNode(name="Child 3")
+        first_child.children.append(third_child)
+
+        tree.save()
+
+        tree_obj = Tree.objects.first()
+        self.assertEqual(len(tree.children), 1)
+        self.assertEqual(tree.children[0].name, first_child.name)
+        self.assertEqual(tree.children[0].children[0].name, second_child.name)
+        self.assertEqual(tree.children[0].children[1].name, third_child.name)
+
     def test_undefined_reference(self):
         """Ensure that ReferenceFields may reference undefined Documents.
         """