Merge branch 'master' into 391

docs/apireference.rst

@@ -49,6 +49,11 @@ Querying
 
    .. automethod:: mongoengine.queryset.QuerySet.__call__
 
+.. autoclass:: mongoengine.queryset.QuerySetNoCache
+   :members:
+
+   .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__
+
 .. autofunction:: mongoengine.queryset.queryset_manager
 
 Fields

docs/changelog.rst

@@ -4,7 +4,8 @@ Changelog
 
 Changes in 0.8.3
 ================
-- Fixed sum and average mapreduce dot notation support (#375, #376)
+- Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365)
+- Fixed sum and average mapreduce dot notation support (#375, #376, #393)
 - Fixed as_pymongo to return the id (#386)
 - Document.select_related() now respects `db_alias` (#377)
 - Reload uses shard_key if applicable (#384)

docs/guide/querying.rst

@@ -16,7 +16,9 @@ fetch documents from the database::
 .. note::
 
     As of MongoEngine 0.8 the querysets utilise a local cache.  So iterating
-    it multiple times will only cause a single query.
+    it multiple times will only cause a single query.  If this is not the
+    desired behavour you can call :class:`~mongoengine.QuerySet.no_cache` to
+    return a non-caching queryset.
 
 Filtering queries
 =================

mongoengine/base/document.py

@@ -160,7 +160,7 @@ class BaseDocument(object):
                   '_fields_ordered', '_dynamic_fields'):
             if k in data:
                 setattr(self, k, data[k])
-        for k in data.get('_dynamic_fields').keys():
+        for k in data.get('_dynamic_fields', SON()).keys():
             setattr(self, k, data["_data"].get(k))
 
     def __iter__(self):

mongoengine/fields.py

@@ -279,14 +279,14 @@ class DecimalField(BaseField):
         :param precision: Number of decimal places to store.
         :param rounding: The rounding rule from the python decimal libary:
 
-            - decimial.ROUND_CEILING (towards Infinity)
-            - decimial.ROUND_DOWN (towards zero)
-            - decimial.ROUND_FLOOR (towards -Infinity)
-            - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero)
-            - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer)
-            - decimial.ROUND_HALF_UP (to nearest with ties going away from zero)
-            - decimial.ROUND_UP (away from zero)
-            - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
+            - decimal.ROUND_CEILING (towards Infinity)
+            - decimal.ROUND_DOWN (towards zero)
+            - decimal.ROUND_FLOOR (towards -Infinity)
+            - decimal.ROUND_HALF_DOWN (to nearest with ties going towards zero)
+            - decimal.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer)
+            - decimal.ROUND_HALF_UP (to nearest with ties going away from zero)
+            - decimal.ROUND_UP (away from zero)
+            - decimal.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero)
 
             Defaults to: ``decimal.ROUND_HALF_UP``
 

tests/queryset/queryset.py

@@ -2246,6 +2246,145 @@ class QuerySetTest(unittest.TestCase):
         self.Person(name='weightless person').save()
         self.assertEqual(int(self.Person.objects.sum('age')), sum(ages))
 
+    def test_embedded_average(self):
+        class Pay(EmbeddedDocument):
+            value = DecimalField()
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(value=150)).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(value=530)).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(value=165)).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(value=115)).save()
+
+        self.assertEqual(
+            Doc.objects.average('pay.value'),
+            240)
+
+    def test_embedded_array_average(self):
+        class Pay(EmbeddedDocument):
+            values = ListField(DecimalField())
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(values=[150, 100])).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(values=[530, 100])).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(values=[165, 100])).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(values=[115, 100])).save()
+
+        self.assertEqual(
+            Doc.objects.average('pay.values'),
+            170)
+
+    def test_array_average(self):
+        class Doc(Document):
+            values = ListField(DecimalField())
+
+        Doc.drop_collection()
+
+        Doc(values=[150, 100]).save()
+        Doc(values=[530, 100]).save()
+        Doc(values=[165, 100]).save()
+        Doc(values=[115, 100]).save()
+
+        self.assertEqual(
+            Doc.objects.average('values'),
+            170)
+
+    def test_embedded_sum(self):
+        class Pay(EmbeddedDocument):
+            value = DecimalField()
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(value=150)).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(value=530)).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(value=165)).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(value=115)).save()
+
+        self.assertEqual(
+            Doc.objects.sum('pay.value'),
+            960)
+
+
+    def test_embedded_array_sum(self):
+        class Pay(EmbeddedDocument):
+            values = ListField(DecimalField())
+
+        class Doc(Document):
+            name = StringField()
+            pay = EmbeddedDocumentField(
+                Pay)
+
+        Doc.drop_collection()
+
+        Doc(name=u"Wilson Junior",
+            pay=Pay(values=[150, 100])).save()
+
+        Doc(name=u"Isabella Luanna",
+            pay=Pay(values=[530, 100])).save()
+
+        Doc(name=u"Tayza mariana",
+            pay=Pay(values=[165, 100])).save()
+
+        Doc(name=u"Eliana Costa",
+            pay=Pay(values=[115, 100])).save()
+
+        self.assertEqual(
+            Doc.objects.sum('pay.values'),
+            1360)
+
+    def test_array_sum(self):
+        class Doc(Document):
+            values = ListField(DecimalField())
+
+        Doc.drop_collection()
+
+        Doc(values=[150, 100]).save()
+        Doc(values=[530, 100]).save()
+        Doc(values=[165, 100]).save()
+        Doc(values=[115, 100]).save()
+
+        self.assertEqual(
+            Doc.objects.sum('values'),
+            1360)
+
     def test_distinct(self):
         """Ensure that the QuerySet.distinct method works.
         """
@@ -3254,7 +3393,7 @@ class QuerySetTest(unittest.TestCase):
         User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
 
         results = User.objects.only('id', 'name').as_pymongo()
-        self.assertEqual(results[0].keys(), ['_id', 'name'])
+        self.assertEqual(sorted(results[0].keys()), sorted(['_id', 'name']))
 
         users = User.objects.only('name', 'price').as_pymongo()
         results = list(users)
@@ -3365,6 +3504,34 @@ class QuerySetTest(unittest.TestCase):
         self.assertEqual("%s" % users, "[<User: Bob>]")
         self.assertEqual(1, len(users._result_cache))
 
+    def test_no_cache(self):
+        """Ensure you can add meta data to file"""
+
+        class Noddy(Document):
+            fields = DictField()
+
+        Noddy.drop_collection()
+        for i in xrange(100):
+            noddy = Noddy()
+            for j in range(20):
+                noddy.fields["key"+str(j)] = "value "+str(j)
+            noddy.save()
+
+        docs = Noddy.objects.no_cache()
+
+        counter = len([1 for i in docs])
+        self.assertEquals(counter, 100)
+
+        self.assertEquals(len(list(docs)), 100)
+        self.assertRaises(TypeError, lambda: len(docs))
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            list(docs)
+            self.assertEqual(q, 1)
+            list(docs)
+            self.assertEqual(q, 2)
+
     def test_nested_queryset_iterator(self):
         # Try iterating the same queryset twice, nested.
         names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George']