Merge branch 'dev' of git://github.com/hmarr/mongoengine into dev

AUTHORS

@@ -1,3 +1,5 @@
+The PRIMARY AUTHORS are (and/or have been):
+
 Harry Marr <harry@hmarr.com>
 Matt Dennewitz <mattdennewitz@gmail.com>
 Deepak Thukral <iapain@yahoo.com>
@@ -5,3 +7,62 @@ Florian Schlachter <flori@n-schlachter.de>
 Steve Challis <steve@stevechallis.com>
 Ross Lawley <ross.lawley@gmail.com>
 Wilson Júnior <wilsonpjunior@gmail.com>
+Dan Crosta https://github.com/dcrosta
+
+CONTRIBUTORS
+
+Dervived from the git logs, inevitably incomplete but all of whom and others
+have submitted patches, reported bugs and generally helped make MongoEngine
+that much better:
+
+ * Harry Marr
+ * Ross Lawley
+ * blackbrrr
+ * Florian Schlachter
+ * Vincent Driessen
+ * Steve Challis
+ * flosch
+ * Deepak Thukral
+ * Colin Howe
+ * Wilson Júnior
+ * Alistair Roche
+ * Dan Crosta
+ * Viktor Kerkez
+ * Stephan Jaekel
+ * Rached Ben Mustapha
+ * Greg Turner
+ * Daniel Hasselrot
+ * Mircea Pasoi
+ * Matt Chisholm
+ * James Punteney
+ * TimothéePeignier
+ * Stuart Rackham
+ * Serge Matveenko
+ * Matt Dennewitz
+ * Don Spaulding
+ * Ales Zoulek
+ * sshwsfc
+ * sib
+ * Samuel Clay
+ * Nick Vlku
+ * martin
+ * Flavio Amieiro
+ * Анхбаяр Лхагвадорж
+ * Zak Johnson
+ * Victor Farazdagi
+ * vandersonmota
+ * Theo Julienne
+ * sp
+ * Slavi Pantaleev
+ * Richard Henry
+ * Nicolas Perriault
+ * Nick Vlku Jr
+ * Michael Henson
+ * Leo Honkanen
+ * kuno
+ * Josh Ourisman
+ * Jaime
+ * Igor Ivanov
+ * Gregg Lind
+ * Gareth Lloyd
+ * Albert Choi

docs/changelog.rst

@@ -5,6 +5,8 @@ Changelog
 Changes in dev
 ==============
 
+- Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments
+- Added where() - filter to allowing users to specify query expressions as Javascript
 - Added SequenceField - for creating sequential counters
 - Added update() convenience method to a document
 - Added cascading saves - so changes to Referenced documents are saved on .save()
@@ -28,7 +30,7 @@ Changes in dev
 - Added insert method for bulk inserts
 - Added blinker signal support
 - Added query_counter context manager for tests
-- Added optional map_reduce method item_frequencies
+- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments)
 - Added inline_map_reduce option to map_reduce
 - Updated connection exception so it provides more info on the cause.
 - Added searching multiple levels deep in ``DictField``

mongoengine/base.py

@@ -812,7 +812,8 @@ class BaseDocument(object):
                 field_cls = field.document_type
                 if field_cls in inspected_classes:
                     continue
-                geo_indices += field_cls._geo_indices(inspected_classes)
+                if hasattr(field_cls, '_geo_indices'):
+                    geo_indices += field_cls._geo_indices(inspected_classes)
             elif field._geo_index:
                 geo_indices.append(field)
         return geo_indices

mongoengine/fields.py

@@ -620,7 +620,7 @@ class GenericReferenceField(BaseField):
     """A reference to *any* :class:`~mongoengine.document.Document` subclass
     that will be automatically dereferenced on access (lazily).
 
-    note:  Any documents used as a generic reference must be registered in the
+    ..note ::  Any documents used as a generic reference must be registered in the
     document registry.  Importing the model will automatically register it.
 
     .. versionadded:: 0.3
@@ -925,7 +925,7 @@ class SequenceField(IntField):
         return value
 
     def __set__(self, instance, value):
-        
+
         if value is None and instance._initialised:
             value = self.generate_new_value()
 

mongoengine/queryset.py

@@ -1397,22 +1397,45 @@ class QuerySet(object):
         db = _get_db()
         return db.eval(code, *fields)
 
+    def where(self, where_clause):
+        """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript
+        expression). Performs automatic field name substitution like
+        :meth:`mongoengine.queryset.Queryset.exec_js`.
+
+        .. note:: When using this mode of query, the database will call your
+                  function, or evaluate your predicate clause, for each object
+                  in the collection.
+        """
+        where_clause = self._sub_js_fields(where_clause)
+        self._where_clause = where_clause
+        return self
+
     def sum(self, field):
         """Sum over the values of the specified field.
 
         :param field: the field to sum over; use dot-notation to refer to
             embedded document fields
         """
-        sum_func = """
+        map_func = pymongo.code.Code("""
-            function(sumField) {
+            function() {
-                var total = 0.0;
+                emit(1, this[field] || 0);
-                db[collection].find(query).forEach(function(doc) {
-                    total += (doc[sumField] || 0.0);
-                });
-                return total;
             }
-        """
+        """, scope={'field': field})
-        return self.exec_js(sum_func, field)
+
+        reduce_func = pymongo.code.Code("""
+            function(key, values) {
+                var sum = 0;
+                for (var i in values) {
+                    sum += values[i];
+                }
+                return sum;
+            }
+        """)
+
+        for result in self.map_reduce(map_func, reduce_func, output='inline'):
+            return result.value
+        else:
+            return 0
 
     def average(self, field):
         """Average over the values of the specified field.
@@ -1420,22 +1443,38 @@ class QuerySet(object):
         :param field: the field to average over; use dot-notation to refer to
             embedded document fields
         """
-        average_func = """
+        map_func = pymongo.code.Code("""
-            function(averageField) {
+            function() {
-                var total = 0.0;
+                if (this.hasOwnProperty(field))
-                var num = 0;
+                    emit(1, {t: this[field] || 0, c: 1});
-                db[collection].find(query).forEach(function(doc) {
-                    if (doc[averageField] !== undefined) {
-                        total += doc[averageField];
-                        num += 1;
-                    }
-                });
-                return total / num;
             }
-        """
+        """, scope={'field': field})
-        return self.exec_js(average_func, field)
 
-    def item_frequencies(self, field, normalize=False, map_reduce=False):
+        reduce_func = pymongo.code.Code("""
+            function(key, values) {
+                var out = {t: 0, c: 0};
+                for (var i in values) {
+                    var value = values[i];
+                    out.t += value.t;
+                    out.c += value.c;
+                }
+                return out;
+            }
+        """)
+
+        finalize_func = pymongo.code.Code("""
+            function(key, value) {
+                return value.t / value.c;
+            }
+        """)
+
+        for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'):
+            return result.value
+        else:
+            return 0
+
+
+    def item_frequencies(self, field, normalize=False, map_reduce=True):
         """Returns a dictionary of all items present in a field across
         the whole queried set of documents, and their corresponding frequency.
         This is useful for generating tag clouds, or searching documents.

tests/document.py

@@ -690,6 +690,57 @@ class DocumentTest(unittest.TestCase):
 
         BlogPost.drop_collection()
 
+    def test_embedded_document_index(self):
+        """Tests settings an index on an embedded document
+        """
+        class Date(EmbeddedDocument):
+            year = IntField(db_field='yr')
+
+        class BlogPost(Document):
+            title = StringField()
+            date = EmbeddedDocumentField(Date)
+
+            meta = {
+                'indexes': [
+                    '-date.year'
+                ],
+            }
+
+        BlogPost.drop_collection()
+
+        info = BlogPost.objects._collection.index_information()
+        self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_'])
+        BlogPost.drop_collection()
+
+    def test_list_embedded_document_index(self):
+        """Ensure list embedded documents can be indexed
+        """
+        class Tag(EmbeddedDocument):
+            name = StringField(db_field='tag')
+
+        class BlogPost(Document):
+            title = StringField()
+            tags = ListField(EmbeddedDocumentField(Tag))
+
+            meta = {
+                'indexes': [
+                    'tags.name'
+                ],
+            }
+
+        BlogPost.drop_collection()
+
+        info = BlogPost.objects._collection.index_information()
+        # we don't use _types in with list fields by default
+        self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1'])
+
+        post1 = BlogPost(title="Embedded Indexes tests in place",
+                        tags=[Tag(name="about"), Tag(name="time")]
+                )
+        post1.save()
+        BlogPost.drop_collection()
+
+
     def test_geo_indexes_recursion(self):
 
         class User(Document):

tests/queryset.py

@@ -2502,6 +2502,47 @@ class QuerySetTest(unittest.TestCase):
                 for key, value in info.iteritems()]
         self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info)
 
+    def test_where(self):
+        """Ensure that where clauses work.
+        """
+
+        class IntPair(Document):
+            fielda = IntField()
+            fieldb = IntField()
+
+        IntPair.objects._collection.remove()
+
+        a = IntPair(fielda=1, fieldb=1)
+        b = IntPair(fielda=1, fieldb=2)
+        c = IntPair(fielda=2, fieldb=1)
+        a.save()
+        b.save()
+        c.save()
+
+        query = IntPair.objects.where('this[~fielda] >= this[~fieldb]')
+        self.assertEqual('this["fielda"] >= this["fieldb"]', query._where_clause)
+        results = list(query)
+        self.assertEqual(2, len(results))
+        self.assertTrue(a in results)
+        self.assertTrue(c in results)
+
+        query = IntPair.objects.where('this[~fielda] == this[~fieldb]')
+        results = list(query)
+        self.assertEqual(1, len(results))
+        self.assertTrue(a in results)
+
+        query = IntPair.objects.where('function() { return this[~fielda] >= this[~fieldb] }')
+        self.assertEqual('function() { return this["fielda"] >= this["fieldb"] }', query._where_clause)
+        results = list(query)
+        self.assertEqual(2, len(results))
+        self.assertTrue(a in results)
+        self.assertTrue(c in results)
+
+        def invalid_where():
+            list(IntPair.objects.where(fielda__gte=3))
+
+        self.assertRaises(TypeError, invalid_where)
+
 
 class QTest(unittest.TestCase):