From 13afead9fb43ff6c1150bd21a5084b260116596a Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Wed, 20 Jul 2011 12:41:20 -0400 Subject: [PATCH 1/9] add where() method to QuerySet --- mongoengine/queryset.py | 5 +++++ tests/queryset.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11c7a804..b1185ee0 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1397,6 +1397,11 @@ class QuerySet(object): db = _get_db() return db.eval(code, *fields) + def where(self, where_clause): + where_clause = self._sub_js_fields(where_clause) + self._where_clause = where_clause + return self + def sum(self, field): """Sum over the values of the specified field. diff --git a/tests/queryset.py b/tests/queryset.py index a21bae69..ce64a004 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2502,6 +2502,34 @@ class QuerySetTest(unittest.TestCase): for key, value in info.iteritems()] self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + def test_where(self): + """Ensure that where clauses work. + """ + + class IntPair(Document): + fielda = IntField() + fieldb = IntField() + + IntPair.objects._collection.remove() + + a = IntPair(fielda=1, fieldb=1) + b = IntPair(fielda=1, fieldb=2) + c = IntPair(fielda=2, fieldb=1) + a.save() + b.save() + c.save() + + query = IntPair.objects.where('this[~fielda] >= this[~fieldb]') + self.assertEqual('this["fielda"] >= this["fieldb"]', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + query = IntPair.objects.where('this[~fielda] == this[~fieldb]') + results = list(query) + self.assertEqual(1, len(results)) + self.assertTrue(a in results) class QTest(unittest.TestCase): From 4abfcb0188257abb7955efec24e54a4ea3d12e7d Mon Sep 17 00:00:00 2001 From: Gareth Lloyd Date: Mon, 15 Aug 2011 10:01:48 +0100 Subject: [PATCH 2/9] check for presence of _geo_indices on field class before referencing --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index e224367c..6be5c3de 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -812,7 +812,8 @@ class BaseDocument(object): field_cls = field.document_type if field_cls in inspected_classes: continue - geo_indices += field_cls._geo_indices(inspected_classes) + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected_classes) elif field._geo_index: geo_indices.append(field) return geo_indices From 81b69648efaf01e64aefe0260aac78cb8147d407 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Mon, 15 Aug 2011 16:56:42 -0400 Subject: [PATCH 3/9] docstring for `where()` --- mongoengine/queryset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b1185ee0..c8945142 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1398,6 +1398,10 @@ class QuerySet(object): return db.eval(code, *fields) def where(self, where_clause): + """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript + expression). Performs automatic field name substitution like + :meth:`mongoengine.queryset.Queryset.exec_js`. + """ where_clause = self._sub_js_fields(where_clause) self._where_clause = where_clause return self From 3f301f6b0f5c56b731a37ae82442a5e90315972e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 10:32:21 +0100 Subject: [PATCH 4/9] Finishing touches to where implementation - thanks to dcrosta Refs #242 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++-- mongoengine/queryset.py | 4 ++++ tests/queryset.py | 13 +++++++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index b13af2b0..fbf78cf6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,3 +5,4 @@ Florian Schlachter Steve Challis Ross Lawley Wilson Júnior +Dan Crosta https://github.com/dcrosta diff --git a/docs/changelog.rst b/docs/changelog.rst index f3a4b944..87247d57 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters - Added update() convenience method to a document - Added cascading saves - so changes to Referenced documents are saved on .save() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b2f1e2a2..619b8c60 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -620,7 +620,7 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - note: Any documents used as a generic reference must be registered in the + ..note :: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. .. versionadded:: 0.3 @@ -925,7 +925,7 @@ class SequenceField(IntField): return value def __set__(self, instance, value): - + if value is None and instance._initialised: value = self.generate_new_value() diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c8945142..303fcc1b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1401,6 +1401,10 @@ class QuerySet(object): """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript expression). Performs automatic field name substitution like :meth:`mongoengine.queryset.Queryset.exec_js`. + + .. note:: When using this mode of query, the database will call your + function, or evaluate your predicate clause, for each object + in the collection. """ where_clause = self._sub_js_fields(where_clause) self._where_clause = where_clause diff --git a/tests/queryset.py b/tests/queryset.py index ce64a004..6ae1c10f 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2531,6 +2531,19 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(1, len(results)) self.assertTrue(a in results) + query = IntPair.objects.where('function() { return this[~fielda] >= this[~fieldb] }') + self.assertEqual('function() { return this["fielda"] >= this["fieldb"] }', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + def invalid_where(): + list(IntPair.objects.where(fielda__gte=3)) + + self.assertRaises(TypeError, invalid_where) + + class QTest(unittest.TestCase): def setUp(self): From 8bdb42827c3433d69757325113b7f14fe22509d9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 11:33:41 +0100 Subject: [PATCH 5/9] Updated AUTHORS Thanks to all those that have contributed to MongoEngine --- AUTHORS | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/AUTHORS b/AUTHORS index fbf78cf6..ed022c2e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,5 @@ +The PRIMARY AUTHORS are (and/or have been): + Harry Marr Matt Dennewitz Deepak Thukral @@ -6,3 +8,61 @@ Steve Challis Ross Lawley Wilson Júnior Dan Crosta https://github.com/dcrosta + +CONTRIBUTORS + +Dervived from the git logs, inevitably incomplete but all of whom and others +have submitted patches, reported bugs and generally helped make MongoEngine +that much better: + + * Harry Marr + * Ross Lawley + * blackbrrr + * Florian Schlachter + * Vincent Driessen + * Steve Challis + * flosch + * Deepak Thukral + * Colin Howe + * Wilson Júnior + * Alistair Roche + * Dan Crosta + * Viktor Kerkez + * Stephan Jaekel + * Rached Ben Mustapha + * Greg Turner + * Daniel Hasselrot + * Mircea Pasoi + * Matt Chisholm + * James Punteney + * TimothéePeignier + * Stuart Rackham + * Serge Matveenko + * Matt Dennewitz + * Don Spaulding + * Ales Zoulek + * sshwsfc + * sib + * Samuel Clay + * Nick Vlku + * martin + * Flavio Amieiro + * Анхбаяр Лхагвадорж + * Zak Johnson + * Victor Farazdagi + * vandersonmota + * Theo Julienne + * sp + * Slavi Pantaleev + * Richard Henry + * Nicolas Perriault + * Nick Vlku Jr + * Michael Henson + * Leo Honkanen + * kuno + * Josh Ourisman + * Jaime + * Igor Ivanov + * Gregg Lind + * Gareth Lloyd + * Albert Choi From 5f058434035121335c375d0e4d8b4d7447327144 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 16 Aug 2011 08:20:06 -0400 Subject: [PATCH 6/9] prefer to use map-reduce to db.eval where possible --- mongoengine/queryset.py | 72 ++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 303fcc1b..a2716089 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1416,16 +1416,26 @@ class QuerySet(object): :param field: the field to sum over; use dot-notation to refer to embedded document fields """ - sum_func = """ - function(sumField) { - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - total += (doc[sumField] || 0.0); - }); - return total; + map_func = pymongo.code.Code(""" + function() { + emit(1, this[field] || 0); } - """ - return self.exec_js(sum_func, field) + """, scope={'field': field}) + + reduce_func = pymongo.code.Code(""" + function(key, values) { + var sum = 0; + for (var i in values) { + sum += values[i]; + } + return sum; + } + """) + + for result in self.map_reduce(map_func, reduce_func, output='inline'): + return result.value + else: + return 0 def average(self, field): """Average over the values of the specified field. @@ -1433,22 +1443,38 @@ class QuerySet(object): :param field: the field to average over; use dot-notation to refer to embedded document fields """ - average_func = """ - function(averageField) { - var total = 0.0; - var num = 0; - db[collection].find(query).forEach(function(doc) { - if (doc[averageField] !== undefined) { - total += doc[averageField]; - num += 1; - } - }); - return total / num; + map_func = pymongo.code.Code(""" + function() { + if (this.hasOwnProperty(field)) + emit(1, {t: this[field] || 0, c: 1}); } - """ - return self.exec_js(average_func, field) + """, scope={'field': field}) - def item_frequencies(self, field, normalize=False, map_reduce=False): + reduce_func = pymongo.code.Code(""" + function(key, values) { + var out = {t: 0, c: 0}; + for (var i in values) { + var value = values[i]; + out.t += value.t; + out.c += value.c; + } + return out; + } + """) + + finalize_func = pymongo.code.Code(""" + function(key, value) { + return value.t / value.c; + } + """) + + for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'): + return result.value + else: + return 0 + + + def item_frequencies(self, field, normalize=False, map_reduce=True): """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. From fd2e40d735126e788c688a296f8bf7bf45e34dc6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 15:24:37 +0100 Subject: [PATCH 7/9] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 87247d57..551d3b2d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments - Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters - Added update() convenience method to a document From 2a8543b3b730974376324a27e5e46c33905a86fa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 15:26:11 +0100 Subject: [PATCH 8/9] Updated changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 551d3b2d..787b9c91 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,7 +30,7 @@ Changes in dev - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests -- Added optional map_reduce method item_frequencies +- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments) - Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. - Added searching multiple levels deep in ``DictField`` From 3c8cbcfee757334b4e58606dd2e64d2082fa1e7e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 16:50:50 +0100 Subject: [PATCH 9/9] Added tests for showing how to set embedded document indexes refs #257 --- tests/document.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/document.py b/tests/document.py index 90a0bc25..6984ef3e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -690,6 +690,57 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_embedded_document_index(self): + """Tests settings an index on an embedded document + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + + meta = { + 'indexes': [ + '-date.year' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) + BlogPost.drop_collection() + + def test_list_embedded_document_index(self): + """Ensure list embedded documents can be indexed + """ + class Tag(EmbeddedDocument): + name = StringField(db_field='tag') + + class BlogPost(Document): + title = StringField() + tags = ListField(EmbeddedDocumentField(Tag)) + + meta = { + 'indexes': [ + 'tags.name' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # we don't use _types in with list fields by default + self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) + + post1 = BlogPost(title="Embedded Indexes tests in place", + tags=[Tag(name="about"), Tag(name="time")] + ) + post1.save() + BlogPost.drop_collection() + + def test_geo_indexes_recursion(self): class User(Document):