diff --git a/AUTHORS b/AUTHORS index b13af2b0..ed022c2e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,5 @@ +The PRIMARY AUTHORS are (and/or have been): + Harry Marr Matt Dennewitz Deepak Thukral @@ -5,3 +7,62 @@ Florian Schlachter Steve Challis Ross Lawley Wilson Júnior +Dan Crosta https://github.com/dcrosta + +CONTRIBUTORS + +Dervived from the git logs, inevitably incomplete but all of whom and others +have submitted patches, reported bugs and generally helped make MongoEngine +that much better: + + * Harry Marr + * Ross Lawley + * blackbrrr + * Florian Schlachter + * Vincent Driessen + * Steve Challis + * flosch + * Deepak Thukral + * Colin Howe + * Wilson Júnior + * Alistair Roche + * Dan Crosta + * Viktor Kerkez + * Stephan Jaekel + * Rached Ben Mustapha + * Greg Turner + * Daniel Hasselrot + * Mircea Pasoi + * Matt Chisholm + * James Punteney + * TimothéePeignier + * Stuart Rackham + * Serge Matveenko + * Matt Dennewitz + * Don Spaulding + * Ales Zoulek + * sshwsfc + * sib + * Samuel Clay + * Nick Vlku + * martin + * Flavio Amieiro + * Анхбаяр Лхагвадорж + * Zak Johnson + * Victor Farazdagi + * vandersonmota + * Theo Julienne + * sp + * Slavi Pantaleev + * Richard Henry + * Nicolas Perriault + * Nick Vlku Jr + * Michael Henson + * Leo Honkanen + * kuno + * Josh Ourisman + * Jaime + * Igor Ivanov + * Gregg Lind + * Gareth Lloyd + * Albert Choi diff --git a/docs/changelog.rst b/docs/changelog.rst index f3a4b944..787b9c91 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments +- Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters - Added update() convenience method to a document - Added cascading saves - so changes to Referenced documents are saved on .save() @@ -28,7 +30,7 @@ Changes in dev - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests -- Added optional map_reduce method item_frequencies +- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments) - Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. - Added searching multiple levels deep in ``DictField`` diff --git a/mongoengine/base.py b/mongoengine/base.py index e224367c..6be5c3de 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -812,7 +812,8 @@ class BaseDocument(object): field_cls = field.document_type if field_cls in inspected_classes: continue - geo_indices += field_cls._geo_indices(inspected_classes) + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected_classes) elif field._geo_index: geo_indices.append(field) return geo_indices diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b2f1e2a2..619b8c60 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -620,7 +620,7 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - note: Any documents used as a generic reference must be registered in the + ..note :: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. .. versionadded:: 0.3 @@ -925,7 +925,7 @@ class SequenceField(IntField): return value def __set__(self, instance, value): - + if value is None and instance._initialised: value = self.generate_new_value() diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11c7a804..a2716089 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1397,22 +1397,45 @@ class QuerySet(object): db = _get_db() return db.eval(code, *fields) + def where(self, where_clause): + """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript + expression). Performs automatic field name substitution like + :meth:`mongoengine.queryset.Queryset.exec_js`. + + .. note:: When using this mode of query, the database will call your + function, or evaluate your predicate clause, for each object + in the collection. + """ + where_clause = self._sub_js_fields(where_clause) + self._where_clause = where_clause + return self + def sum(self, field): """Sum over the values of the specified field. :param field: the field to sum over; use dot-notation to refer to embedded document fields """ - sum_func = """ - function(sumField) { - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - total += (doc[sumField] || 0.0); - }); - return total; + map_func = pymongo.code.Code(""" + function() { + emit(1, this[field] || 0); } - """ - return self.exec_js(sum_func, field) + """, scope={'field': field}) + + reduce_func = pymongo.code.Code(""" + function(key, values) { + var sum = 0; + for (var i in values) { + sum += values[i]; + } + return sum; + } + """) + + for result in self.map_reduce(map_func, reduce_func, output='inline'): + return result.value + else: + return 0 def average(self, field): """Average over the values of the specified field. @@ -1420,22 +1443,38 @@ class QuerySet(object): :param field: the field to average over; use dot-notation to refer to embedded document fields """ - average_func = """ - function(averageField) { - var total = 0.0; - var num = 0; - db[collection].find(query).forEach(function(doc) { - if (doc[averageField] !== undefined) { - total += doc[averageField]; - num += 1; - } - }); - return total / num; + map_func = pymongo.code.Code(""" + function() { + if (this.hasOwnProperty(field)) + emit(1, {t: this[field] || 0, c: 1}); } - """ - return self.exec_js(average_func, field) + """, scope={'field': field}) - def item_frequencies(self, field, normalize=False, map_reduce=False): + reduce_func = pymongo.code.Code(""" + function(key, values) { + var out = {t: 0, c: 0}; + for (var i in values) { + var value = values[i]; + out.t += value.t; + out.c += value.c; + } + return out; + } + """) + + finalize_func = pymongo.code.Code(""" + function(key, value) { + return value.t / value.c; + } + """) + + for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'): + return result.value + else: + return 0 + + + def item_frequencies(self, field, normalize=False, map_reduce=True): """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. diff --git a/tests/document.py b/tests/document.py index 90a0bc25..6984ef3e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -690,6 +690,57 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_embedded_document_index(self): + """Tests settings an index on an embedded document + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + + meta = { + 'indexes': [ + '-date.year' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) + BlogPost.drop_collection() + + def test_list_embedded_document_index(self): + """Ensure list embedded documents can be indexed + """ + class Tag(EmbeddedDocument): + name = StringField(db_field='tag') + + class BlogPost(Document): + title = StringField() + tags = ListField(EmbeddedDocumentField(Tag)) + + meta = { + 'indexes': [ + 'tags.name' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # we don't use _types in with list fields by default + self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) + + post1 = BlogPost(title="Embedded Indexes tests in place", + tags=[Tag(name="about"), Tag(name="time")] + ) + post1.save() + BlogPost.drop_collection() + + def test_geo_indexes_recursion(self): class User(Document): diff --git a/tests/queryset.py b/tests/queryset.py index a21bae69..6ae1c10f 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2502,6 +2502,47 @@ class QuerySetTest(unittest.TestCase): for key, value in info.iteritems()] self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + def test_where(self): + """Ensure that where clauses work. + """ + + class IntPair(Document): + fielda = IntField() + fieldb = IntField() + + IntPair.objects._collection.remove() + + a = IntPair(fielda=1, fieldb=1) + b = IntPair(fielda=1, fieldb=2) + c = IntPair(fielda=2, fieldb=1) + a.save() + b.save() + c.save() + + query = IntPair.objects.where('this[~fielda] >= this[~fieldb]') + self.assertEqual('this["fielda"] >= this["fieldb"]', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + query = IntPair.objects.where('this[~fielda] == this[~fieldb]') + results = list(query) + self.assertEqual(1, len(results)) + self.assertTrue(a in results) + + query = IntPair.objects.where('function() { return this[~fielda] >= this[~fieldb] }') + self.assertEqual('function() { return this["fielda"] >= this["fieldb"] }', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + def invalid_where(): + list(IntPair.objects.where(fielda__gte=3)) + + self.assertRaises(TypeError, invalid_where) + class QTest(unittest.TestCase):