From 19314e7e06c3b4b393b0017868182c689c75f9f9 Mon Sep 17 00:00:00 2001 From: Kirill Pavlov Date: Mon, 3 Mar 2014 13:09:26 +0800 Subject: [PATCH 01/44] fix docstring for DictField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 82642cda..b0e51a24 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -760,7 +760,7 @@ class DictField(ComplexBaseField): similar to an embedded document, but the structure is not defined. .. note:: - Required means it cannot be empty - as the default for ListFields is [] + Required means it cannot be empty - as the default for DictFields is {} .. versionadded:: 0.3 .. versionchanged:: 0.5 - Can now handle complex / varying types of data From d27a1103fac41fd3eb4231a89fc35044641b317a Mon Sep 17 00:00:00 2001 From: Damien Churchill Date: Wed, 12 Mar 2014 17:19:49 +0000 Subject: [PATCH 02/44] workaround a dateutil bug In the latest released version of dateutil, there's a bug whereby a TypeError can be raised whilst parsing a date. This is because it calls a method which it expects to return 2 arguments, however it can return 1 depending upon the input, which results in a TypeError: ArgType not iterable exception. Since this is equivalent to a failed parse anyway, we can treat it the same as a ValueError. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 82642cda..85a1a743 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -391,7 +391,7 @@ class DateTimeField(BaseField): if dateutil: try: return dateutil.parser.parse(value) - except ValueError: + except (TypeError, ValueError): return None # split usecs, because they are not recognized by strptime. From c1f88a4e14f319b8a849dad44849d6f6884c8e75 Mon Sep 17 00:00:00 2001 From: Falcon Dai Date: Mon, 17 Mar 2014 22:29:53 -0500 Subject: [PATCH 03/44] minor change to geo-related docs --- docs/guide/defining-documents.rst | 2 ++ mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 5d8b628a..07bce3bb 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -531,6 +531,8 @@ field name to the index definition. Sometimes its more efficient to index parts of Embedded / dictionary fields, in this case use 'dot' notation to identify the value to index eg: `rank.title` +.. _geospatial-indexes: + Geospatial indexes ------------------ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 82642cda..48db6dfd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1613,7 +1613,12 @@ class UUIDField(BaseField): class GeoPointField(BaseField): - """A list storing a latitude and longitude. + """A list storing a longitude and latitude coordinate. + + .. note:: this represents a generic point in a 2D plane and a legacy way of + representing a geo point. It admits 2d indexes but not "2dsphere" indexes + in MongoDB > 2.4 which are more natural for modeling geospatial points. + See :ref:`geospatial-indexes` .. versionadded:: 0.4 """ @@ -1635,7 +1640,7 @@ class GeoPointField(BaseField): class PointField(GeoJsonBaseField): - """A geo json field storing a latitude and longitude. + """A GeoJSON field storing a longitude and latitude coordinate. The data is represented as: @@ -1654,7 +1659,7 @@ class PointField(GeoJsonBaseField): class LineStringField(GeoJsonBaseField): - """A geo json field storing a line of latitude and longitude coordinates. + """A GeoJSON field storing a line of longitude and latitude coordinates. The data is represented as: @@ -1672,7 +1677,7 @@ class LineStringField(GeoJsonBaseField): class PolygonField(GeoJsonBaseField): - """A geo json field storing a polygon of latitude and longitude coordinates. + """A GeoJSON field storing a polygon of longitude and latitude coordinates. The data is represented as: From 4d7b988018c2d150401fb96db36da1ebdd6a2c4d Mon Sep 17 00:00:00 2001 From: poly Date: Tue, 1 Apr 2014 19:52:21 +0800 Subject: [PATCH 04/44] Fixed uncorrectly split a query key, when it ends with "_" --- mongoengine/queryset/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index e31a8b7d..27e41ad2 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -38,7 +38,7 @@ def query(_doc_cls=None, _field_operation=False, **query): mongo_query.update(value) continue - parts = key.split('__') + parts = key.rsplit('__') indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is From 803caddbd40ab2ea6cbee8e6b65a5921277fcafc Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Wed, 9 Apr 2014 14:25:53 +0400 Subject: [PATCH 05/44] Raise NotUniqueError in Document.update() on pymongo.errors.DuplicateKeyError --- mongoengine/queryset/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..c34c1479 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -443,6 +443,8 @@ class BaseQuerySet(object): return result elif result: return result['n'] + except pymongo.errors.DuplicateKeyError, err: + raise NotUniqueError(u'Update failed (%s)' % unicode(err)) except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' From b45a601ad2a208b209af5e8f45893aaf16ebd59f Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Tue, 15 Apr 2014 19:32:42 +0400 Subject: [PATCH 06/44] Test raising NotUniqueError by Document.update() --- tests/document/instance.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..c57102da 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -15,7 +15,7 @@ from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest, from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, - InvalidQueryError) + InvalidQueryError, NotUniqueError) from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document @@ -990,6 +990,16 @@ class InstanceTest(unittest.TestCase): self.assertRaises(InvalidQueryError, update_no_op_raises) + def test_update_unique_field(self): + class Doc(Document): + name = StringField(unique=True) + + doc1 = Doc(name="first").save() + doc2 = Doc(name="second").save() + + self.assertRaises(NotUniqueError, lambda: + doc2.update(set__name=doc1.name)) + def test_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields From 12809ebc741dc04677a4e3b0f7c3ed5c23563d7f Mon Sep 17 00:00:00 2001 From: Jatin Chopra Date: Tue, 6 May 2014 00:25:55 -0700 Subject: [PATCH 07/44] Updated Jatin's name and github name --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index d6994d50..5fa0cb3d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -171,7 +171,7 @@ that much better: * Michael Bartnett (https://github.com/michaelbartnett) * Alon Horev (https://github.com/alonho) * Kelvin Hammond (https://github.com/kelvinhammond) - * Jatin- (https://github.com/jatin-) + * Jatin Chopra (https://github.com/jatin) * Paul Uithol (https://github.com/PaulUithol) * Thom Knowles (https://github.com/fleat) * Paul (https://github.com/squamous) From abcacc82f359d37922dc6c19c6b860b4c633e8e0 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 21 May 2014 22:21:46 -0700 Subject: [PATCH 08/44] dont use a system collection --- tests/document/instance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 07db85a0..6c4da162 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2411,7 +2411,7 @@ class InstanceTest(unittest.TestCase): for parameter_name, parameter in self.parameters.iteritems(): parameter.expand() - class System(Document): + class NodesSystem(Document): name = StringField(required=True) nodes = MapField(ReferenceField(Node, dbref=False)) @@ -2419,18 +2419,18 @@ class InstanceTest(unittest.TestCase): for node_name, node in self.nodes.iteritems(): node.expand() node.save(*args, **kwargs) - super(System, self).save(*args, **kwargs) + super(NodesSystem, self).save(*args, **kwargs) - System.drop_collection() + NodesSystem.drop_collection() Node.drop_collection() - system = System(name="system") + system = NodesSystem(name="system") system.nodes["node"] = Node() system.save() system.nodes["node"].parameters["param"] = Parameter() system.save() - system = System.objects.first() + system = NodesSystem.objects.first() self.assertEqual("UNDEFINED", system.nodes["node"].parameters["param"].macros["test"].value) def test_embedded_document_equality(self): From 3faf3c84be55f709d828ea7837db988c56b0239e Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Tue, 27 May 2014 16:33:38 -0300 Subject: [PATCH 09/44] Avoid to open all documents from cursors in an if stmt Using a cursos in an if statement: cursor = Collection.objects if cursor: (...) Will open all documents, because there are not an __nonzero__ method. This change check only one document (if present) and returns True or False. --- mongoengine/queryset/base.py | 9 +++++++++ tests/queryset/queryset.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..823bc164 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,6 +154,15 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError + def __nonzero__(self): + """ Avoid to open all records in an if stmt """ + + for value in self: + self.rewind() + return True + + return False + # Core functions def all(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..f274e0ee 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3814,6 +3814,29 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Example.objects(size=instance_size).count(), 1) self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1) + def test_cursor_in_an_if_stmt(self): + + class Test(Document): + test_field = StringField() + + Test.drop_collection() + queryset = Test.objects + + if queryset: + raise AssertionError('Empty cursor returns True') + + test = Test() + test.test_field = 'test' + test.save() + + queryset = Test.objects + if not test: + raise AssertionError('There is data, but cursor returned False') + + queryset.next() + if queryset: + raise AssertionError('There is no data left in cursor') + if __name__ == '__main__': unittest.main() From 9ba657797ed64d50a876c95bc2a243bed3037e19 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 08:33:22 -0300 Subject: [PATCH 10/44] Authors updated according guideline --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index d6994d50..5d6a45fa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -189,3 +189,4 @@ that much better: * Tom (https://github.com/tomprimozic) * j0hnsmith (https://github.com/j0hnsmith) * Damien Churchill (https://github.com/damoxc) + * Jonathan Simon Prates (https://github.com/jonathansp) \ No newline at end of file From 47f0de9836a7e340047c6639dee9ade29b9755cb Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 08:36:57 -0300 Subject: [PATCH 11/44] Py3 fix --- mongoengine/queryset/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 823bc164..022b7c1f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -161,7 +161,12 @@ class BaseQuerySet(object): self.rewind() return True - return False + return False + + def __bool__(self): + """ Same behaviour in Py3 """ + + return self.__nonzero__(): # Core functions From edcdfeb0573f253227a196b3080344e849f48109 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 09:03:12 -0300 Subject: [PATCH 12/44] Fix syntax error --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 022b7c1f..0dfff7cc 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -166,7 +166,7 @@ class BaseQuerySet(object): def __bool__(self): """ Same behaviour in Py3 """ - return self.__nonzero__(): + return self.__nonzero__() # Core functions From dfdecef8e72fdd0379cdec4452f3175f0826b892 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 09:40:22 -0300 Subject: [PATCH 13/44] Fix py2 and py3 --- mongoengine/queryset/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0dfff7cc..09fb5bf6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,7 +154,7 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def __nonzero__(self): + def _bool(self): """ Avoid to open all records in an if stmt """ for value in self: @@ -163,10 +163,15 @@ class BaseQuerySet(object): return False + def __nonzero__(self): + """ Same behaviour in Py2 """ + + return self._bool() + def __bool__(self): """ Same behaviour in Py3 """ - return self.__nonzero__() + return self._bool() # Core functions From ee0c7fd8bfa08ec885f379b2aa6dd72f48d8d7c5 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 13:21:00 -0300 Subject: [PATCH 14/44] Change for loop to self.first() --- mongoengine/queryset/base.py | 6 +----- tests/queryset/queryset.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 09fb5bf6..099831fe 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -157,11 +157,7 @@ class BaseQuerySet(object): def _bool(self): """ Avoid to open all records in an if stmt """ - for value in self: - self.rewind() - return True - - return False + return False if self.first() is None else True def __nonzero__(self): """ Same behaviour in Py2 """ diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f274e0ee..f6adaf39 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3834,8 +3834,8 @@ class QuerySetTest(unittest.TestCase): raise AssertionError('There is data, but cursor returned False') queryset.next() - if queryset: - raise AssertionError('There is no data left in cursor') + if not queryset: + raise AssertionError('There is data, cursor must return True') if __name__ == '__main__': From 30964f65e465f7e29960cd49caf29d5c9d4ac756 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:06:15 -0300 Subject: [PATCH 15/44] Remove orderby in if stmt --- mongoengine/queryset/base.py | 8 +++++- tests/queryset/queryset.py | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 099831fe..fe1285c6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -157,7 +157,13 @@ class BaseQuerySet(object): def _bool(self): """ Avoid to open all records in an if stmt """ - return False if self.first() is None else True + queryset = self.clone() + queryset._ordering = [] + try: + queryset[0] + return True + except IndexError: + return False def __nonzero__(self): """ Same behaviour in Py2 """ diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f6adaf39..65f7255b 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3837,6 +3837,59 @@ class QuerySetTest(unittest.TestCase): if not queryset: raise AssertionError('There is data, cursor must return True') + def test_bool_performance(self): + + class Person(Document): + name = StringField() + + Person.drop_collection() + for i in xrange(100): + Person(name="No: %s" % i).save() + + with query_counter() as q: + if Person.objects: + pass + + self.assertEqual(q, 1) + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertEqual(op['nreturned'], 1) + + + def test_bool_with_ordering(self): + + class Person(Document): + name = StringField() + + Person.drop_collection() + Person(name="Test").save() + + qs = Person.objects.order_by('name') + + with query_counter() as q: + + if qs: + pass + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertFalse('$orderby' in op['query'], + 'BaseQuerySet cannot use orderby in if stmt') + + with query_counter() as p: + + for x in qs: + pass + + op = p.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + + self.assertTrue('$orderby' in op['query'], + 'BaseQuerySet cannot remove orderby in for loop') + if __name__ == '__main__': unittest.main() From 39735594bd935f3003d85009c3dc14a8d3d71f23 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:15:48 -0300 Subject: [PATCH 16/44] Removed blank line --- tests/queryset/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 65f7255b..c5fea003 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3886,7 +3886,6 @@ class QuerySetTest(unittest.TestCase): op = p.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertTrue('$orderby' in op['query'], 'BaseQuerySet cannot remove orderby in for loop') From c87801f0a911bd02f4b4ae910751c8b04717fd02 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 28 May 2014 17:26:28 -0300 Subject: [PATCH 17/44] Using first() from cloned queryset --- mongoengine/queryset/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index fe1285c6..85d2dc3f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -159,11 +159,7 @@ class BaseQuerySet(object): queryset = self.clone() queryset._ordering = [] - try: - queryset[0] - return True - except IndexError: - return False + return False if queryset.first() is None else True def __nonzero__(self): """ Same behaviour in Py2 """ From c744104a18829f7f34d491a580914b89ecf3c620 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 10:53:20 -0300 Subject: [PATCH 18/44] Added test with meta --- tests/queryset/queryset.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c5fea003..bad0d1e5 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3889,6 +3889,33 @@ class QuerySetTest(unittest.TestCase): self.assertTrue('$orderby' in op['query'], 'BaseQuerySet cannot remove orderby in for loop') + def test_bool_with_ordering_from_meta_dict(self): + + class Person(Document): + name = StringField() + meta = { + 'ordering': ['name'] + } + + Person.drop_collection() + + Person(name="B").save() + Person(name="C").save() + Person(name="A").save() + + with query_counter() as q: + + if Person.objects: + pass + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query'], + 'BaseQuerySet cannot remove orderby from meta in boolen test') + + self.assertEqual(Person.objects.first().name, 'A') + if __name__ == '__main__': unittest.main() From 819ff2a90286030740e2d6d886c2f14e29d0ccc2 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 14:36:30 -0300 Subject: [PATCH 19/44] Renamed to has_data() --- AUTHORS | 3 ++- mongoengine/queryset/base.py | 12 ++++++------ tests/queryset/queryset.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5d6a45fa..7b466d6b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -189,4 +189,5 @@ that much better: * Tom (https://github.com/tomprimozic) * j0hnsmith (https://github.com/j0hnsmith) * Damien Churchill (https://github.com/damoxc) - * Jonathan Simon Prates (https://github.com/jonathansp) \ No newline at end of file + * Jonathan Simon Prates (https://github.com/jonathansp) + * Thiago Papageorgiou (https://github.com/tmpapageorgiou) \ No newline at end of file diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 85d2dc3f..ef8cd2a7 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,22 +154,22 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def _bool(self): - """ Avoid to open all records in an if stmt """ + def has_data(self): + """ Retrieves whether cursor has any data. """ queryset = self.clone() queryset._ordering = [] return False if queryset.first() is None else True def __nonzero__(self): - """ Same behaviour in Py2 """ + """ Avoid to open all records in an if stmt in Py2. """ - return self._bool() + return self.has_data() def __bool__(self): - """ Same behaviour in Py3 """ + """ Avoid to open all records in an if stmt in Py3. """ - return self._bool() + return self.has_data() # Core functions diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bad0d1e5..fe932367 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3831,11 +3831,15 @@ class QuerySetTest(unittest.TestCase): queryset = Test.objects if not test: - raise AssertionError('There is data, but cursor returned False') + raise AssertionError('Cursor has data and returned False') queryset.next() if not queryset: - raise AssertionError('There is data, cursor must return True') + raise AssertionError('Cursor has data and it must returns False,' + ' even in the last item.') + + self.assertTrue(queryset.has_data(), 'Cursor has data and ' + 'returned False') def test_bool_performance(self): From 85187239b629772ba276aaee54eb678c64ad8207 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 29 May 2014 15:21:24 -0300 Subject: [PATCH 20/44] Fix tests msg --- tests/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index fe932367..f68468ff 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3835,7 +3835,7 @@ class QuerySetTest(unittest.TestCase): queryset.next() if not queryset: - raise AssertionError('Cursor has data and it must returns False,' + raise AssertionError('Cursor has data and it must returns True,' ' even in the last item.') self.assertTrue(queryset.has_data(), 'Cursor has data and ' From 1eacc6fbff0bbe14a18d62032fe0b616194c8d26 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Fri, 30 May 2014 15:08:03 -0700 Subject: [PATCH 21/44] clear ordering via empty order_by --- mongoengine/queryset/base.py | 7 ++-- tests/queryset/queryset.py | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..4f451667 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -50,7 +50,7 @@ class BaseQuerySet(object): self._initial_query = {} self._where_clause = None self._loaded_fields = QueryFieldList() - self._ordering = [] + self._ordering = None self._snapshot = False self._timeout = True self._class_check = True @@ -1189,8 +1189,9 @@ class BaseQuerySet(object): if self._ordering: # Apply query ordering self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - # Otherwise, apply the ordering from the document model + elif self._ordering is None and self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model, unless + # it's been explicitly cleared via order_by with no arguments order = self._get_order_by(self._document._meta['ordering']) self._cursor_obj.sort(order) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..d706eda8 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1040,6 +1040,76 @@ class QuerySetTest(unittest.TestCase): expected = [blog_post_1, blog_post_2, blog_post_3] self.assertSequence(qs, expected) + def test_clear_ordering(self): + """ Make sure one can clear the query set ordering by applying a + consecutive order_by() + """ + + class Person(Document): + name = StringField() + + Person.drop_collection() + Person(name="A").save() + Person(name="B").save() + + qs = Person.objects.order_by('-name') + + # Make sure we can clear a previously specified ordering + with query_counter() as q: + lst = list(qs.order_by()) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' not in op['query']) + self.assertEqual(lst[0].name, 'A') + + # Make sure previously specified ordering is preserved during + # consecutive calls to the same query set + with query_counter() as q: + lst = list(qs) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query']) + self.assertEqual(lst[0].name, 'B') + + def test_clear_default_ordering(self): + + class Person(Document): + name = StringField() + meta = { + 'ordering': ['-name'] + } + + Person.drop_collection() + Person(name="A").save() + Person(name="B").save() + + qs = Person.objects + + # Make sure clearing default ordering works + with query_counter() as q: + lst = list(qs.order_by()) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' not in op['query']) + self.assertEqual(lst[0].name, 'A') + + # Make sure default ordering is preserved during consecutive calls + # to the same query set + with query_counter() as q: + lst = list(qs) + + op = q.db.system.profile.find({"ns": + {"$ne": "%s.system.indexes" % q.db.name}})[0] + + self.assertTrue('$orderby' in op['query']) + self.assertEqual(lst[0].name, 'B') + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ From 7bb2fe128a4b8c8b08f4d800f15229e2814bfac8 Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Thu, 12 Jun 2014 11:08:41 -0300 Subject: [PATCH 22/44] Added PR #657 --- mongoengine/queryset/base.py | 11 +++++------ tests/queryset/queryset.py | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 94a6e4b5..cb48f6ca 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -154,22 +154,21 @@ class BaseQuerySet(object): def __iter__(self): raise NotImplementedError - def has_data(self): + def _has_data(self): """ Retrieves whether cursor has any data. """ - queryset = self.clone() - queryset._ordering = [] + queryset = self.order_by() return False if queryset.first() is None else True def __nonzero__(self): """ Avoid to open all records in an if stmt in Py2. """ - return self.has_data() + return self._has_data() def __bool__(self): """ Avoid to open all records in an if stmt in Py3. """ - return self.has_data() + return self._has_data() # Core functions @@ -1410,7 +1409,7 @@ class BaseQuerySet(object): pass key_list.append((key, direction)) - if self._cursor_obj: + if self._cursor_obj and key_list: self._cursor_obj.sort(key_list) return key_list diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 38499df4..a2438e21 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3908,9 +3908,6 @@ class QuerySetTest(unittest.TestCase): raise AssertionError('Cursor has data and it must returns True,' ' even in the last item.') - self.assertTrue(queryset.has_data(), 'Cursor has data and ' - 'returned False') - def test_bool_performance(self): class Person(Document): @@ -3985,10 +3982,12 @@ class QuerySetTest(unittest.TestCase): op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] - self.assertTrue('$orderby' in op['query'], - 'BaseQuerySet cannot remove orderby from meta in boolen test') + self.assertFalse('$orderby' in op['query'], + 'BaseQuerySet must remove orderby from meta in boolen test') self.assertEqual(Person.objects.first().name, 'A') + self.assertTrue(Person.objects._has_data(), + 'Cursor has data and returned False') if __name__ == '__main__': From 03559a3cc4b0ee3a5509c8eb2f1f54bf342284b4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Jun 2014 19:20:15 +0300 Subject: [PATCH 23/44] Added Python 3.4 to the build process. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5739909b..7ba22694 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" env: - PYMONGO=dev DJANGO=1.6 - PYMONGO=dev DJANGO=1.5.5 From bb461b009f6aa0c64c6fb2cf845a4714ee324908 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Jun 2014 19:39:27 +0300 Subject: [PATCH 24/44] Travis build improvements. The latest patch version of each Django minor version is used. The build now installs existing pymongo versions. The build now actually tests against the specified Django version. Replaced PIL with Pillow. Added PyPy and Python 3.4 to the build. Rebase Log: Installing Pillow instead of PIL for testing since it's recommended and it supports PyPy. Excluding Django versions that do not work with Python 3. Improved formatting of .travis.yml. Specifying Pillow 2.0.0 and above since it's the first version that is supported in Python 3. PIL should not be installed alongside Pillow. Also, I installed some libraries that both PIL and Pillow depend on. It seems I have to be explicit on all envvars in order to exclude Django 1.4 from the build matrix. The build is now installing pymongo versions that actually exist. openjpeg has a different name on Ubuntu 12.04. Restoring libz hack. Also installing all Pillow requirements just in case. Fixed the build matrix. Acting according to @BanzaiMan's advice in travis-ci/travis-ci/#1492. --- .travis.yml | 58 +++++++++++++++++++++++++++++++++++++++++------------ setup.py | 4 ++-- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..d00dde3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,54 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" + - "pypy" env: - - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 - - PYMONGO=2.5 DJANGO=1.6 - - PYMONGO=2.5 DJANGO=1.5.5 - - PYMONGO=2.5 DJANGO=1.4.10 - - PYMONGO=3.2 DJANGO=1.6 - - PYMONGO=3.2 DJANGO=1.5.5 - - PYMONGO=3.3 DJANGO=1.6 - - PYMONGO=3.3 DJANGO=1.5.5 + - PYMONGO=dev DJANGO=1.6.5 + - PYMONGO=dev DJANGO=1.5.8 + - PYMONGO=dev DJANGO=1.4.13 + - PYMONGO=2.5.2 DJANGO=1.6.5 + - PYMONGO=2.5.2 DJANGO=1.5.8 + - PYMONGO=2.5.2 DJANGO=1.4.13 + - PYMONGO=2.6.3 DJANGO=1.6.5 + - PYMONGO=2.6.3 DJANGO=1.5.8 + - PYMONGO=2.6.3 DJANGO=1.4.13 + - PYMONGO=2.7.1 DJANGO=1.6.5 + - PYMONGO=2.7.1 DJANGO=1.5.8 + - PYMONGO=2.7.1 DJANGO=1.4.13 + +matrix: + exclude: + - python: "3.2" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.2" + env: PYMONGO=2.7.1 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.3" + env: PYMONGO=2.7.1 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=dev DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.5.2 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.6.3 DJANGO=1.4.13 + - python: "3.4" + env: PYMONGO=2.7.1 DJANGO=1.4.13 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi + - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk + - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/ - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi + - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi + - pip install Django==$DJANGO - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b - python setup.py install script: diff --git a/setup.py b/setup.py index 85707d00..40c27ebe 100644 --- a/setup.py +++ b/setup.py @@ -51,12 +51,12 @@ CLASSIFIERS = [ extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'Pillow>=2.0.0', 'django>=1.5.1'] if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'] = find_packages() extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'Pillow>=2.0.0', 'jinja2>=2.6', 'python-dateutil'] setup(name='mongoengine', version=VERSION, From 8e852bce02b14a0149c0e189642631edc2de31fe Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 10:58:00 +0300 Subject: [PATCH 25/44] Pillow provides a more descriptive error message, therefor the build failure. --- tests/fields/file_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 902b1512..7ae53e8a 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -279,7 +279,7 @@ class FileTest(unittest.TestCase): t.image.put(f) self.fail("Should have raised an invalidation error") except ValidationError, e: - self.assertEqual("%s" % e, "Invalid image: cannot identify image file") + self.assertEqual("%s" % e, "Invalid image: cannot identify image file %s" % f) t = TestImage() t.image.put(open(TEST_IMAGE_PATH, 'rb')) From adbbc656d41dbc7ecd33ae933c164efe3fd89316 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:12:40 +0300 Subject: [PATCH 26/44] Removing zlib hack since only PIL needs it. The build should pass without it. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d00dde3a..1265ba70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,6 @@ matrix: env: PYMONGO=2.7.1 DJANGO=1.4.13 install: - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk - - cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/ - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO; true; fi - pip install Django==$DJANGO From 8adf1cdd0212e18d26220d7e5997c3534c92a017 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:18:35 +0300 Subject: [PATCH 27/44] Fast finish the build if there are failures since we have a very large build matrix and each build takes a very long time. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1265ba70..4079855d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ env: - PYMONGO=2.7.1 DJANGO=1.4.13 matrix: + fast_finish: true exclude: - python: "3.2" env: PYMONGO=dev DJANGO=1.4.13 From fc3eda55c72e99f0b80bb37bf917099780c2f79e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:32:41 +0300 Subject: [PATCH 28/44] Added a note about optional dependencies to the README file. --- README.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index cc4524ae..8c3ee26e 100644 --- a/README.rst +++ b/README.rst @@ -29,9 +29,18 @@ setup.py install``. Dependencies ============ -- pymongo 2.5+ +- pymongo>=2.5 - sphinx (optional - for documentation generation) +Optional Dependencies +--------------------- +- **Django Integration:** Django>=1.4.0 for Python 2.x or PyPy and Django>=1.5.0 for Python 3.x +- **Image Fields**: Pillow>=2.0.0 or PIL (not recommended since MongoEngine is tested with Pillow) +- dateutil>=2.1.0 + +.. note + MongoEngine always runs it's test suite against the latest patch version of each dependecy. e.g.: Django 1.6.5 + Examples ======== Some simple examples of what MongoEngine code looks like:: From fe2ef4e61c4bd44e5156542728b7755f30ef8148 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 11:39:08 +0300 Subject: [PATCH 29/44] Made the benchmark script compatitable with Python 3 and ensured it runs on every build. --- .travis.yml | 1 + benchmark.py | 41 +++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4079855d..fff35ca9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ install: - python setup.py install script: - python setup.py test + - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" branches: diff --git a/benchmark.py b/benchmark.py index 16b2fd47..cb5e19d8 100644 --- a/benchmark.py +++ b/benchmark.py @@ -113,6 +113,7 @@ def main(): 4.68946313858 ---------------------------------------------------------------------------------------------------- """ + print("Benchmarking...") setup = """ from pymongo import MongoClient @@ -138,10 +139,10 @@ myNoddys = noddy.find() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - Pymongo""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ from pymongo import MongoClient @@ -161,10 +162,10 @@ myNoddys = noddy.find() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) setup = """ from pymongo import MongoClient @@ -190,10 +191,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -208,10 +209,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries without continual assign - MongoEngine""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -224,10 +225,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -240,10 +241,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 + print("-" * 100) print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -256,10 +257,10 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) stmt = """ for i in xrange(10000): @@ -272,11 +273,11 @@ myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""" + print("-" * 100) + print("""Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""") t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) + print(t.timeit(1)) if __name__ == "__main__": - main() \ No newline at end of file + main() From f44c8f120532f407b5eea6fed480183c8aa51758 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 13:11:32 +0300 Subject: [PATCH 30/44] Skipping a test that does not work on PyPy due to a PyPy bug/feature. --- tests/queryset/queryset.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7ff2965d..33f7dd91 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3586,7 +3586,13 @@ class QuerySetTest(unittest.TestCase): [x for x in people] self.assertEqual(100, len(people._result_cache)) - self.assertEqual(None, people._len) + + import platform + + if platform.python_implementation() != "PyPy": + # PyPy evaluates __len__ when iterating with list comprehensions while CPython does not. + # This may be a bug in PyPy (PyPy/#1802) but it does not affect the behavior of MongoEngine. + self.assertEqual(None, people._len) self.assertEqual(q, 1) list(people) From 1914032e35ef917ea95efc192b8280858ff78c82 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 14:20:54 +0300 Subject: [PATCH 31/44] Missed some of the print statements in the benchmarks script. --- benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark.py b/benchmark.py index cb5e19d8..5613eaf8 100644 --- a/benchmark.py +++ b/benchmark.py @@ -210,7 +210,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries without continual assign - MongoEngine""" + print("""Creating 10000 dictionaries without continual assign - MongoEngine""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -242,7 +242,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" + print("""Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) From 7f7745071af87957cdc5705b409b96ed49a1e481 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 15:47:54 +0300 Subject: [PATCH 32/44] Found more print statements that were not turned into function calls. --- benchmark.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark.py b/benchmark.py index 5613eaf8..d33e58db 100644 --- a/benchmark.py +++ b/benchmark.py @@ -140,7 +140,7 @@ myNoddys = noddy.find() """ print("-" * 100) - print """Creating 10000 dictionaries - Pymongo""" + print("""Creating 10000 dictionaries - Pymongo""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -163,7 +163,7 @@ myNoddys = noddy.find() """ print("-" * 100) - print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" + print("""Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) @@ -226,7 +226,7 @@ myNoddys = Noddy.objects() """ print("-" * 100) - print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" + print("""Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""") t = timeit.Timer(stmt=stmt, setup=setup) print(t.timeit(1)) From 29309dac9a926077962fc0778e5b2fbaf1d29cc2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 16:53:24 +0300 Subject: [PATCH 33/44] Mongo clients with the same settings should be shared since they manage a connection pool. Also, I removed old code that was supposed to support Pymongo<2.1 which we don't support anymore. --- mongoengine/connection.py | 33 ++++++++++++++++++--------------- tests/test_connection.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 7cc626f4..d3efac62 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -93,20 +93,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionError(msg) conn_settings = _connection_settings[alias].copy() - if hasattr(pymongo, 'version_tuple'): # Support for 2.1+ - conn_settings.pop('name', None) - conn_settings.pop('slaves', None) - conn_settings.pop('is_slave', None) - conn_settings.pop('username', None) - conn_settings.pop('password', None) - else: - # Get all the slave connections - if 'slaves' in conn_settings: - slaves = [] - for slave_alias in conn_settings['slaves']: - slaves.append(get_connection(slave_alias)) - conn_settings['slaves'] = slaves - conn_settings.pop('read_preference', None) + conn_settings.pop('name', None) + conn_settings.pop('slaves', None) + conn_settings.pop('is_slave', None) + conn_settings.pop('username', None) + conn_settings.pop('password', None) connection_class = MongoClient if 'replicaSet' in conn_settings: @@ -119,7 +110,19 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): connection_class = MongoReplicaSetClient try: - _connections[alias] = connection_class(**conn_settings) + connection = None + connection_settings_iterator = ((alias, settings.copy()) for alias, settings in _connection_settings.iteritems()) + for alias, connection_settings in connection_settings_iterator: + connection_settings.pop('name', None) + connection_settings.pop('slaves', None) + connection_settings.pop('is_slave', None) + connection_settings.pop('username', None) + connection_settings.pop('password', None) + if conn_settings == connection_settings and _connections.get(alias, None): + connection = _connections[alias] + break + + _connections[alias] = connection if connection else connection_class(**conn_settings) except Exception, e: raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e)) return _connections[alias] diff --git a/tests/test_connection.py b/tests/test_connection.py index 96135bc5..bf615ceb 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -34,6 +34,17 @@ class ConnectionTest(unittest.TestCase): conn = get_connection('testdb') self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) + def test_sharing_connections(self): + """Ensure that connections are shared when the connection settings are exactly the same + """ + connect('mongoenginetest', alias='testdb1') + + expected_connection = get_connection('testdb1') + + connect('mongoenginetest', alias='testdb2') + actual_connection = get_connection('testdb2') + self.assertIs(expected_connection, actual_connection) + def test_connect_uri(self): """Ensure that the connect() method works properly with uri's """ From b8d568761e0e3130ce37d5675f8b218b11b88e29 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Jun 2014 17:24:52 +0300 Subject: [PATCH 34/44] Getting rid of xrange since it's not in Python 3 and does not affect the benchmark. --- benchmark.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark.py b/benchmark.py index d33e58db..53ecf32c 100644 --- a/benchmark.py +++ b/benchmark.py @@ -15,7 +15,7 @@ def cprofile_main(): class Noddy(Document): fields = DictField() - for i in xrange(1): + for i in range(1): noddy = Noddy() for j in range(20): noddy.fields["key" + str(j)] = "value " + str(j) @@ -128,7 +128,7 @@ connection = MongoClient() db = connection.timeit_test noddy = db.noddy -for i in xrange(10000): +for i in range(10000): example = {'fields': {}} for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) @@ -151,7 +151,7 @@ connection = MongoClient() db = connection.timeit_test noddy = db.noddy -for i in xrange(10000): +for i in range(10000): example = {'fields': {}} for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) @@ -181,7 +181,7 @@ class Noddy(Document): """ stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -197,7 +197,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() fields = {} for j in range(20): @@ -215,7 +215,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -231,7 +231,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -247,7 +247,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) @@ -263,7 +263,7 @@ myNoddys = Noddy.objects() print(t.timeit(1)) stmt = """ -for i in xrange(10000): +for i in range(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) From 67a65a2aa9ecd77de372511aa54541100923d3d3 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 11:17:57 +0300 Subject: [PATCH 35/44] Installing unittest2 on Python 2.6. --- setup.py | 3 +++ tests/test_connection.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85707d00..f8f0d6da 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,9 @@ if sys.version_info[0] == 3: else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] + if sys.version_info[0] == 2 and sys.version_info[1] == 6: + extra_opts['tests_require'].append('unittest2') + setup(name='mongoengine', version=VERSION, author='Harry Marr', diff --git a/tests/test_connection.py b/tests/test_connection.py index bf615ceb..a5b1b089 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,6 +1,11 @@ import sys sys.path[0:0] = [""] -import unittest + +try: + import unittest2 as unittest +except ImportError: + import unittest + import datetime import pymongo From cae91ce0c564089b418d0fd508d80c7f923df949 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 12:31:07 +0300 Subject: [PATCH 36/44] Convert codebase to Python 3 using 2to3 before running benchmarks. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fff35ca9..6281e6b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ install: - python setup.py install script: - python setup.py test + - 2to3 . -w - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" From 0e7878b406bc23b46140edd54912e4491595fac4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Jun 2014 12:41:26 +0300 Subject: [PATCH 37/44] Only run 2to3 on Python 3.x. Makes sense no? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6281e6b4..5f10339b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ install: - python setup.py install script: - python setup.py test - - 2to3 . -w + - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then 2to3 . -w; fi; - python benchmark.py notifications: irc: "irc.freenode.org#mongoengine" From b02a31d4b920981f6b7330de23e78518f334cc0d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 14:44:44 +0100 Subject: [PATCH 38/44] Updated .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..c7397fd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,7 @@ python: - "3.3" env: - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 + - PYMONGO=dev DJANGO=1.5.8 - PYMONGO=2.5 DJANGO=1.6 - PYMONGO=2.5 DJANGO=1.5.5 - PYMONGO=2.5 DJANGO=1.4.10 @@ -31,3 +30,4 @@ notifications: branches: only: - master + - "0.8" From dd51589f67fd02f3049aadea5268a79cd3799092 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:02:40 +0100 Subject: [PATCH 39/44] Updates --- setup.py | 2 +- tests/document/instance.py | 4 ++-- tests/queryset/queryset.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 85707d00..25075275 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo>=2.5'], + install_requires=['pymongo>=2.7'], test_suite='nose.collector', **extra_opts ) diff --git a/tests/document/instance.py b/tests/document/instance.py index afb27e0d..6f04ac1d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -57,7 +57,7 @@ class InstanceTest(unittest.TestCase): date = DateTimeField(default=datetime.now) meta = { 'max_documents': 10, - 'max_size': 90000, + 'max_size': 4096, } Log.drop_collection() @@ -75,7 +75,7 @@ class InstanceTest(unittest.TestCase): options = Log.objects._collection.options() self.assertEqual(options['capped'], True) self.assertEqual(options['max'], 10) - self.assertEqual(options['size'], 90000) + self.assertTrue(options['size'] >= 4096) # Check that the document cannot be redefined with different options def recreate_log_document(): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index a2438e21..688b1bce 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -650,7 +650,7 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 1) # 1 for the insert + self.assertEqual(q, 99) # profiling logs each doc now :( Blog.drop_collection() Blog.ensure_indexes() @@ -659,7 +659,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) Blog.objects.insert(blogs) - self.assertEqual(q, 2) # 1 for insert, and 1 for in bulk fetch + self.assertEqual(q, 100) # 99 or insert, and 1 for in bulk fetch Blog.drop_collection() @@ -3943,7 +3943,7 @@ class QuerySetTest(unittest.TestCase): if qs: pass - op = q.db.system.profile.find({"ns": + op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] self.assertFalse('$orderby' in op['query'], @@ -3969,7 +3969,7 @@ class QuerySetTest(unittest.TestCase): } Person.drop_collection() - + Person(name="B").save() Person(name="C").save() Person(name="A").save() @@ -3979,13 +3979,13 @@ class QuerySetTest(unittest.TestCase): if Person.objects: pass - op = q.db.system.profile.find({"ns": + op = q.db.system.profile.find({"ns": {"$ne": "%s.system.indexes" % q.db.name}})[0] self.assertFalse('$orderby' in op['query'], 'BaseQuerySet must remove orderby from meta in boolen test') - self.assertEqual(Person.objects.first().name, 'A') + self.assertEqual(Person.objects.first().name, 'A') self.assertTrue(Person.objects._has_data(), 'Cursor has data and returned False') From 5b6c8c191f1418471d8b9f616d5fbded4a797238 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 14:44:44 +0100 Subject: [PATCH 40/44] Updated .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5739909b..3d9d6611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,7 @@ python: - "3.3" env: - PYMONGO=dev DJANGO=1.6 - - PYMONGO=dev DJANGO=1.5.5 - - PYMONGO=dev DJANGO=1.4.10 + - PYMONGO=dev DJANGO=1.5.8 - PYMONGO=2.5 DJANGO=1.6 - PYMONGO=2.5 DJANGO=1.5.5 - PYMONGO=2.5 DJANGO=1.4.10 @@ -31,3 +30,4 @@ notifications: branches: only: - master + - "0.9" From eb9003187d4ce70bdbb32709fc6cb9f20ce7eb16 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:13:01 +0100 Subject: [PATCH 41/44] Updated changelog & authors #673 --- AUTHORS | 1 + docs/changelog.rst | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 170a00e5..c86df67c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -191,3 +191,4 @@ that much better: * Damien Churchill (https://github.com/damoxc) * Jonathan Simon Prates (https://github.com/jonathansp) * Thiago Papageorgiou (https://github.com/tmpapageorgiou) + * Omer Katz (https://github.com/thedrow) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51134238..b94388a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= + +Changes in 0.9.X - DEV +====================== +- pypy support #673 + Changes in 0.8.7 ================ - Calling reload on deleted / nonexistant documents raises DoesNotExist (#538) From 11724aa5555ad7f87e0208b3ce01ac8b7f31323c Mon Sep 17 00:00:00 2001 From: Dmitry Konishchev Date: Thu, 26 Jun 2014 16:18:42 +0100 Subject: [PATCH 42/44] QuerySet.modify() method to provide find_and_modify() like behaviour --- docs/guide/querying.rst | 5 +- mongoengine/queryset/base.py | 54 +++++++++++++++++++ tests/queryset/__init__.py | 1 + tests/queryset/modify.py | 102 +++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/queryset/modify.py diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 32cbb94e..96beea5f 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -488,8 +488,9 @@ calling it with keyword arguments:: Atomic updates ============== Documents may be updated atomically by using the -:meth:`~mongoengine.queryset.QuerySet.update_one` and -:meth:`~mongoengine.queryset.QuerySet.update` methods on a +:meth:`~mongoengine.queryset.QuerySet.update_one`, +:meth:`~mongoengine.queryset.QuerySet.update` and +:meth:`~mongoengine.queryset.QuerySet.modify` methods on a :meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers" that you may use with these methods: diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 89a5e5fb..db60deba 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -10,6 +10,7 @@ import warnings from bson.code import Code from bson import json_util import pymongo +import pymongo.errors from pymongo.common import validate_read_preference from mongoengine import signals @@ -484,6 +485,59 @@ class BaseQuerySet(object): return self.update( upsert=upsert, multi=False, write_concern=write_concern, **update) + def modify(self, upsert=False, full_response=False, remove=False, new=False, **update): + """Update and return the updated document. + + Returns either the document before or after modification based on `new` + parameter. If no documents match the query and `upsert` is false, + returns ``None``. If upserting and `new` is false, returns ``None``. + + If the full_response parameter is ``True``, the return value will be + the entire response object from the server, including the 'ok' and + 'lastErrorObject' fields, rather than just the modified document. + This is useful mainly because the 'lastErrorObject' document holds + information about the command's execution. + + :param upsert: insert if document doesn't exist (default ``False``) + :param full_response: return the entire response object from the + server (default ``False``) + :param remove: remove rather than updating (default ``False``) + :param new: return updated rather than original document + (default ``False``) + :param update: Django-style update keyword arguments + + .. versionadded:: 0.9 + """ + + if remove and new: + raise OperationError("Conflicting parameters: remove and new") + + if not update and not upsert and not remove: + raise OperationError("No update parameters, must either update or remove") + + queryset = self.clone() + query = queryset._query + update = transform.update(queryset._document, **update) + sort = queryset._ordering + + try: + result = queryset._collection.find_and_modify( + query, update, upsert=upsert, sort=sort, remove=remove, new=new, + full_response=full_response, **self._cursor_args) + except pymongo.errors.DuplicateKeyError, err: + raise NotUniqueError(u"Update failed (%s)" % err) + except pymongo.errors.OperationFailure, err: + raise OperationError(u"Update failed (%s)" % err) + + if full_response: + if result["value"] is not None: + result["value"] = self._document._from_son(result["value"]) + else: + if result is not None: + result = self._document._from_son(result) + + return result + def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only and raises InvalidQueryError if a filter has been applied. Returns diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py index 8a93c19f..c36b2684 100644 --- a/tests/queryset/__init__.py +++ b/tests/queryset/__init__.py @@ -3,3 +3,4 @@ from field_list import * from queryset import * from visitor import * from geo import * +from modify import * \ No newline at end of file diff --git a/tests/queryset/modify.py b/tests/queryset/modify.py new file mode 100644 index 00000000..e0c7d1fe --- /dev/null +++ b/tests/queryset/modify.py @@ -0,0 +1,102 @@ +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import connect, Document, IntField + +__all__ = ("FindAndModifyTest",) + + +class Doc(Document): + id = IntField(primary_key=True) + value = IntField() + + +class FindAndModifyTest(unittest.TestCase): + + def setUp(self): + connect(db="mongoenginetest") + Doc.drop_collection() + + def assertDbEqual(self, docs): + self.assertEqual(list(Doc._collection.find().sort("id")), docs) + + def test_modify(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(set__value=-1) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_with_new(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + new_doc = Doc.objects(id=1).modify(set__value=-1, new=True) + doc.value = -1 + self.assertEqual(new_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_not_existing(self): + Doc(id=0, value=0).save() + self.assertEqual(Doc.objects(id=1).modify(set__value=-1), None) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_modify_with_upsert(self): + Doc(id=0, value=0).save() + old_doc = Doc.objects(id=1).modify(set__value=1, upsert=True) + self.assertEqual(old_doc, None) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}]) + + def test_modify_with_upsert_existing(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(set__value=-1, upsert=True) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + def test_modify_with_upsert_with_new(self): + Doc(id=0, value=0).save() + new_doc = Doc.objects(id=1).modify(upsert=True, new=True, set__value=1) + self.assertEqual(new_doc.to_mongo(), {"_id": 1, "value": 1}) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": 1}]) + + def test_modify_with_remove(self): + Doc(id=0, value=0).save() + doc = Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).modify(remove=True) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_find_and_modify_with_remove_not_existing(self): + Doc(id=0, value=0).save() + self.assertEqual(Doc.objects(id=1).modify(remove=True), None) + self.assertDbEqual([{"_id": 0, "value": 0}]) + + def test_modify_with_order_by(self): + Doc(id=0, value=3).save() + Doc(id=1, value=2).save() + Doc(id=2, value=1).save() + doc = Doc(id=3, value=0).save() + + old_doc = Doc.objects().order_by("-id").modify(set__value=-1) + self.assertEqual(old_doc.to_json(), doc.to_json()) + self.assertDbEqual([ + {"_id": 0, "value": 3}, {"_id": 1, "value": 2}, + {"_id": 2, "value": 1}, {"_id": 3, "value": -1}]) + + def test_modify_with_fields(self): + Doc(id=0, value=0).save() + Doc(id=1, value=1).save() + + old_doc = Doc.objects(id=1).only("id").modify(set__value=-1) + self.assertEqual(old_doc.to_mongo(), {"_id": 1}) + self.assertDbEqual([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}]) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 2c07d7736847fa1157ab38c6247ecaa0ac34c0e1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:24:37 +0100 Subject: [PATCH 43/44] Updated changelog Enabled connection pooling --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b94388a1..bead491c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in 0.9.X - DEV ====================== - pypy support #673 +- Enabled connection pooling #674 Changes in 0.8.7 ================ From d1d59722774f59d3ee9b935bfda9ff7e373b43c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jun 2014 16:34:02 +0100 Subject: [PATCH 44/44] Removed support for old versions - Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. - Removing support for Python < 2.6.6 --- .travis.yml | 34 +-------------------------------- docs/changelog.rst | 2 ++ mongoengine/base/document.py | 7 +------ mongoengine/python_support.py | 32 ------------------------------- mongoengine/queryset/visitor.py | 5 +++-- setup.py | 6 ++++-- 6 files changed, 11 insertions(+), 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index c20e52af..40736165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,44 +11,12 @@ python: env: - PYMONGO=dev DJANGO=1.6.5 - PYMONGO=dev DJANGO=1.5.8 - - PYMONGO=dev DJANGO=1.4.13 - - PYMONGO=2.5.2 DJANGO=1.6.5 - - PYMONGO=2.5.2 DJANGO=1.5.8 - - PYMONGO=2.5.2 DJANGO=1.4.13 - - PYMONGO=2.6.3 DJANGO=1.6.5 - - PYMONGO=2.6.3 DJANGO=1.5.8 - - PYMONGO=2.6.3 DJANGO=1.4.13 - PYMONGO=2.7.1 DJANGO=1.6.5 - PYMONGO=2.7.1 DJANGO=1.5.8 - - PYMONGO=2.7.1 DJANGO=1.4.13 matrix: fast_finish: true - exclude: - - python: "3.2" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.2" - env: PYMONGO=2.7.1 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.3" - env: PYMONGO=2.7.1 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=dev DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.5.2 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.6.3 DJANGO=1.4.13 - - python: "3.4" - env: PYMONGO=2.7.1 DJANGO=1.4.13 + install: - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi diff --git a/docs/changelog.rst b/docs/changelog.rst index bead491c..c980e904 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,8 @@ Changes in 0.9.X - DEV ====================== - pypy support #673 - Enabled connection pooling #674 +- Removing support for Django 1.4.x, pymongo 2.5.x, pymongo 2.6.x. +- Removing support for Python < 2.6.6 Changes in 0.8.7 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f5eae8ff..43b865ce 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -13,8 +13,7 @@ from mongoengine import signals from mongoengine.common import _import_class from mongoengine.errors import (ValidationError, InvalidDocumentError, LookUpError) -from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, - to_str_keys_recursive) +from mongoengine.python_support import PY3, txt_type from mongoengine.base.common import get_document, ALLOW_INHERITANCE from mongoengine.base.datastructures import BaseDict, BaseList @@ -545,10 +544,6 @@ class BaseDocument(object): # class if unavailable class_name = son.get('_cls', cls._class_name) data = dict(("%s" % key, value) for key, value in son.iteritems()) - if not UNICODE_KWARGS: - # python 2.6.4 and lower cannot handle unicode keys - # passed to class constructor example: cls(**data) - to_str_keys_recursive(data) # Return correct subclass for document type if class_name != cls._class_name: diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py index 097740eb..2c4df00c 100644 --- a/mongoengine/python_support.py +++ b/mongoengine/python_support.py @@ -3,8 +3,6 @@ import sys PY3 = sys.version_info[0] == 3 -PY25 = sys.version_info[:2] == (2, 5) -UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264 if PY3: import codecs @@ -29,33 +27,3 @@ else: txt_type = unicode str_types = (bin_type, txt_type) - -if PY25: - def product(*args, **kwds): - pools = map(tuple, args) * kwds.get('repeat', 1) - result = [[]] - for pool in pools: - result = [x + [y] for x in result for y in pool] - for prod in result: - yield tuple(prod) - reduce = reduce -else: - from itertools import product - from functools import reduce - - -# For use with Python 2.5 -# converts all keys from unicode to str for d and all nested dictionaries -def to_str_keys_recursive(d): - if isinstance(d, list): - for val in d: - if isinstance(val, (dict, list)): - to_str_keys_recursive(val) - elif isinstance(d, dict): - for key, val in d.items(): - if isinstance(val, (dict, list)): - to_str_keys_recursive(val) - if isinstance(key, unicode): - d[str(key)] = d.pop(key) - else: - raise ValueError("non list/dict parameter not allowed") diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 41f4ebf8..a39b05f0 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -1,8 +1,9 @@ import copy -from mongoengine.errors import InvalidQueryError -from mongoengine.python_support import product, reduce +from itertools import product +from functools import reduce +from mongoengine.errors import InvalidQueryError from mongoengine.queryset import transform __all__ = ('Q',) diff --git a/setup.py b/setup.py index d37cede1..7270331a 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,14 @@ CLASSIFIERS = [ 'Operating System :: OS Independent', 'Programming Language :: Python', "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.6.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.1", "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ]