From 6a4c342e45fbd556dc1938923b7bed17031c79c7 Mon Sep 17 00:00:00 2001 From: Dmitry Voronenkov Date: Tue, 18 Jun 2019 16:13:29 +0300 Subject: [PATCH 01/58] Supported updates of an array by negative index --- mongoengine/base/datastructures.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index fafc08b7..9307556f 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -108,6 +108,9 @@ class BaseList(list): super(BaseList, self).__init__(list_items) def __getitem__(self, key): + # change index to positive value because MongoDB does not support negative one + if isinstance(key, int) and key < 0: + key = len(self) + key value = super(BaseList, self).__getitem__(key) if isinstance(key, slice): From c68e3e1238bf77997dbe65415935abe8528ffef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 24 Jul 2019 21:37:16 +0200 Subject: [PATCH 02/58] Add test case for list update by negative index --- tests/document/instance.py | 49 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index d8841a40..d0193b60 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -39,10 +39,10 @@ from tests.utils import MongoDBTestCase, get_as_pymongo TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), "../fields/mongoengine.png") -__all__ = ("InstanceTest",) +__all__ = ("TestDocumentInstance",) -class InstanceTest(MongoDBTestCase): +class TestDocumentInstance(MongoDBTestCase): def setUp(self): class Job(EmbeddedDocument): name = StringField() @@ -3599,6 +3599,51 @@ class InstanceTest(MongoDBTestCase): self.assertEqual(b._instance, a) self.assertEqual(idx, 2) + def test_updating_listfield_manipulate_list(self): + class Company(Document): + name = StringField() + employees = ListField(field=DictField()) + + Company.drop_collection() + + comp = Company(name="BigBank", employees=[{"name": "John"}]) + comp.save() + comp.employees.append({"name": "Bill"}) + comp.save() + + stored_comp = get_as_pymongo(comp) + self.assertEqual( + stored_comp, + { + "_id": comp.id, + "employees": [{"name": "John"}, {"name": "Bill"}], + "name": "BigBank", + }, + ) + + comp = comp.reload() + comp.employees[0]["color"] = "red" + comp.employees[-1]["color"] = "blue" + comp.employees[-1].update({"size": "xl"}) + comp.save() + + assert len(comp.employees) == 2 + assert comp.employees[0] == {"name": "John", "color": "red"} + assert comp.employees[1] == {"name": "Bill", "size": "xl", "color": "blue"} + + stored_comp = get_as_pymongo(comp) + self.assertEqual( + stored_comp, + { + "_id": comp.id, + "employees": [ + {"name": "John", "color": "red"}, + {"size": "xl", "color": "blue", "name": "Bill"}, + ], + "name": "BigBank", + }, + ) + def test_falsey_pk(self): """Ensure that we can create and update a document with Falsey PK.""" From 8f288fe45875050ac39985dcf9bcc304b6c1f15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 29 Sep 2019 22:48:46 +0200 Subject: [PATCH 03/58] add mongodb 4.0 to travis and docs --- .travis.yml | 10 ++++++---- README.rst | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54a6befd..21321841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ # with a very large number of jobs, hence we only test a subset of all the # combinations: # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, and PyPy. +# tested against Python v2.7, v3.5, v3.6, v3.7, PyPy and PyPy3. # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo # combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. @@ -30,15 +30,16 @@ dist: xenial env: global: + - MONGODB_4_0=4.0.12 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 + - PYMONGO_3_9=3.9 - PYMONGO_3_6=3.6 - PYMONGO_3_4=3.4 matrix: - - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_6} + - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9} matrix: - # Finish the build as soon as one job fails fast_finish: true @@ -47,7 +48,8 @@ matrix: env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - + - python: 3.7 + env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_9} install: # Install Mongo diff --git a/README.rst b/README.rst index 679980f8..82b32893 100644 --- a/README.rst +++ b/README.rst @@ -26,10 +26,10 @@ an `API reference `_. Supported MongoDB Versions ========================== -MongoEngine is currently tested against MongoDB v3.4 and v3.6. Future versions +MongoEngine is currently tested against MongoDB v3.4, v3.6 and v4.0. Future versions should be supported as well, but aren't actively tested at the moment. Make sure to open an issue or submit a pull request if you experience any problems -with MongoDB version > 3.6. +with MongoDB version > 4.0. Installation ============ From b61c8cd104975b5fb47681387abc443c8c8430b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 1 Oct 2019 22:17:19 +0200 Subject: [PATCH 04/58] fix tox envs --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a1ae8444..a7921c61 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36} +envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39} [testenv] commands = @@ -8,5 +8,6 @@ deps = nose mg34: pymongo>=3.4,<3.5 mg36: pymongo>=3.6,<3.7 + mg39: pymongo>=3.9,<4.0 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs From cb77bb6b69bb80c79c97d2f0792d173fc2f443d4 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:21:03 +0100 Subject: [PATCH 05/58] Implement __bool__ on Q and QCombination --- mongoengine/queryset/base.py | 2 +- mongoengine/queryset/visitor.py | 20 ++++++++++++++++---- tests/queryset/test_visitor.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a648391e..c6f467cc 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -686,7 +686,7 @@ class BaseQuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ queryset = self.clone() - if not queryset._query_obj.empty: + if queryset._query_obj: msg = "Cannot use a filter whilst using `with_id`" raise InvalidQueryError(msg) return queryset.filter(pk=object_id).first() diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 058c722a..a7295ae5 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -2,6 +2,8 @@ import copy from mongoengine.errors import InvalidQueryError from mongoengine.queryset import transform +import warnings + __all__ = ("Q", "QNode") @@ -101,13 +103,15 @@ class QNode(object): return self # Or if this Q is empty, ignore it and just use `other`. - if self.empty: + if not self: return other return QCombination(operation, [self, other]) @property def empty(self): + msg = "'empty' property is deprecated in favour of using 'not bool(filter)" + warnings.warn(msg, DeprecationWarning) return False def __or__(self, other): @@ -137,6 +141,9 @@ class QCombination(QNode): op = " & " if self.operation is self.AND else " | " return "(%s)" % op.join([repr(node) for node in self.children]) + def __bool__(self): + return bool(self.children) + def accept(self, visitor): for i in range(len(self.children)): if isinstance(self.children[i], QNode): @@ -146,6 +153,8 @@ class QCombination(QNode): @property def empty(self): + msg = "'empty' property is deprecated in favour of using 'not bool(filter)" + warnings.warn(msg, DeprecationWarning) return not bool(self.children) def __eq__(self, other): @@ -167,12 +176,15 @@ class Q(QNode): def __repr__(self): return "Q(**%s)" % repr(self.query) + def __bool__(self): + return bool(self.query) + + def __eq__(self, other): + return self.__class__ == other.__class__ and self.query == other.query + def accept(self, visitor): return visitor.visit_query(self) @property def empty(self): return not bool(self.query) - - def __eq__(self, other): - return self.__class__ == other.__class__ and self.query == other.query diff --git a/tests/queryset/test_visitor.py b/tests/queryset/test_visitor.py index afa00839..81e0f253 100644 --- a/tests/queryset/test_visitor.py +++ b/tests/queryset/test_visitor.py @@ -407,6 +407,17 @@ class TestQ(unittest.TestCase): def test_combine_or_both_empty(self): assert Q() | Q() == Q() + def test_q_bool(self): + assert Q(name="John") + assert not Q() + + def test_combine_bool(self): + assert not Q() & Q() + assert Q() & Q(name="John") + assert Q(name="John") & Q() + assert Q() | Q(name="John") + assert Q(name="John") | Q() + if __name__ == "__main__": unittest.main() From bd6c52e025fbe60473a6f009b100eb4d8edbfe83 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:30:03 +0100 Subject: [PATCH 06/58] Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99081957..e2ffa41e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Development - In bulk write insert, the detailed error message would raise in exception. - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 +- Added ability to check if Q or Q operations is empty by parsing them to bool #2210 Changes in 0.18.2 ================= From 5f14d958ac32925df18e757f77f729f3bfb79c5a Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:46:57 +0100 Subject: [PATCH 07/58] Sort imports --- mongoengine/queryset/visitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index a7295ae5..8038d23f 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -1,9 +1,8 @@ import copy +import warnings from mongoengine.errors import InvalidQueryError from mongoengine.queryset import transform -import warnings - __all__ = ("Q", "QNode") From 6e8196d475953f88bd70207e81234bc07e1526d0 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 01:31:37 +0100 Subject: [PATCH 08/58] Python 2.x compatibility --- mongoengine/queryset/visitor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 8038d23f..7faed897 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -143,6 +143,8 @@ class QCombination(QNode): def __bool__(self): return bool(self.children) + __nonzero__ = __bool__ # For Py2 support + def accept(self, visitor): for i in range(len(self.children)): if isinstance(self.children[i], QNode): @@ -178,6 +180,8 @@ class Q(QNode): def __bool__(self): return bool(self.query) + __nonzero__ = __bool__ # For Py2 support + def __eq__(self, other): return self.__class__ == other.__class__ and self.query == other.query From 1b38309d70efc122720d9c5d3fcc6d362436ed62 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Fri, 6 Dec 2019 10:14:22 +0100 Subject: [PATCH 09/58] Revert 'empty' usage to it's previous state --- mongoengine/queryset/base.py | 2 +- mongoengine/queryset/visitor.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c6f467cc..a648391e 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -686,7 +686,7 @@ class BaseQuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ queryset = self.clone() - if queryset._query_obj: + if not queryset._query_obj.empty: msg = "Cannot use a filter whilst using `with_id`" raise InvalidQueryError(msg) return queryset.filter(pk=object_id).first() diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 7faed897..470839c1 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -102,14 +102,14 @@ class QNode(object): return self # Or if this Q is empty, ignore it and just use `other`. - if not self: + if self.empty: return other return QCombination(operation, [self, other]) @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)" + msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" warnings.warn(msg, DeprecationWarning) return False @@ -154,7 +154,7 @@ class QCombination(QNode): @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)" + msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" warnings.warn(msg, DeprecationWarning) return not bool(self.children) From e83132f32c254f04cb505d1ede1d90f0dac84b18 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Tue, 10 Dec 2019 11:51:33 +0100 Subject: [PATCH 10/58] Note deprecation of 'empty' in changelog --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2ffa41e..bc01a403 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,9 @@ Development - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it. - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103 - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required. +- DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210 + - Added ability to check if Q or QNode are empty by parsing them to bool. + - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``. - Improve error message related to InvalidDocumentError #2180 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110 @@ -25,7 +28,6 @@ Development - In bulk write insert, the detailed error message would raise in exception. - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 -- Added ability to check if Q or Q operations is empty by parsing them to bool #2210 Changes in 0.18.2 ================= From b2053144241f8f579993eff2c4de3084f074535f Mon Sep 17 00:00:00 2001 From: Matt Simpson Date: Tue, 10 Dec 2019 11:09:22 -0500 Subject: [PATCH 11/58] Add ability for dict keys to have . or $ in MongoDB >= 3.6 Starting in MongoDB >= 3.6, it is valid for dictionary keys to have $ or . in them as long as they don't start with $. Additional tests added. --- AUTHORS | 1 + mongoengine/fields.py | 19 ++++++++++++++++++- tests/fields/test_dict_field.py | 21 ++++++++++++++++----- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index aa044bd2..1271a8d9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -253,3 +253,4 @@ that much better: * Gaurav Dadhania (https://github.com/GVRV) * Yurii Andrieiev (https://github.com/yandrieiev) * Filip Kucharczyk (https://github.com/Pacu2) + * Matthew Simpson (https://github.com/mcsimps2) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f8f527a3..c3d93740 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -41,6 +41,7 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.document import Document, EmbeddedDocument from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError +from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version from mongoengine.python_support import StringIO from mongoengine.queryset import DO_NOTHING from mongoengine.queryset.base import BaseQuerySet @@ -1051,6 +1052,15 @@ def key_has_dot_or_dollar(d): return True +def key_starts_with_dollar(d): + """Helper function to recursively determine if any key in a + dictionary starts with a dollar + """ + for k, v in d.items(): + if (k.startswith("$")) or (isinstance(v, dict) and key_starts_with_dollar(v)): + return True + + class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -1077,11 +1087,18 @@ class DictField(ComplexBaseField): if key_not_string(value): msg = "Invalid dictionary key - documents must have only string keys" self.error(msg) - if key_has_dot_or_dollar(value): + + curr_mongo_ver = get_mongodb_version() + + if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value): self.error( 'Invalid dictionary key name - keys may not contain "."' ' or startswith "$" characters' ) + elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value): + self.error( + 'Invalid dictionary key name - keys may not startswith "$" characters' + ) super(DictField, self).validate(value) def lookup_member(self, member_name): diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index e88128f9..44e628f6 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -3,6 +3,7 @@ import pytest from mongoengine import * from mongoengine.base import BaseDict +from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version from tests.utils import MongoDBTestCase, get_as_pymongo @@ -43,11 +44,7 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() - post.info = {"the.title": "test"} - with pytest.raises(ValidationError): - post.validate() - - post.info = {"nested": {"the.title": "test"}} + post.info = {"$title.test": "test"} with pytest.raises(ValidationError): post.validate() @@ -55,6 +52,20 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() + post.info = {"nested": {"the.title": "test"}} + if get_mongodb_version() < MONGODB_36: + with pytest.raises(ValidationError): + post.validate() + else: + post.validate() + + post.info = {"dollar_and_dot": {"te$st.test": "test"}} + if get_mongodb_version() < MONGODB_36: + with pytest.raises(ValidationError): + post.validate() + else: + post.validate() + post.info = {"title": "test"} post.save() From 3b099f936a02444b3bf02c7dcdf13b1f2fc3b895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 13 Dec 2019 21:32:45 +0100 Subject: [PATCH 12/58] provide additional details on how inheritance works in doc --- docs/guide/defining-documents.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 9dcca88c..652c5cd9 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -744,7 +744,7 @@ Document inheritance To create a specialised type of a :class:`~mongoengine.Document` you have defined, you may subclass it and add any extra fields or methods you may need. -As this is new class is not a direct subclass of +As this new class is not a direct subclass of :class:`~mongoengine.Document`, it will not be stored in its own collection; it will use the same collection as its superclass uses. This allows for more convenient and efficient retrieval of related documents -- all you need do is @@ -767,6 +767,27 @@ document.:: Setting :attr:`allow_inheritance` to True should also be used in :class:`~mongoengine.EmbeddedDocument` class in case you need to subclass it +When it comes to querying using :attr:`.objects()`, querying `Page.objects()` will query +both `Page` and `DatedPage` whereas querying `DatedPage` will only query the `DatedPage` documents. +Behind the scenes, MongoEngine deals with inheritance by adding a :attr:`_cls` attribute that contains +the class name in every documents. When a document is loaded, MongoEngine checks +it's :attr:`_cls` attribute and use that class to construct the instance.:: + + Page(title='a funky title').save() + DatedPage(title='another title', date=datetime.utcnow()).save() + + print(Page.objects().count()) # 2 + print(DatedPage.objects().count()) # 1 + + # print documents in their native form + # we remove 'id' to avoid polluting the output with unnecessary detail + qs = Page.objects.exclude('id').as_pymongo() + print(list(qs)) + # [ + # {'_cls': u 'Page', 'title': 'a funky title'}, + # {'_cls': u 'Page.DatedPage', 'title': u 'another title', 'date': datetime.datetime(2019, 12, 13, 20, 16, 59, 993000)} + # ] + Working with existing data -------------------------- As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and From 280a73af3bea8e9232a5ebe761d451840f025135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 14 Dec 2019 21:44:59 +0100 Subject: [PATCH 13/58] minor fix in doc of NULLIFY to improve #834 --- docs/guide/defining-documents.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 9dcca88c..82388d3d 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -352,7 +352,7 @@ Its value can take any of the following constants: Deletion is denied if there still exist references to the object being deleted. :const:`mongoengine.NULLIFY` - Any object's fields still referring to the object being deleted are removed + Any object's fields still referring to the object being deleted are set to None (using MongoDB's "unset" operation), effectively nullifying the relationship. :const:`mongoengine.CASCADE` Any object containing fields that are referring to the object being deleted From 50882e5bb09b74faddfac3cb93afd278ea94ced2 Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Wed, 16 Oct 2019 09:49:40 -0400 Subject: [PATCH 14/58] Add failing test Test that __eq__ for EmbeddedDocuments with LazyReferenceFields works as expected. --- tests/document/test_instance.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 173e02f2..6ba6827e 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -3319,6 +3319,38 @@ class TestInstance(MongoDBTestCase): f1.ref # Dereferences lazily assert f1 == f2 + def test_embedded_document_equality_with_lazy_ref(self): + class Job(EmbeddedDocument): + boss = LazyReferenceField('Person') + + class Person(Document): + job = EmbeddedDocumentField(Job) + + Person.drop_collection() + + boss = Person() + worker = Person(job=Job(boss=boss)) + boss.save() + worker.save() + + worker1 = Person.objects.get(id=worker.id) + + # worker1.job should be equal to the job used originally to create the + # document. + self.assertEqual(worker1.job, worker.job) + + # worker1.job should be equal to a newly created Job EmbeddedDocument + # using either the Boss object or his ID. + self.assertEqual(worker1.job, Job(boss=boss)) + self.assertEqual(worker1.job, Job(boss=boss.id)) + + # The above equalities should also hold after worker1.job.boss has been + # fetch()ed. + worker1.job.boss.fetch() + self.assertEqual(worker1.job, worker.job) + self.assertEqual(worker1.job, Job(boss=boss)) + self.assertEqual(worker1.job, Job(boss=boss.id)) + def test_dbref_equality(self): class Test2(Document): name = StringField() From dc7b96a5691335e970b13fb30ef62426b126e2bd Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Wed, 16 Oct 2019 09:50:47 -0400 Subject: [PATCH 15/58] Make python value for LazyReferenceFields be a DBRef Previously, when reading a LazyReferenceField from the DB, it was stored internally in the parent document's _data field as an ObjectId. However, this meant that equality tests using an enclosing EmbeddedDocument would not return True when the EmbeddedDocument being compared to contained a DBRef or Document in _data. Enclosing Documents were largely unaffected because they look at the primary key for equality (which EmbeddedDocuments lack). This makes the internal Python representation of a LazyReferenceField (before the LazyReference itself has been constructed) a DBRef, using code identical to ReferenceField. --- mongoengine/fields.py | 9 +++++++++ tests/document/test_instance.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f8f527a3..0c29d1bc 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -2502,6 +2502,15 @@ class LazyReferenceField(BaseField): else: return pk + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type.""" + if not self.dbref and not isinstance( + value, (DBRef, Document, EmbeddedDocument) + ): + collection = self.document_type._get_collection_name() + value = DBRef(collection, self.document_type.id.to_python(value)) + return value + def validate(self, value): if isinstance(value, LazyReference): if value.collection != self.document_type._get_collection_name(): diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 6ba6827e..07376b4b 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -3321,7 +3321,7 @@ class TestInstance(MongoDBTestCase): def test_embedded_document_equality_with_lazy_ref(self): class Job(EmbeddedDocument): - boss = LazyReferenceField('Person') + boss = LazyReferenceField("Person") class Person(Document): job = EmbeddedDocumentField(Job) From 0d4e61d489a9264863cecdfed08fc9e67a74d03a Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Wed, 16 Oct 2019 10:01:19 -0400 Subject: [PATCH 16/58] Add daewok to AUTHORS per contributing guidelines --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index aa044bd2..374e2f7f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -253,3 +253,4 @@ that much better: * Gaurav Dadhania (https://github.com/GVRV) * Yurii Andrieiev (https://github.com/yandrieiev) * Filip Kucharczyk (https://github.com/Pacu2) + * Eric Timmons (https://github.com/daewok) From 68dc2925fbea13702fa23ced0afd786d77b2ca28 Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Sun, 15 Dec 2019 12:08:04 -0500 Subject: [PATCH 17/58] Add LazyReferenceField with dbref=True to embedded_document equality test --- tests/document/test_instance.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 07376b4b..b899684f 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -3322,6 +3322,7 @@ class TestInstance(MongoDBTestCase): def test_embedded_document_equality_with_lazy_ref(self): class Job(EmbeddedDocument): boss = LazyReferenceField("Person") + boss_dbref = LazyReferenceField("Person", dbref=True) class Person(Document): job = EmbeddedDocumentField(Job) @@ -3329,7 +3330,7 @@ class TestInstance(MongoDBTestCase): Person.drop_collection() boss = Person() - worker = Person(job=Job(boss=boss)) + worker = Person(job=Job(boss=boss, boss_dbref=boss)) boss.save() worker.save() @@ -3341,15 +3342,15 @@ class TestInstance(MongoDBTestCase): # worker1.job should be equal to a newly created Job EmbeddedDocument # using either the Boss object or his ID. - self.assertEqual(worker1.job, Job(boss=boss)) - self.assertEqual(worker1.job, Job(boss=boss.id)) + self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss)) + self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id)) # The above equalities should also hold after worker1.job.boss has been # fetch()ed. worker1.job.boss.fetch() self.assertEqual(worker1.job, worker.job) - self.assertEqual(worker1.job, Job(boss=boss)) - self.assertEqual(worker1.job, Job(boss=boss.id)) + self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss)) + self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id)) def test_dbref_equality(self): class Test2(Document): From 329f030a41da4d93aaec1f3ccee634e898f7d289 Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Sun, 15 Dec 2019 20:15:13 -0500 Subject: [PATCH 18/58] Always store a DBRef, Document, or EmbeddedDocument in LazyReferenceField._data This is required to handle the case of equality tests on a LazyReferenceField with dbref=True when comparing against a field instantiated with an ObjectId. --- mongoengine/fields.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0c29d1bc..a385559d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -2504,9 +2504,7 @@ class LazyReferenceField(BaseField): def to_python(self, value): """Convert a MongoDB-compatible type to a Python type.""" - if not self.dbref and not isinstance( - value, (DBRef, Document, EmbeddedDocument) - ): + if not isinstance(value, (DBRef, Document, EmbeddedDocument)): collection = self.document_type._get_collection_name() value = DBRef(collection, self.document_type.id.to_python(value)) return value From cfd4d6a161556ef4a8aa355468384554eb684442 Mon Sep 17 00:00:00 2001 From: Eric Timmons Date: Sun, 15 Dec 2019 12:02:24 -0500 Subject: [PATCH 19/58] Add breaking change to changelog for LazyReferenceField representation in _data --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bc01a403..b308c5fb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Development - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it. - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103 - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required. +- BREAKING CHANGE: A ``LazyReferenceField`` is now stored in the ``_data`` field of its parent as a ``DBRef``, ``Document``, or ``EmbeddedDocument`` (``ObjectId`` is no longer allowed). #2182 - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210 - Added ability to check if Q or QNode are empty by parsing them to bool. - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``. From ae326678ec0d7345645120c864b8b15cac741dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 22:09:04 +0100 Subject: [PATCH 20/58] updated changelog for upcoming 0.19.0 --- docs/changelog.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b308c5fb..5c1e838a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,8 +6,9 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). -- Documentation improvements: - - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver. + +Changes in 0.19.0 +================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 - Instead of ``Doc.objects(foo=bar, read_preference=...)`` use ``Doc.objects(foo=bar).read_preference(...)``. - Instead of ``Doc.objects(foo=bar, class_check=False)`` use ``Doc.objects(foo=bar).clear_cls_query(...)``. @@ -21,14 +22,16 @@ Development - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210 - Added ability to check if Q or QNode are empty by parsing them to bool. - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``. -- Improve error message related to InvalidDocumentError #2180 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110 -- Switch from nosetest to pytest as test runner #2114 -- The codebase is now formatted using ``black``. #2109 -- In bulk write insert, the detailed error message would raise in exception. +- Improve error message related to InvalidDocumentError #2180 +- Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152 - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 +- Switch from nosetest to pytest as test runner #2114 +- The codebase is now formatted using ``black``. #2109 +- Documentation improvements: + - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver. Changes in 0.18.2 ================= From 12d8bd5a22f0cee1f99f87b37bbaec94f0a002bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 22:11:43 +0100 Subject: [PATCH 21/58] bump version to 0.19.0 --- mongoengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index d7093d28..c41b5e70 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 18, 2) +VERSION = (0, 19, 0) def get_version(): From d44533d95651559a65ce72664dca9ede98aa58e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 22:41:22 +0100 Subject: [PATCH 22/58] completed the changelog with missing details --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5c1e838a..93776a70 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,11 +23,13 @@ Changes in 0.19.0 - Added ability to check if Q or QNode are empty by parsing them to bool. - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``. - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125 +- Only set no_cursor_timeout when requested (fixes an incompatibility with MongoDB 4.2) #2148 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110 - Improve error message related to InvalidDocumentError #2180 - Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152 - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 +- Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024 - Switch from nosetest to pytest as test runner #2114 - The codebase is now formatted using ``black``. #2109 - Documentation improvements: From 332bd767d43af14bbb783e779d585cd2dbcf21de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 22:51:08 +0100 Subject: [PATCH 23/58] minor fixes in tests --- docs/guide/mongomock.rst | 6 +++--- tests/document/test_instance.py | 22 +++++++++++----------- tests/fields/test_file_field.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst index d70ee6a6..040ff912 100644 --- a/docs/guide/mongomock.rst +++ b/docs/guide/mongomock.rst @@ -2,10 +2,10 @@ Use mongomock for testing ============================== -`mongomock `_ is a package to do just +`mongomock `_ is a package to do just what the name implies, mocking a mongo database. -To use with mongoengine, simply specify mongomock when connecting with +To use with mongoengine, simply specify mongomock when connecting with mongoengine: .. code-block:: python @@ -45,4 +45,4 @@ Example of test file: pers.save() fresh_pers = Person.objects().first() - self.assertEqual(fresh_pers.name, 'John') + assert fresh_pers.name == 'John' diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index b899684f..609d0690 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -3338,19 +3338,19 @@ class TestInstance(MongoDBTestCase): # worker1.job should be equal to the job used originally to create the # document. - self.assertEqual(worker1.job, worker.job) + assert worker1.job == worker.job # worker1.job should be equal to a newly created Job EmbeddedDocument # using either the Boss object or his ID. - self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss)) - self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id)) + assert worker1.job == Job(boss=boss, boss_dbref=boss) + assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id) # The above equalities should also hold after worker1.job.boss has been # fetch()ed. worker1.job.boss.fetch() - self.assertEqual(worker1.job, worker.job) - self.assertEqual(worker1.job, Job(boss=boss, boss_dbref=boss)) - self.assertEqual(worker1.job, Job(boss=boss.id, boss_dbref=boss.id)) + assert worker1.job == worker.job + assert worker1.job == Job(boss=boss, boss_dbref=boss) + assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id) def test_dbref_equality(self): class Test2(Document): @@ -3693,13 +3693,13 @@ class TestInstance(MongoDBTestCase): value = u"I_should_be_a_dict" coll.insert_one({"light_saber": value}) - with self.assertRaises(InvalidDocumentError) as cm: + with pytest.raises(InvalidDocumentError) as exc_info: list(Jedi.objects) - self.assertEqual( - str(cm.exception), - "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" - % type(value), + assert str( + exc_info.value + ) == "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" % type( + value ) diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py index bfc86511..b8ece1a9 100644 --- a/tests/fields/test_file_field.py +++ b/tests/fields/test_file_field.py @@ -151,7 +151,7 @@ class TestFileField(MongoDBTestCase): result = StreamFile.objects.first() assert streamfile == result assert result.the_file.read() == text + more_text - # self.assertEqual(result.the_file.content_type, content_type) + # assert result.the_file.content_type == content_type result.the_file.seek(0) assert result.the_file.tell() == 0 assert result.the_file.read(len(text)) == text From 1170de1e8e30b976895c1c92cca134089dc5b806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 23:16:29 +0100 Subject: [PATCH 24/58] added explicit doc for order_by #2117 --- docs/guide/mongomock.rst | 2 +- docs/guide/querying.rst | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst index 040ff912..141d7b69 100644 --- a/docs/guide/mongomock.rst +++ b/docs/guide/mongomock.rst @@ -21,7 +21,7 @@ or with an alias: conn = get_connection('testdb') Example of test file: --------- +--------------------- .. code-block:: python import unittest diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index d64c169c..121325ae 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -222,6 +222,18 @@ keyword argument:: .. versionadded:: 0.4 +Sorting/Ordering results +======================== +It is possible to order the results by 1 or more keys using :meth:`~mongoengine.queryset.QuerySet.order_by`. +The order may be specified by prepending each of the keys by "+" or "-". Ascending order is assumed if there's no prefix.:: + + # Order by ascending date + blogs = BlogPost.objects().order_by('date') # equivalent to .order_by('+date') + + # Order by ascending date first, then descending title + blogs = BlogPost.objects().order_by('+date', '-title') + + Limiting and skipping results ============================= Just as with traditional ORMs, you may limit the number of results returned or @@ -585,7 +597,8 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: ['database', 'mongodb'] From MongoDB version 2.6, push operator supports $position value which allows -to push values with index. +to push values with index:: + >>> post = BlogPost(title="Test", tags=["mongo"]) >>> post.save() >>> post.update(push__tags__0=["database", "code"]) From 8e892dccfe01e284a99928a6ee4fbb1e0c35519d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 20 Dec 2019 23:51:01 +0100 Subject: [PATCH 25/58] document recent merged PR in changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 93776a70..06eb8d0c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,7 @@ Changes in 0.19.0 - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 - Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024 +- Fix updates of a list field by negative index #2094 - Switch from nosetest to pytest as test runner #2114 - The codebase is now formatted using ``black``. #2109 - Documentation improvements: From 488604ff2e6cdadeed538de0dfb9fe36e27473b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 24 Dec 2019 00:00:15 +0100 Subject: [PATCH 26/58] test python 3.8 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cbf34cde..c825571b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ python: - 3.5 - 3.6 - 3.7 +- 3.8 - pypy - pypy3 @@ -50,7 +51,8 @@ matrix: env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} - + - python: 3.8 + env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} install: # Install Mongo From 62c8597a3b33dfd34cc785687c747606a0bbba9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 24 Dec 2019 11:15:23 +0100 Subject: [PATCH 27/58] fix pypi deployment version match --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cbf34cde..6680a7e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,5 +110,5 @@ deploy: on: tags: true repo: MongoEngine/mongoengine - condition: ($PYMONGO = ${PYMONGO_3_6}) && ($MONGODB = ${MONGODB_3_4}) + condition: ($PYMONGO = ${PYMONGO_3_9}) && ($MONGODB = ${MONGODB_3_4}) python: 2.7 From ca4967311d3328fcc7abb84f8e9cdda7409e8dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Thu, 26 Dec 2019 21:00:34 +0100 Subject: [PATCH 28/58] update python 3.8 config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 971a51a2..42f2a112 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,8 +51,6 @@ matrix: env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} - - python: 3.8 - env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} install: # Install Mongo From f0d1ee2cb41f1c9794eeee4de445fb27a7a77f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Thu, 26 Dec 2019 21:03:34 +0100 Subject: [PATCH 29/58] update travis config + improve readmecode --- .travis.yml | 2 +- README.rst | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42f2a112..809fbad8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ # with a very large number of jobs, hence we only test a subset of all the # combinations: # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, and PyPy. +# tested against Python v2.7, v3.5, v3.6, v3.7, v3.8 and PyPy. # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo # combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. diff --git a/README.rst b/README.rst index 853d8fbe..3b85fd48 100644 --- a/README.rst +++ b/README.rst @@ -91,12 +91,11 @@ Some simple examples of what MongoEngine code looks like: # Iterate over all posts using the BlogPost superclass >>> for post in BlogPost.objects: - ... print '===', post.title, '===' + ... print('===', post.title, '===') ... if isinstance(post, TextPost): - ... print post.content + ... print(post.content) ... elif isinstance(post, LinkPost): - ... print 'Link:', post.url - ... print + ... print('Link:', post.url) ... # Count all blog posts and its subtypes From aa02f87b69787ac678d4ee740e1cb2e5e6753fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 27 Dec 2019 09:23:15 +0100 Subject: [PATCH 30/58] change & deprecate .aggregate api to mimic pymongo's interface + separate the aggregation tests from the large test_queryset.py file --- docs/guide/querying.rst | 4 +- mongoengine/queryset/base.py | 29 ++- tests/queryset/test_queryset.py | 236 +----------------- tests/queryset/test_queryset_aggregation.py | 255 ++++++++++++++++++++ 4 files changed, 277 insertions(+), 247 deletions(-) create mode 100644 tests/queryset/test_queryset_aggregation.py diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 121325ae..07de0378 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -400,7 +400,7 @@ would be generating "tag-clouds":: MongoDB aggregation API ----------------------- -If you need to run aggregation pipelines, MongoEngine provides an entry point `Pymongo's aggregation framework `_ +If you need to run aggregation pipelines, MongoEngine provides an entry point to `Pymongo's aggregation framework `_ through :meth:`~mongoengine.queryset.QuerySet.aggregate`. Check out Pymongo's documentation for the syntax and pipeline. An example of its use would be:: @@ -414,7 +414,7 @@ An example of its use would be:: {"$sort" : {"name" : -1}}, {"$project": {"_id": 0, "name": {"$toUpper": "$name"}}} ] - data = Person.objects().aggregate(*pipeline) + data = Person.objects().aggregate(pipeline) assert data == [{'name': 'BOB'}, {'name': 'JOHN'}] Query efficiency and performance diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a648391e..aa5f2584 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1255,16 +1255,25 @@ class BaseQuerySet(object): for data in son_data ] - def aggregate(self, *pipeline, **kwargs): - """ - Perform a aggregate function based in your queryset params + def aggregate(self, pipeline, *suppl_pipeline, **kwargs): + """Perform a aggregate function based in your queryset params + :param pipeline: list of aggregation commands,\ see: http://docs.mongodb.org/manual/core/aggregation-pipeline/ - + :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface) + parameter will be removed shortly .. versionadded:: 0.9 """ - initial_pipeline = [] + using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline) + user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline) + if using_deprecated_interface: + msg = "Calling .aggregate() with un unpacked list (*pipeline) is deprecated, it will soon change and will expect a list (similar to pymongo.Collection.aggregate interface), see documentation" + warnings.warn(msg, DeprecationWarning) + + user_pipeline += suppl_pipeline + + initial_pipeline = [] if self._query: initial_pipeline.append({"$match": self._query}) @@ -1281,14 +1290,14 @@ class BaseQuerySet(object): if self._skip is not None: initial_pipeline.append({"$skip": self._skip}) - pipeline = initial_pipeline + list(pipeline) + final_pipeline = initial_pipeline + user_pipeline + collection = self._collection if self._read_preference is not None: - return self._collection.with_options( + collection = self._collection.with_options( read_preference=self._read_preference - ).aggregate(pipeline, cursor={}, **kwargs) - - return self._collection.aggregate(pipeline, cursor={}, **kwargs) + ) + return collection.aggregate(final_pipeline, cursor={}, **kwargs) # JS functionality def map_reduce( diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 7812ab66..b30350e6 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -14,7 +14,7 @@ import six from six import iteritems from mongoengine import * -from mongoengine.connection import get_connection, get_db +from mongoengine.connection import get_db from mongoengine.context_managers import query_counter, switch_db from mongoengine.errors import InvalidQueryError from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version @@ -4658,21 +4658,6 @@ class TestQueryset(unittest.TestCase): ) assert_read_pref(bars, ReadPreference.SECONDARY_PREFERRED) - def test_read_preference_aggregation_framework(self): - class Bar(Document): - txt = StringField() - - meta = {"indexes": ["txt"]} - - # Aggregates with read_preference - bars = Bar.objects.read_preference( - ReadPreference.SECONDARY_PREFERRED - ).aggregate() - assert ( - bars._CommandCursor__collection.read_preference - == ReadPreference.SECONDARY_PREFERRED - ) - def test_json_simple(self): class Embedded(EmbeddedDocument): string = StringField() @@ -5399,225 +5384,6 @@ class TestQueryset(unittest.TestCase): assert Person.objects.first().name == "A" assert Person.objects._has_data(), "Cursor has data and returned False" - def test_queryset_aggregation_framework(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects(age__lte=22).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - ] - - data = ( - Person.objects(age__lte=22) - .order_by("-name") - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [ - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - ] - - data = ( - Person.objects(age__gte=17, age__lte=40) - .order_by("-age") - .aggregate( - {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}} - ) - ) - assert list(data) == [{"_id": None, "avg": 29, "total": 2}] - - data = Person.objects().aggregate({"$match": {"name": "Isabella Luanna"}}) - assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}] - - def test_queryset_aggregation_with_skip(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.skip(1).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - ] - - def test_queryset_aggregation_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.limit(1).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] - - def test_queryset_aggregation_with_sort(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.order_by("name").aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - ] - - def test_queryset_aggregation_with_skip_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = list( - Person.objects.skip(1) - .limit(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] - - # Make sure limit/skip chaining order has no impact - data2 = ( - Person.objects.limit(1) - .skip(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert data == list(data2) - - def test_queryset_aggregation_with_sort_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - ] - - # Verify adding limit/skip steps works as expected - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}) - ) - - assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] - - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate( - {"$project": {"name": {"$toUpper": "$name"}}}, - {"$skip": 1}, - {"$limit": 1}, - ) - ) - - assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] - - def test_queryset_aggregation_with_sort_with_skip(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .skip(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] - - def test_queryset_aggregation_with_sort_with_skip_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .skip(1) - .limit(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] - def test_delete_count(self): [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)] assert ( diff --git a/tests/queryset/test_queryset_aggregation.py b/tests/queryset/test_queryset_aggregation.py new file mode 100644 index 00000000..00e04a36 --- /dev/null +++ b/tests/queryset/test_queryset_aggregation.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +import unittest +import warnings + +from pymongo.read_preferences import ReadPreference + +from mongoengine import * +from tests.utils import MongoDBTestCase + + +class TestQuerysetAggregate(MongoDBTestCase): + def test_read_preference_aggregation_framework(self): + class Bar(Document): + txt = StringField() + + meta = {"indexes": ["txt"]} + + # Aggregates with read_preference + pipeline = [] + bars = Bar.objects.read_preference( + ReadPreference.SECONDARY_PREFERRED + ).aggregate(pipeline) + assert ( + bars._CommandCursor__collection.read_preference + == ReadPreference.SECONDARY_PREFERRED + ) + + def test_queryset_aggregation_framework(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects(age__lte=22).aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + ] + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects(age__lte=22).order_by("-name").aggregate(pipeline) + + assert list(data) == [ + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + ] + + pipeline = [ + {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}} + ] + data = ( + Person.objects(age__gte=17, age__lte=40) + .order_by("-age") + .aggregate(pipeline) + ) + assert list(data) == [{"_id": None, "avg": 29, "total": 2}] + + pipeline = [{"$match": {"name": "Isabella Luanna"}}] + data = Person.objects().aggregate(pipeline) + assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}] + + def test_queryset_aggregation_with_skip(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.skip(1).aggregate(pipeline) + + assert list(data) == [ + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + def test_queryset_aggregation_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.limit(1).aggregate(pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + def test_queryset_aggregation_with_sort(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + ] + + def test_queryset_aggregation_with_skip_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = list(Person.objects.skip(1).limit(1).aggregate(pipeline)) + + assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] + + # Make sure limit/skip chaining order has no impact + data2 = Person.objects.limit(1).skip(1).aggregate(pipeline) + + assert data == list(data2) + + def test_queryset_aggregation_with_sort_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + # Verify adding limit/skip steps works as expected + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + pipeline = [ + {"$project": {"name": {"$toUpper": "$name"}}}, + {"$skip": 1}, + {"$limit": 1}, + ] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] + + def test_queryset_aggregation_with_sort_with_skip(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").skip(2).aggregate(pipeline) + + assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] + + def test_queryset_aggregation_with_sort_with_skip_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").skip(1).limit(1).aggregate(pipeline) + + assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] + + def test_queryset_aggregation_deprecated_interface(self): + class Person(Document): + name = StringField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna") + p2 = Person(name="Wilson Junior") + p3 = Person(name="Sandra Mara") + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + + # Make sure a warning is emitted + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with self.assertRaises(DeprecationWarning): + Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + # Make sure old interface works as expected with a 1-step pipeline + data = Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + # Make sure old interface works as expected with a 2-steps pipeline + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}] + data = Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + +if __name__ == "__main__": + unittest.main() From 99e660c66d85620b427f2ab48eba557a6faa5a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 27 Dec 2019 09:32:05 +0100 Subject: [PATCH 31/58] update changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 06eb8d0c..5fe7d6b4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of + pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 Changes in 0.19.0 ================= From b3dbb87c3c394cab5708bd321f58932d0c6b1063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 27 Dec 2019 10:06:27 +0100 Subject: [PATCH 32/58] improve doc of aggregate kwargs --- mongoengine/queryset/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index aa5f2584..50cb37ac 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -302,7 +302,7 @@ class BaseQuerySet(object): ``insert(..., {w: 2, fsync: True})`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. - :parm signal_kwargs: (optional) kwargs dictionary to be passed to + :param signal_kwargs: (optional) kwargs dictionary to be passed to the signal calls. By default returns document instances, set ``load_bulk`` to False to @@ -1262,6 +1262,8 @@ class BaseQuerySet(object): see: http://docs.mongodb.org/manual/core/aggregation-pipeline/ :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface) parameter will be removed shortly + :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call + See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate .. versionadded:: 0.9 """ using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline) From e7c7a66cd1d32fb1aa4595e3d8935c57de4beab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 27 Dec 2019 10:38:03 +0100 Subject: [PATCH 33/58] improve doc of GridFS, emphasize that subsequent call to read() requires to rewind the file with seek(0) --- docs/guide/gridfs.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index f7380e89..6e4a75a5 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -10,8 +10,9 @@ Writing GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field object. This field acts as a file-like object and provides a couple of different ways of inserting and retrieving data. Arbitrary metadata such as -content type can also be stored alongside the files. In the following example, -a document is created to store details about animals, including a photo:: +content type can also be stored alongside the files. The object returned when accessing a +FileField is a proxy to `Pymongo's GridFS `_ +In the following example, a document is created to store details about animals, including a photo:: class Animal(Document): genus = StringField() @@ -34,6 +35,20 @@ field. The file can also be retrieved just as easily:: photo = marmot.photo.read() content_type = marmot.photo.content_type +.. note:: If you need to read() the content of a file multiple times, you'll need to "rewind" + the file-like object using `seek`:: + + marmot = Animal.objects(genus='Marmota').first() + content1 = marmot.photo.read() + assert content1 != "" + + content2 = marmot.photo.read() # will be empty + assert content2 == "" + + marmot.photo.seek(0) # rewind the file by setting the current position of the cursor in the file to 0 + content3 = marmot.photo.read() + assert content3 == content1 + Streaming --------- From 152b51fd339a1bb9db254555461b5e6651bc3a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 29 Dec 2019 14:36:50 +0100 Subject: [PATCH 34/58] improve gridfs example (properly opening file) --- docs/guide/gridfs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 6e4a75a5..0baf88e0 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -21,8 +21,8 @@ In the following example, a document is created to store details about animals, marmot = Animal(genus='Marmota', family='Sciuridae') - marmot_photo = open('marmot.jpg', 'rb') - marmot.photo.put(marmot_photo, content_type = 'image/jpeg') + with open('marmot.jpg', 'rb') as fd: + marmot.photo.put(fd, content_type = 'image/jpeg') marmot.save() Retrieval From 4edad4601c7ccf30120cb8e45df7e1fb700a9347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 3 Jan 2020 14:03:17 +0100 Subject: [PATCH 35/58] Bump version to 0.19.1 + force pillow to be < 7.0.0 --- docs/changelog.rst | 4 ++++ mongoengine/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5fe7d6b4..b8e6ae56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,10 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). + +Changes in 0.19.1 +================= +- Requires Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index c41b5e70..e45dfc2b 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 19, 0) +VERSION = (0, 19, 1) def get_version(): diff --git a/setup.py b/setup.py index ceb5afad..2d69e44a 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ extra_opts = { "pytest-cov", "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", - "Pillow>=2.0.0", + "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support ], } if sys.version_info[0] == 3: From 75ee282a3dfc90ab2c6a4194e8f2851ce7f4543d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 3 Jan 2020 14:15:24 +0100 Subject: [PATCH 36/58] black setup.py to please CI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d69e44a..6c3ef8db 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ extra_opts = { "pytest-cov", "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", - "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support + "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support ], } if sys.version_info[0] == 3: From f8f267a880bcb9b17a51df61b006e7d9b92644b3 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 5 Jan 2020 20:48:16 +1100 Subject: [PATCH 37/58] Fix simple typo: thorougly -> thoroughly Closes #2236 --- docs/upgrade.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 082dbadc..250347bf 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -52,7 +52,7 @@ rename its occurrences. This release includes a major rehaul of MongoEngine's code quality and introduces a few breaking changes. It also touches many different parts of the package and although all the changes have been tested and scrutinized, -you're encouraged to thorougly test the upgrade. +you're encouraged to thoroughly test the upgrade. First breaking change involves renaming `ConnectionError` to `MongoEngineConnectionError`. If you import or catch this exception, you'll need to rename it in your code. From 59fbd505a04dd8cf822a098b834fdd8829395fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 5 Jan 2020 20:20:13 +0100 Subject: [PATCH 38/58] include latest pymongo version in travis --- .travis.yml | 7 +++++-- setup.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 809fbad8..a299eea9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,11 +33,12 @@ env: global: - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 + - PYMONGO_3_10=3.10 - PYMONGO_3_9=3.9 - PYMONGO_3_6=3.6 - PYMONGO_3_4=3.4 matrix: - - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9} + - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} matrix: @@ -51,6 +52,8 @@ matrix: env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} + - python: 3.7 + env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10} install: # Install Mongo @@ -110,5 +113,5 @@ deploy: on: tags: true repo: MongoEngine/mongoengine - condition: ($PYMONGO = ${PYMONGO_3_9}) && ($MONGODB = ${MONGODB_3_4}) + condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4}) python: 2.7 diff --git a/setup.py b/setup.py index 6c3ef8db..5ba84e06 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=["pymongo>=3.4", "six>=1.10.0"], + install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0"], cmdclass={"test": PyTest}, **extra_opts ) From 705c55ce24c006dd2bd06437f3d7fb945ed1de17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 5 Jan 2020 20:30:56 +0100 Subject: [PATCH 39/58] update tox file to account for mg310 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 349b5577..a3d2df60 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ commands = deps = mg34: pymongo>=3.4,<3.5 mg36: pymongo>=3.6,<3.7 - mg39: pymongo>=3.9,<4.0 + mg39: pymongo>=3.9,<3.10 + mg310: pymongo>=3.10,<3.11 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs From d73846213977b2fb1e382b38d86406bdf9a7e81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 11 Jan 2020 23:15:30 +0100 Subject: [PATCH 40/58] Fix bug introduced in -1.19 related to DictField validate failing without default connection --- docs/changelog.rst | 5 ++++ mongoengine/fields.py | 14 +++++----- mongoengine/mongodb_support.py | 2 +- tests/fields/test_dict_field.py | 48 ++++++++++++++++++++------------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b8e6ae56..2b532da9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,11 @@ Development =========== - (Fill this out as you fix issues and develop your features). +Changes in 0.19.2 +================= +- DictField validate failed without default connection (bug introduced in 0.19.0) #2239 + + Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7ec8c0f3..d502dba3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1088,14 +1088,12 @@ class DictField(ComplexBaseField): msg = "Invalid dictionary key - documents must have only string keys" self.error(msg) - curr_mongo_ver = get_mongodb_version() - - if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value): - self.error( - 'Invalid dictionary key name - keys may not contain "."' - ' or startswith "$" characters' - ) - elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value): + # Following condition applies to MongoDB >= 3.6 + # older Mongo has stricter constraints but + # it will be rejected upon insertion anyway + # Having a validation that depends on the MongoDB version + # is not straightforward as the field isn't aware of the connected Mongo + if key_starts_with_dollar(value): self.error( 'Invalid dictionary key name - keys may not startswith "$" characters' ) diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 5d437fef..522f064e 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -11,7 +11,7 @@ MONGODB_36 = (3, 6) def get_mongodb_version(): - """Return the version of the connected mongoDB (first 2 digits) + """Return the version of the default connected mongoDB (first 2 digits) :return: tuple(int, int) """ diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index 44e628f6..6850cd58 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import pytest +from bson import InvalidDocument from mongoengine import * from mongoengine.base import BaseDict @@ -19,22 +20,24 @@ class TestDictField(MongoDBTestCase): post = BlogPost(info=info).save() assert get_as_pymongo(post) == {"_id": post.id, "info": info} - def test_general_things(self): - """Ensure that dict types work as expected.""" + def test_validate_invalid_type(self): + class BlogPost(Document): + info = DictField() + BlogPost.drop_collection() + + invalid_infos = ["my post", ["test", "test"], {1: "test"}] + for invalid_info in invalid_infos: + with pytest.raises(ValidationError): + BlogPost(info=invalid_info).validate() + + def test_keys_with_dots_or_dollars(self): class BlogPost(Document): info = DictField() BlogPost.drop_collection() post = BlogPost() - post.info = "my post" - with pytest.raises(ValidationError): - post.validate() - - post.info = ["test", "test"] - with pytest.raises(ValidationError): - post.validate() post.info = {"$title": "test"} with pytest.raises(ValidationError): @@ -48,25 +51,34 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() - post.info = {1: "test"} - with pytest.raises(ValidationError): - post.validate() - post.info = {"nested": {"the.title": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + # MongoDB < 3.6 rejects dots + # To avoid checking the mongodb version from the DictField class + # we rely on MongoDB to reject the data during the save + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() post.info = {"dollar_and_dot": {"te$st.test": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() - post.info = {"title": "test"} + def test_general_things(self): + """Ensure that dict types work as expected.""" + + class BlogPost(Document): + info = DictField() + + BlogPost.drop_collection() + + post = BlogPost(info={"title": "test"}) post.save() post = BlogPost() From 18b68f1b8034457384daa5bdbd7d780055e21690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:29:18 +0100 Subject: [PATCH 41/58] update travis mongo 4.0 to latest 4.0.13 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f7880649..a7d6da1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ dist: xenial env: global: - - MONGODB_4_0=4.0.12 + - MONGODB_4_0=4.0.13 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 - PYMONGO_3_10=3.10 From e0565ddac5cd62f9fd05c69187b49a20bb644035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:31:28 +0100 Subject: [PATCH 42/58] update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b8e6ae56..d924a2c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- Add Mongo 4.0 to Travis Changes in 0.19.1 ================= From 605de59bd08d9d4f54f695c7f73a1458975d479f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:37:32 +0100 Subject: [PATCH 43/58] improve travis + fix tox mg310 --- .travis.yml | 9 +++++---- tox.ini | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7d6da1a..62bbacb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,13 +31,14 @@ dist: xenial env: global: - - MONGODB_4_0=4.0.13 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 - - PYMONGO_3_10=3.10 - - PYMONGO_3_9=3.9 - - PYMONGO_3_6=3.6 + - MONGODB_4_0=4.0.13 + - PYMONGO_3_4=3.4 + - PYMONGO_3_6=3.6 + - PYMONGO_3_9=3.9 + - PYMONGO_3_10=3.10 matrix: - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} diff --git a/tox.ini b/tox.ini index c3789b7d..396817ca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39} +envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310} [testenv] commands = From 86e965f854763863de2d12e54b4416b4761df55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 30 Dec 2019 21:58:57 +0100 Subject: [PATCH 44/58] remove very old deprecated method --- docs/changelog.rst | 3 ++- mongoengine/queryset/base.py | 20 -------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..b96a85ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,8 @@ Changes in 0.19.1 - Requires Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 - +- BREAKING CHANGE: Removed Queryset._ensure_indexes and Queryset.ensure_indexes that were deprecated in 2013. + Document.ensure_indexes still exists Changes in 0.19.0 ================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..805a3d0a 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1958,23 +1958,3 @@ class BaseQuerySet(object): setattr(queryset, "_" + method_name, val) return queryset - - # Deprecated - def ensure_index(self, **kwargs): - """Deprecated use :func:`Document.ensure_index`""" - msg = ( - "Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ( - "Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() From 095217e7977b933c53697e9736a6a8bcb37830cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 13 Jan 2020 23:53:24 +0100 Subject: [PATCH 45/58] remove methods that were derecated in 2013... --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b96a85ed..5fe34f91 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,14 +7,15 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- BREAKING CHANGE: Removed ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes`` that were deprecated in 2013. + ``Document.ensure_indexes`` still exists and is the right method to use Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 -- BREAKING CHANGE: Removed Queryset._ensure_indexes and Queryset.ensure_indexes that were deprecated in 2013. - Document.ensure_indexes still exists + Changes in 0.19.0 ================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 From 450658d7ac2b26be7849f8f0599fff06bb7e6ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 4 Feb 2020 22:51:02 +0100 Subject: [PATCH 46/58] fix indirect library version that dropped python2 support recently --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5ba84e06..5cba5d9e 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,10 @@ CLASSIFIERS = [ "Topic :: Software Development :: Libraries :: Python Modules", ] +PYTHON_VERSION = sys.version_info[0] +PY3 = PYTHON_VERSION == 3 +PY2 = PYTHON_VERSION == 2 + extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), "tests_require": [ @@ -116,9 +120,10 @@ extra_opts = { "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support + "zipp<2.0.0", # (dependency of pytest) dropped python2 support ], } -if sys.version_info[0] == 3: +if PY3: extra_opts["use_2to3"] = True if "test" in sys.argv: extra_opts["packages"] = find_packages() From 4bca3de42f08a5ca4fdd0b4021d6a54f25e24003 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Fri, 14 Feb 2020 16:43:07 -0300 Subject: [PATCH 47/58] Add support for the elemMatch projection operator. Add basic tests to the fields queryset method. --- mongoengine/queryset/base.py | 2 +- tests/queryset/test_queryset.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..710259df 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1037,7 +1037,7 @@ class BaseQuerySet(object): """ # Check for an operator and transform to mongo-style if there is - operators = ["slice"] + operators = ["slice", "elemMatch"] cleaned_fields = [] for key, value in kwargs.items(): parts = key.split("__") diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index b30350e6..5ebd545f 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4476,6 +4476,57 @@ class TestQueryset(unittest.TestCase): expected = "[u'A1', u'A2']" assert expected == "%s" % sorted(names) + def test_fields(self): + class Bar(EmbeddedDocument): + v = StringField() + z = StringField() + + + class Foo(Document): + x = StringField() + y = IntField() + items = EmbeddedDocumentListField(Bar) + + + Foo.drop_collection() + + Foo(x='foo1', y=1).save() + Foo(x='foo2', y=2, items=[]).save() + Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() + Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() + Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + + foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + + assert all(o.x is not None for o in foos_with_x) + + foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + + assert all(o.y is None for o in foos_with_x) + + foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + + assert foos_with_sliced_items[0].items == [] + assert foos_with_sliced_items[1].items == [] + assert len(foos_with_sliced_items[2].items) == 1 + assert foos_with_sliced_items[2].items[0].z == 'a' + assert len(foos_with_sliced_items[3].items) == 1 + assert foos_with_sliced_items[3].items[0].z == 'a' + assert len(foos_with_sliced_items[4].items) == 1 + assert foos_with_sliced_items[4].items[0].z == 'b' + + foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + + assert foos_with_elem_match_items[0].items == [] + assert foos_with_elem_match_items[1].items == [] + assert foos_with_elem_match_items[2].items == [] + assert len(foos_with_elem_match_items[3].items) == 1 + assert foos_with_elem_match_items[3].items[0].z == 'b' + assert foos_with_elem_match_items[3].items[0].v == 'W' + assert len(foos_with_elem_match_items[4].items) == 1 + assert foos_with_elem_match_items[4].items[0].z == 'b' + + def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() From 81f9b351b3838fe27924937e706300805e9eb82b Mon Sep 17 00:00:00 2001 From: Leonardo Domingues Date: Fri, 21 Feb 2020 19:14:34 -0300 Subject: [PATCH 48/58] Add return info in the save function docstring --- AUTHORS | 1 + mongoengine/document.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b9a81c63..7d3000ce 100644 --- a/AUTHORS +++ b/AUTHORS @@ -255,3 +255,4 @@ that much better: * Filip Kucharczyk (https://github.com/Pacu2) * Eric Timmons (https://github.com/daewok) * Matthew Simpson (https://github.com/mcsimps2) + * Leonardo Domingues (https://github.com/leodmgs) diff --git a/mongoengine/document.py b/mongoengine/document.py index 23968f17..5e812510 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -332,7 +332,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): ): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be - created. + created. Returns the saved object instance. :param force_insert: only try to create a new document, don't allow updates of existing documents. From cfb4943986189a22ae75fd4f378f38f7ec0a77bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 2 Mar 2020 22:49:21 +0100 Subject: [PATCH 49/58] reformat with black --- tests/queryset/test_queryset.py | 53 ++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 5ebd545f..f6d1a916 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4481,51 +4481,68 @@ class TestQueryset(unittest.TestCase): v = StringField() z = StringField() - class Foo(Document): x = StringField() y = IntField() items = EmbeddedDocumentListField(Bar) - Foo.drop_collection() - Foo(x='foo1', y=1).save() - Foo(x='foo2', y=2, items=[]).save() - Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() - Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() - Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + Foo(x="foo1", y=1).save() + Foo(x="foo2", y=2, items=[]).save() + Foo(x="foo3", y=3, items=[Bar(z="a", v="V")]).save() + Foo( + x="foo4", + y=4, + items=[ + Bar(z="a", v="V"), + Bar(z="b", v="W"), + Bar(z="b", v="X"), + Bar(z="c", v="V"), + ], + ).save() + Foo( + x="foo5", + y=5, + items=[ + Bar(z="b", v="X"), + Bar(z="c", v="V"), + Bar(z="d", v="V"), + Bar(z="e", v="V"), + ], + ).save() - foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + foos_with_x = list(Foo.objects.order_by("y").fields(x=1)) assert all(o.x is not None for o in foos_with_x) - foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + foos_without_y = list(Foo.objects.order_by("y").fields(y=0)) assert all(o.y is None for o in foos_with_x) - foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1)) assert foos_with_sliced_items[0].items == [] assert foos_with_sliced_items[1].items == [] assert len(foos_with_sliced_items[2].items) == 1 - assert foos_with_sliced_items[2].items[0].z == 'a' + assert foos_with_sliced_items[2].items[0].z == "a" assert len(foos_with_sliced_items[3].items) == 1 - assert foos_with_sliced_items[3].items[0].z == 'a' + assert foos_with_sliced_items[3].items[0].z == "a" assert len(foos_with_sliced_items[4].items) == 1 - assert foos_with_sliced_items[4].items[0].z == 'b' + assert foos_with_sliced_items[4].items[0].z == "b" - foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + foos_with_elem_match_items = list( + Foo.objects.order_by("y").fields(elemMatch__items={"z": "b"}) + ) assert foos_with_elem_match_items[0].items == [] assert foos_with_elem_match_items[1].items == [] assert foos_with_elem_match_items[2].items == [] assert len(foos_with_elem_match_items[3].items) == 1 - assert foos_with_elem_match_items[3].items[0].z == 'b' - assert foos_with_elem_match_items[3].items[0].v == 'W' + assert foos_with_elem_match_items[3].items[0].z == "b" + assert foos_with_elem_match_items[3].items[0].v == "W" assert len(foos_with_elem_match_items[4].items) == 1 - assert foos_with_elem_match_items[4].items[0].z == 'b' - + assert foos_with_elem_match_items[4].items[0].z == "b" def test_elem_match(self): class Foo(EmbeddedDocument): From d287f480e5016090384650515c9342c767c4bbb7 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Tue, 4 Feb 2020 12:35:03 +0100 Subject: [PATCH 50/58] Fix for combining raw and regular filters --- mongoengine/queryset/transform.py | 4 ++-- tests/queryset/test_transform.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 0b73e99b..659a97e2 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs): key = ".".join(parts) - if op is None or key not in mongo_query: + if key not in mongo_query: mongo_query[key] = value - elif key in mongo_query: + else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 3898809e..8d6c2d06 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -24,6 +24,12 @@ class TestTransform(unittest.TestCase): } assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}} assert transform.query(name__exists=True) == {"name": {"$exists": True}} + assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == { + "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}] + } + assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == { + "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}] + } def test_transform_update(self): class LisDoc(Document): From fda2e2b47ab5085b6ff5b6e27b40794a3fa9e77f Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Tue, 4 Feb 2020 12:58:25 +0100 Subject: [PATCH 51/58] Update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..8dcea62a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 Changes in 0.19.1 ================= From aa4a6ae0234be23f237f68c3061f9db28c274741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 21:02:44 +0100 Subject: [PATCH 52/58] Fix invalid escape seq in codebase --- mongoengine/document.py | 2 +- mongoengine/queryset/base.py | 6 +++--- tests/fields/test_complex_datetime_field.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 5e812510..3cc0046e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class InvalidCollectionError(Exception): class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): - """A :class:`~mongoengine.Document` that isn't stored in its own + r"""A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..7941e970 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -694,8 +694,8 @@ class BaseQuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - :param object_ids: a list or tuple of ``ObjectId``\ s - :rtype: dict of ObjectIds as keys and collection-specific + :param object_ids: a list or tuple of ObjectId's + :rtype: dict of ObjectId's as keys and collection-specific Document subclasses as values. .. versionadded:: 0.3 @@ -1140,7 +1140,7 @@ class BaseQuerySet(object): def explain(self): """Return an explain plan record for the - :class:`~mongoengine.queryset.QuerySet`\ 's cursor. + :class:`~mongoengine.queryset.QuerySet` cursor. """ return self._cursor.explain() diff --git a/tests/fields/test_complex_datetime_field.py b/tests/fields/test_complex_datetime_field.py index f0a6b96e..5bd6c56b 100644 --- a/tests/fields/test_complex_datetime_field.py +++ b/tests/fields/test_complex_datetime_field.py @@ -65,7 +65,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond): stored = LogEntry(date=datetime.datetime(*values)).to_mongo()["date"] assert ( - re.match("^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) + re.match(r"^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) is not None ) @@ -74,7 +74,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): "date_with_dots" ] assert ( - re.match("^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None + re.match(r"^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None ) def test_complexdatetime_usage(self): From c0c0efce188b32f4e59da89ead845a17b824b236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 22:14:26 +0100 Subject: [PATCH 53/58] improve docstring related to #2267 and document the change in the changelog --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dcea62a..41ff8c85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,7 @@ Development - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 +- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 Changes in 0.19.1 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 9671a3dc..fbf0a1ba 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1026,9 +1026,11 @@ class BaseQuerySet(object): posts = BlogPost.objects(...).fields(comments=0) - To retrieve a subrange of array elements: + To retrieve a subrange or sublist of array elements, + support exist for both the `slice` and `elemMatch` projection operator: posts = BlogPost.objects(...).fields(slice__comments=5) + posts = BlogPost.objects(...).fields(elemMatch__comments="test") :param kwargs: A set of keyword arguments identifying what to include, exclude, or slice. From ad0669a32605bb7d9984b1830349d82c208b3c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 16 Mar 2020 22:39:31 +0100 Subject: [PATCH 54/58] update changelog --- docs/changelog.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7f68bbe5..1bfb190d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,13 +8,9 @@ Development - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 -- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - -Changes in 0.19.2 -================= +- Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 - Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support From aadc6262edc5627bafbef41a7c814fedbe66fdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:10:52 +0100 Subject: [PATCH 55/58] remove qs.slave_okay() that is deprecated since pymongo3 --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1bfb190d..b2090645 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 +- Remove method queryset.slave_okay() that was deprecated a while ago and disappeared since pymongo3 Changes in 0.19.1 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index fbf0a1ba..95606f2a 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -60,7 +60,6 @@ class BaseQuerySet(object): self._ordering = None self._snapshot = False self._timeout = True - self._slave_okay = False self._read_preference = None self._iter = False self._scalar = [] @@ -775,7 +774,6 @@ class BaseQuerySet(object): "_ordering", "_snapshot", "_timeout", - "_slave_okay", "_read_preference", "_iter", "_scalar", @@ -1172,20 +1170,6 @@ class BaseQuerySet(object): queryset._timeout = enabled return queryset - # DEPRECATED. Has no more impact on PyMongo 3+ - def slave_okay(self, enabled): - """Enable or disable the slave_okay when querying. - - :param enabled: whether or not the slave_okay is enabled - - .. deprecated:: Ignored with PyMongo 3+ - """ - msg = "slave_okay is deprecated as it has no impact when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) - queryset = self.clone() - queryset._slave_okay = enabled - return queryset - def read_preference(self, read_preference): """Change the read_preference when querying. From 8eb51790b50858d351068c3f37316d16143d3b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:26:41 +0100 Subject: [PATCH 56/58] Remove Field(name='...') which was deprecated when db_field was introduced a while ago --- docs/changelog.rst | 1 + mongoengine/base/fields.py | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1bfb190d..32cfe647 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 +- Remove name parameter in Field constructor e.g `StringField(name="...")`, it was deprecated a while ago in favor of db_field Changes in 0.19.1 ================= diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index cd1039cb..379098e5 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -36,7 +36,6 @@ class BaseField(object): def __init__( self, db_field=None, - name=None, required=False, default=None, unique=False, @@ -51,7 +50,6 @@ class BaseField(object): """ :param db_field: The database field to store this field in (defaults to the name of the field) - :param name: Deprecated - use db_field :param required: If the field is required. Whether it has to have a value or not. Defaults to False. :param default: (optional) The default value for this field if no value @@ -75,11 +73,8 @@ class BaseField(object): existing attributes. Common metadata includes `verbose_name` and `help_text`. """ - self.db_field = (db_field or name) if not primary_key else "_id" + self.db_field = db_field if not primary_key else "_id" - if name: - msg = 'Field\'s "name" attribute deprecated in favour of "db_field"' - warnings.warn(msg, DeprecationWarning) self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) From ee2d50b2d1b3549887512bfaefdb1abff3b831fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:38:50 +0100 Subject: [PATCH 57/58] remove drop_dups index option, deprecated with MongoDB3 --- docs/changelog.rst | 6 ++++-- docs/guide/defining-documents.rst | 6 ------ mongoengine/base/metaclasses.py | 1 - mongoengine/document.py | 15 +-------------- tests/document/test_indexes.py | 16 ---------------- tests/document/test_inheritance.py | 1 - 6 files changed, 5 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0784050..1037ca13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,8 +10,10 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 -- Remove name parameter in Field constructor e.g `StringField(name="...")`, it was deprecated a while ago in favor of db_field -- Remove method queryset.slave_okay() that was deprecated a while ago and disappeared since pymongo3 +- Remove methods deprecated years ago: + - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field + - Queryset.slave_okay() was deprecated since pymongo3 + - dropDups was dropped with MongoDB3 Changes in 0.19.1 ================= diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index bd2b43e2..6dc35c30 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -555,7 +555,6 @@ There are a few top level defaults for all indexes that can be set:: 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'index_drop_dups': True, } @@ -574,11 +573,6 @@ There are a few top level defaults for all indexes that can be set:: in systems where indexes are managed separately. Disabling this will improve performance. -:attr:`index_drop_dups` (Optional) - Set the default value for if an index should drop duplicates - Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning - and has no effect - Compound Indexes and Indexing sub documents ------------------------------------------- diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index e4d26811..3bba796b 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -284,7 +284,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): "indexes": [], # indexes to be ensured at runtime "id_field": None, "index_background": False, - "index_drop_dups": False, "index_opts": None, "delete_rules": None, # allow_inheritance can be True, False, and None. True means diff --git a/mongoengine/document.py b/mongoengine/document.py index 3cc0046e..c8710fb5 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -851,17 +851,13 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): index_spec = cls._build_index_spec(keys) index_spec = index_spec.copy() fields = index_spec.pop("fields") - drop_dups = kwargs.get("drop_dups", False) - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) index_spec["background"] = background index_spec.update(kwargs) return cls._get_collection().create_index(fields, **index_spec) @classmethod - def ensure_index(cls, key_or_list, drop_dups=False, background=False, **kwargs): + def ensure_index(cls, key_or_list, background=False, **kwargs): """Ensure that the given indexes are in place. Deprecated in favour of create_index. @@ -869,12 +865,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering :param background: Allows index creation in the background - :param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value - will be removed if PyMongo3+ is used """ - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) return cls.create_index(key_or_list, background=background, **kwargs) @classmethod @@ -887,12 +878,8 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): `auto_create_index` to False in the documents meta data """ background = cls._meta.get("index_background", False) - drop_dups = cls._meta.get("index_drop_dups", False) index_opts = cls._meta.get("index_opts") or {} index_cls = cls._meta.get("index_cls", True) - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) collection = cls._get_collection() # 746: when connection is via mongos, the read preference is not necessarily an indication that diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index be857b59..b08306a0 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -806,18 +806,6 @@ class TestIndexes(unittest.TestCase): info = Log.objects._collection.index_information() assert 3600 == info["created_1"]["expireAfterSeconds"] - def test_index_drop_dups_silently_ignored(self): - class Customer(Document): - cust_id = IntField(unique=True, required=True) - meta = { - "indexes": ["cust_id"], - "index_drop_dups": True, - "allow_inheritance": False, - } - - Customer.drop_collection() - Customer.objects.first() - def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by meta.indexes. @@ -1058,10 +1046,6 @@ class TestIndexes(unittest.TestCase): del index_info[key][ "ns" ] # drop the index namespace - we don't care about that here, MongoDB 3+ - if "dropDups" in index_info[key]: - del index_info[key][ - "dropDups" - ] # drop the index dropDups - it is deprecated in MongoDB 3+ assert index_info == { "txt_1": {"key": [("txt", 1)], "background": False}, diff --git a/tests/document/test_inheritance.py b/tests/document/test_inheritance.py index 5072f841..d7bd0632 100644 --- a/tests/document/test_inheritance.py +++ b/tests/document/test_inheritance.py @@ -523,7 +523,6 @@ class TestInheritance(MongoDBTestCase): defaults = { "index_background": True, - "index_drop_dups": True, "index_opts": {"hello": "world"}, "allow_inheritance": True, "queryset_class": "QuerySet", From 476b07af6ebe26e0418d1ab8da9e8179794ddc4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 23:59:54 +0100 Subject: [PATCH 58/58] reformat changelog --- docs/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a844761..9b864b02 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,8 +7,6 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis -- BREAKING CHANGE: Removed ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes`` that were deprecated in 2013. - ``Document.ensure_indexes`` still exists and is the right method to use - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 @@ -16,6 +14,7 @@ Development - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field - Queryset.slave_okay() was deprecated since pymongo3 - dropDups was dropped with MongoDB3 + - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes`` Changes in 0.19.1 =================