From 4bca3de42f08a5ca4fdd0b4021d6a54f25e24003 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Fri, 14 Feb 2020 16:43:07 -0300 Subject: [PATCH 1/4] Add support for the elemMatch projection operator. Add basic tests to the fields queryset method. --- mongoengine/queryset/base.py | 2 +- tests/queryset/test_queryset.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..710259df 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1037,7 +1037,7 @@ class BaseQuerySet(object): """ # Check for an operator and transform to mongo-style if there is - operators = ["slice"] + operators = ["slice", "elemMatch"] cleaned_fields = [] for key, value in kwargs.items(): parts = key.split("__") diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index b30350e6..5ebd545f 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4476,6 +4476,57 @@ class TestQueryset(unittest.TestCase): expected = "[u'A1', u'A2']" assert expected == "%s" % sorted(names) + def test_fields(self): + class Bar(EmbeddedDocument): + v = StringField() + z = StringField() + + + class Foo(Document): + x = StringField() + y = IntField() + items = EmbeddedDocumentListField(Bar) + + + Foo.drop_collection() + + Foo(x='foo1', y=1).save() + Foo(x='foo2', y=2, items=[]).save() + Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() + Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() + Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + + foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + + assert all(o.x is not None for o in foos_with_x) + + foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + + assert all(o.y is None for o in foos_with_x) + + foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + + assert foos_with_sliced_items[0].items == [] + assert foos_with_sliced_items[1].items == [] + assert len(foos_with_sliced_items[2].items) == 1 + assert foos_with_sliced_items[2].items[0].z == 'a' + assert len(foos_with_sliced_items[3].items) == 1 + assert foos_with_sliced_items[3].items[0].z == 'a' + assert len(foos_with_sliced_items[4].items) == 1 + assert foos_with_sliced_items[4].items[0].z == 'b' + + foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + + assert foos_with_elem_match_items[0].items == [] + assert foos_with_elem_match_items[1].items == [] + assert foos_with_elem_match_items[2].items == [] + assert len(foos_with_elem_match_items[3].items) == 1 + assert foos_with_elem_match_items[3].items[0].z == 'b' + assert foos_with_elem_match_items[3].items[0].v == 'W' + assert len(foos_with_elem_match_items[4].items) == 1 + assert foos_with_elem_match_items[4].items[0].z == 'b' + + def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() From cfb4943986189a22ae75fd4f378f38f7ec0a77bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 2 Mar 2020 22:49:21 +0100 Subject: [PATCH 2/4] reformat with black --- tests/queryset/test_queryset.py | 53 ++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 5ebd545f..f6d1a916 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4481,51 +4481,68 @@ class TestQueryset(unittest.TestCase): v = StringField() z = StringField() - class Foo(Document): x = StringField() y = IntField() items = EmbeddedDocumentListField(Bar) - Foo.drop_collection() - Foo(x='foo1', y=1).save() - Foo(x='foo2', y=2, items=[]).save() - Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() - Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() - Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + Foo(x="foo1", y=1).save() + Foo(x="foo2", y=2, items=[]).save() + Foo(x="foo3", y=3, items=[Bar(z="a", v="V")]).save() + Foo( + x="foo4", + y=4, + items=[ + Bar(z="a", v="V"), + Bar(z="b", v="W"), + Bar(z="b", v="X"), + Bar(z="c", v="V"), + ], + ).save() + Foo( + x="foo5", + y=5, + items=[ + Bar(z="b", v="X"), + Bar(z="c", v="V"), + Bar(z="d", v="V"), + Bar(z="e", v="V"), + ], + ).save() - foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + foos_with_x = list(Foo.objects.order_by("y").fields(x=1)) assert all(o.x is not None for o in foos_with_x) - foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + foos_without_y = list(Foo.objects.order_by("y").fields(y=0)) assert all(o.y is None for o in foos_with_x) - foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1)) assert foos_with_sliced_items[0].items == [] assert foos_with_sliced_items[1].items == [] assert len(foos_with_sliced_items[2].items) == 1 - assert foos_with_sliced_items[2].items[0].z == 'a' + assert foos_with_sliced_items[2].items[0].z == "a" assert len(foos_with_sliced_items[3].items) == 1 - assert foos_with_sliced_items[3].items[0].z == 'a' + assert foos_with_sliced_items[3].items[0].z == "a" assert len(foos_with_sliced_items[4].items) == 1 - assert foos_with_sliced_items[4].items[0].z == 'b' + assert foos_with_sliced_items[4].items[0].z == "b" - foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + foos_with_elem_match_items = list( + Foo.objects.order_by("y").fields(elemMatch__items={"z": "b"}) + ) assert foos_with_elem_match_items[0].items == [] assert foos_with_elem_match_items[1].items == [] assert foos_with_elem_match_items[2].items == [] assert len(foos_with_elem_match_items[3].items) == 1 - assert foos_with_elem_match_items[3].items[0].z == 'b' - assert foos_with_elem_match_items[3].items[0].v == 'W' + assert foos_with_elem_match_items[3].items[0].z == "b" + assert foos_with_elem_match_items[3].items[0].v == "W" assert len(foos_with_elem_match_items[4].items) == 1 - assert foos_with_elem_match_items[4].items[0].z == 'b' - + assert foos_with_elem_match_items[4].items[0].z == "b" def test_elem_match(self): class Foo(EmbeddedDocument): From aa4a6ae0234be23f237f68c3061f9db28c274741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 21:02:44 +0100 Subject: [PATCH 3/4] Fix invalid escape seq in codebase --- mongoengine/document.py | 2 +- mongoengine/queryset/base.py | 6 +++--- tests/fields/test_complex_datetime_field.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 5e812510..3cc0046e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class InvalidCollectionError(Exception): class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): - """A :class:`~mongoengine.Document` that isn't stored in its own + r"""A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..7941e970 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -694,8 +694,8 @@ class BaseQuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - :param object_ids: a list or tuple of ``ObjectId``\ s - :rtype: dict of ObjectIds as keys and collection-specific + :param object_ids: a list or tuple of ObjectId's + :rtype: dict of ObjectId's as keys and collection-specific Document subclasses as values. .. versionadded:: 0.3 @@ -1140,7 +1140,7 @@ class BaseQuerySet(object): def explain(self): """Return an explain plan record for the - :class:`~mongoengine.queryset.QuerySet`\ 's cursor. + :class:`~mongoengine.queryset.QuerySet` cursor. """ return self._cursor.explain() diff --git a/tests/fields/test_complex_datetime_field.py b/tests/fields/test_complex_datetime_field.py index f0a6b96e..5bd6c56b 100644 --- a/tests/fields/test_complex_datetime_field.py +++ b/tests/fields/test_complex_datetime_field.py @@ -65,7 +65,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond): stored = LogEntry(date=datetime.datetime(*values)).to_mongo()["date"] assert ( - re.match("^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) + re.match(r"^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) is not None ) @@ -74,7 +74,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): "date_with_dots" ] assert ( - re.match("^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None + re.match(r"^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None ) def test_complexdatetime_usage(self): From c0c0efce188b32f4e59da89ead845a17b824b236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 22:14:26 +0100 Subject: [PATCH 4/4] improve docstring related to #2267 and document the change in the changelog --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dcea62a..41ff8c85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,7 @@ Development - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 +- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 Changes in 0.19.1 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 9671a3dc..fbf0a1ba 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1026,9 +1026,11 @@ class BaseQuerySet(object): posts = BlogPost.objects(...).fields(comments=0) - To retrieve a subrange of array elements: + To retrieve a subrange or sublist of array elements, + support exist for both the `slice` and `elemMatch` projection operator: posts = BlogPost.objects(...).fields(slice__comments=5) + posts = BlogPost.objects(...).fields(elemMatch__comments="test") :param kwargs: A set of keyword arguments identifying what to include, exclude, or slice.