From 8ef771912dbcd0cc56cfb7a1b5fefb6172518139 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Sun, 28 Feb 2021 14:07:15 +0100 Subject: [PATCH 01/11] fixing incompatibility with mongoengine aggregation to support mongo 4.4 --- mongoengine/mongodb_support.py | 1 + mongoengine/queryset/base.py | 9 +++------ tests/document/test_indexes.py | 5 ++++- tests/queryset/test_queryset.py | 18 +++++++++++++++--- tests/utils.py | 8 ++++++++ 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 522f064e..df51100d 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -8,6 +8,7 @@ from mongoengine.connection import get_connection # get_mongodb_version() MONGODB_34 = (3, 4) MONGODB_36 = (3, 6) +MONGODB_44 = (4, 4) def get_mongodb_version(): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index ae8cd407..47a5f733 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1355,21 +1355,18 @@ class BaseQuerySet: MapReduceDocument = _import_class("MapReduceDocument") - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - map_f_scope = {} if isinstance(map_f, Code): map_f_scope = map_f.scope map_f = str(map_f) - map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) + map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None) reduce_f_scope = {} if isinstance(reduce_f, Code): reduce_f_scope = reduce_f.scope reduce_f = str(reduce_f) reduce_f_code = queryset._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) + reduce_f = Code(reduce_f_code, reduce_f_scope or None) mr_args = {"query": queryset._query} @@ -1379,7 +1376,7 @@ class BaseQuerySet: finalize_f_scope = finalize_f.scope finalize_f = str(finalize_f) finalize_f_code = queryset._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) + finalize_f = Code(finalize_f_code, finalize_f_scope or None) mr_args["finalize"] = finalize_f if scope: diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index 55a56931..17643dd8 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -7,6 +7,7 @@ import pytest from mongoengine import * from mongoengine.connection import get_db +from mongoengine.mongodb_support import MONGODB_44, get_mongodb_version class TestIndexes(unittest.TestCase): @@ -452,9 +453,11 @@ class TestIndexes(unittest.TestCase): .get("stage") == "IXSCAN" ) + mongo_db = get_mongodb_version() + PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_44 else "PROJECTION_COVERED" assert ( query_plan.get("queryPlanner").get("winningPlan").get("stage") - == "PROJECTION" + == PROJECTION_STR ) query_plan = Test.objects(a=1).explain() diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 4d281c60..c346abde 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -21,7 +21,11 @@ from mongoengine.queryset import ( QuerySetManager, queryset_manager, ) -from tests.utils import requires_mongodb_gte_44 +from tests.utils import ( + requires_mongodb_gte_44, + requires_mongodb_lt_42, + requires_mongodb_lte_42, +) class db_ops_tracker(query_counter): @@ -1490,6 +1494,7 @@ class TestQueryset(unittest.TestCase): BlogPost.drop_collection() + @requires_mongodb_lt_42 def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js.""" @@ -1527,6 +1532,7 @@ class TestQueryset(unittest.TestCase): BlogPost.drop_collection() + @requires_mongodb_lt_42 def test_exec_js_field_sub(self): """Ensure that field substitutions occur properly in exec_js functions.""" @@ -3109,6 +3115,7 @@ class TestQueryset(unittest.TestCase): freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True) assert freq == {"CRB": 0.5, None: 0.5} + @requires_mongodb_lte_42 def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): name = StringField() @@ -3137,6 +3144,7 @@ class TestQueryset(unittest.TestCase): ot = Person.objects.item_frequencies("extra.tag", map_reduce=True) assert ot == {None: 1.0, "friend": 1.0} + @requires_mongodb_lte_42 def test_item_frequencies_with_0_values(self): class Test(Document): val = IntField() @@ -3151,6 +3159,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {0: 1} + @requires_mongodb_lte_42 def test_item_frequencies_with_False_values(self): class Test(Document): val = BooleanField() @@ -3165,6 +3174,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {False: 1} + @requires_mongodb_lte_42 def test_item_frequencies_normalize(self): class Test(Document): val = IntField() @@ -3551,7 +3561,8 @@ class TestQueryset(unittest.TestCase): Book.objects.create(title="The Stories", authors=[mark_twain, john_tolkien]) authors = Book.objects.distinct("authors") - assert authors == [mark_twain, john_tolkien] + authors_names = {author.name for author in authors} + assert authors_names == {mark_twain.name, john_tolkien.name} def test_distinct_ListField_EmbeddedDocumentField_EmbeddedDocumentField(self): class Continent(EmbeddedDocument): @@ -3588,7 +3599,8 @@ class TestQueryset(unittest.TestCase): assert country_list == [scotland, tibet] continent_list = Book.objects.distinct("authors.country.continent") - assert continent_list == [europe, asia] + continent_list_names = {c.continent_name for c in continent_list} + assert continent_list_names == {europe.continent_name, asia.continent_name} def test_distinct_ListField_ReferenceField(self): class Bar(Document): diff --git a/tests/utils.py b/tests/utils.py index adb0bdb4..19596afa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,6 +34,14 @@ def get_as_pymongo(doc): return doc.__class__.objects.as_pymongo().get(id=doc.id) +def requires_mongodb_lt_42(func): + return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt) + + +def requires_mongodb_lte_42(func): + return _decorated_with_ver_requirement(func, (4, 2), oper=operator.le) + + def requires_mongodb_gte_44(func): return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge) From b479bb7c6bdf4c8853555dad3f24a67d407db484 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 10:52:46 +0100 Subject: [PATCH 02/11] Fix tests for supporting Mongo4.4 for some reason results comes sorted differently in map reduce when no sort is specified --- tests/queryset/test_queryset.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index c346abde..a4dbabd5 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -2666,6 +2666,8 @@ class TestQueryset(unittest.TestCase): title = StringField(primary_key=True) tags = ListField(StringField()) + BlogPost.drop_collection() + post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"]) post2 = BlogPost(title="Post #2", tags=["django", "mongodb"]) post3 = BlogPost(title="Post #3", tags=["hitchcock films"]) @@ -2694,12 +2696,15 @@ class TestQueryset(unittest.TestCase): } """ - results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults") + results = BlogPost.objects.order_by("_id").map_reduce( + map_f, reduce_f, "myresults2" + ) results = list(results) - assert results[0].object == post1 - assert results[1].object == post2 - assert results[2].object == post3 + assert len(results) == 3 + assert results[0].object.id == post1.id + assert results[1].object.id == post2.id + assert results[2].object.id == post3.id BlogPost.drop_collection() @@ -2707,7 +2712,6 @@ class TestQueryset(unittest.TestCase): """ Test map/reduce custom output """ - register_connection("test2", "mongoenginetest2") class Family(Document): id = IntField(primary_key=True) @@ -2780,6 +2784,7 @@ class TestQueryset(unittest.TestCase): family.persons.push(person); family.totalAge += person.age; }); + family.persons.sort((a, b) => (a.age > b.age)) } }); @@ -2808,10 +2813,10 @@ class TestQueryset(unittest.TestCase): "_id": 1, "value": { "persons": [ - {"age": 21, "name": "Wilson Jr"}, - {"age": 45, "name": "Wilson Father"}, - {"age": 40, "name": "Eliana Costa"}, {"age": 17, "name": "Tayza Mariana"}, + {"age": 21, "name": "Wilson Jr"}, + {"age": 40, "name": "Eliana Costa"}, + {"age": 45, "name": "Wilson Father"}, ], "totalAge": 123, }, @@ -2821,9 +2826,9 @@ class TestQueryset(unittest.TestCase): "_id": 2, "value": { "persons": [ + {"age": 10, "name": "Igor Gabriel"}, {"age": 16, "name": "Isabella Luanna"}, {"age": 36, "name": "Sandra Mara"}, - {"age": 10, "name": "Igor Gabriel"}, ], "totalAge": 62, }, @@ -2833,8 +2838,8 @@ class TestQueryset(unittest.TestCase): "_id": 3, "value": { "persons": [ - {"age": 30, "name": "Arthur WA"}, {"age": 25, "name": "Paula Leonel"}, + {"age": 30, "name": "Arthur WA"}, ], "totalAge": 55, }, From 3b9a167022f73ae0bf9e21e3e1a2c5901e7564bd Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 10:57:18 +0100 Subject: [PATCH 03/11] Add Mongo 4.4 to ci --- .github/workflows/github-actions.yml | 11 ++++++----- docs/changelog.rst | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 70393ebc..02eeccd4 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -11,9 +11,10 @@ on: tags: - 'v[0-9]+\.[0-9]+\.[0-9]+*' env: - MONGODB_3_4: 3.4.19 MONGODB_3_6: 3.6.13 MONGODB_4_0: 4.0.13 + MONGODB_4_2: 4.2.12 + MONGODB_4_4: 4.4.4 PYMONGO_3_4: 3.4 PYMONGO_3_6: 3.6 @@ -47,14 +48,14 @@ jobs: MONGODB: [$MONGODB_4_0] PYMONGO: [$PYMONGO_3_11] include: - - python-version: 3.7 - MONGODB: $MONGODB_3_4 - PYMONGO: $PYMONGO_3_6 - python-version: 3.7 MONGODB: $MONGODB_3_6 PYMONGO: $PYMONGO_3_9 - python-version: 3.7 - MONGODB: $MONGODB_3_6 + MONGODB: MONGODB_4_2 + PYMONGO: $PYMONGO_3_6 + - python-version: 3.7 + MONGODB: $MONGODB_4_4 PYMONGO: $PYMONGO_3_11 steps: - uses: actions/checkout@v2 diff --git a/docs/changelog.rst b/docs/changelog.rst index 5898eb1f..aaf8f3cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 +- Add MongoDB 4.4 to CI Changes in 0.22.1 ================= From 0620ac5641b721e2205647d67718a1ae329d1bac Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 11:30:03 +0100 Subject: [PATCH 04/11] Fix mongo download link as convention changed in official repo with > 4.0 --- .github/workflows/github-actions.yml | 4 ++-- .github/workflows/install_mongo.sh | 7 +++++++ docs/changelog.rst | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 02eeccd4..4ee0fd37 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -11,8 +11,8 @@ on: tags: - 'v[0-9]+\.[0-9]+\.[0-9]+*' env: - MONGODB_3_6: 3.6.13 - MONGODB_4_0: 4.0.13 + MONGODB_3_6: 3.6.14 + MONGODB_4_0: 4.0.23 MONGODB_4_2: 4.2.12 MONGODB_4_4: 4.4.4 diff --git a/.github/workflows/install_mongo.sh b/.github/workflows/install_mongo.sh index a98440a8..5f486666 100644 --- a/.github/workflows/install_mongo.sh +++ b/.github/workflows/install_mongo.sh @@ -2,7 +2,14 @@ MONGODB=$1 +# Mongo > 4.0 follows different name convention for download links mongo_build=mongodb-linux-x86_64-${MONGODB} + +if [[ "$MONGODB" == *"4."* ]] && [[ ! "$MONGODB" == *"4.0"* ]]; then + echo "It's there." + mongo_build=mongodb-linux-x86_64-ubuntu2004-v${MONGODB}-latest +fi + wget http://fastdl.mongodb.org/linux/$mongo_build.tgz tar xzf $mongo_build.tgz ${PWD}/$mongo_build/bin/mongod --version diff --git a/docs/changelog.rst b/docs/changelog.rst index aaf8f3cc..d8f7dde7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 -- Add MongoDB 4.4 to CI +- Add MongoDB 4.2 and 4.4 to CI Changes in 0.22.1 ================= From f2442071681bc29957682348a3e01e83741c4f97 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 11:40:39 +0100 Subject: [PATCH 05/11] fix mongo download link for 4.2 4.4 explicitly --- .github/workflows/github-actions.yml | 6 +++--- .github/workflows/install_mongo.sh | 11 +++++++---- .github/workflows/start_mongo.sh | 3 ++- mongoengine/mongodb_support.py | 1 + tests/document/test_indexes.py | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 4ee0fd37..bfdfa3a4 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -13,8 +13,8 @@ on: env: MONGODB_3_6: 3.6.14 MONGODB_4_0: 4.0.23 - MONGODB_4_2: 4.2.12 - MONGODB_4_4: 4.4.4 + MONGODB_4_2: 4.2 + MONGODB_4_4: 4.4 PYMONGO_3_4: 3.4 PYMONGO_3_6: 3.6 @@ -52,7 +52,7 @@ jobs: MONGODB: $MONGODB_3_6 PYMONGO: $PYMONGO_3_9 - python-version: 3.7 - MONGODB: MONGODB_4_2 + MONGODB: $MONGODB_4_2 PYMONGO: $PYMONGO_3_6 - python-version: 3.7 MONGODB: $MONGODB_4_4 diff --git a/.github/workflows/install_mongo.sh b/.github/workflows/install_mongo.sh index 5f486666..136f5c20 100644 --- a/.github/workflows/install_mongo.sh +++ b/.github/workflows/install_mongo.sh @@ -5,11 +5,14 @@ MONGODB=$1 # Mongo > 4.0 follows different name convention for download links mongo_build=mongodb-linux-x86_64-${MONGODB} -if [[ "$MONGODB" == *"4."* ]] && [[ ! "$MONGODB" == *"4.0"* ]]; then - echo "It's there." - mongo_build=mongodb-linux-x86_64-ubuntu2004-v${MONGODB}-latest +if [[ "$MONGODB" == *"4.2"* ]]; then + mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest +elif [[ "$MONGODB" == *"4.4"* ]]; then + mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest fi wget http://fastdl.mongodb.org/linux/$mongo_build.tgz tar xzf $mongo_build.tgz -${PWD}/$mongo_build/bin/mongod --version + +mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*") +$mongodb_dir/bin/mongod --version diff --git a/.github/workflows/start_mongo.sh b/.github/workflows/start_mongo.sh index dc844dbd..800004c8 100644 --- a/.github/workflows/start_mongo.sh +++ b/.github/workflows/start_mongo.sh @@ -2,7 +2,8 @@ MONGODB=$1 -mongodb_dir=${PWD}/mongodb-linux-x86_64-${MONGODB} +mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*") + mkdir $mongodb_dir/data $mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork mongo --eval 'db.version();' # Make sure mongo is awake diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index df51100d..8c5b3e5d 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -8,6 +8,7 @@ from mongoengine.connection import get_connection # get_mongodb_version() MONGODB_34 = (3, 4) MONGODB_36 = (3, 6) +MONGODB_42 = (4, 2) MONGODB_44 = (4, 4) diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index 17643dd8..2de9307d 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -7,7 +7,7 @@ import pytest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.mongodb_support import MONGODB_44, get_mongodb_version +from mongoengine.mongodb_support import MONGODB_42, get_mongodb_version class TestIndexes(unittest.TestCase): @@ -454,7 +454,7 @@ class TestIndexes(unittest.TestCase): == "IXSCAN" ) mongo_db = get_mongodb_version() - PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_44 else "PROJECTION_COVERED" + PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED" assert ( query_plan.get("queryPlanner").get("winningPlan").get("stage") == PROJECTION_STR From b9b536133de9f777b6643527b2b5fbaa366d4efc Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Thu, 4 Mar 2021 00:10:08 +0100 Subject: [PATCH 06/11] fix test incompat for 4.2 --- tests/queryset/test_queryset.py | 9 ++++----- tests/utils.py | 4 ---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index a4dbabd5..01e6b568 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -24,7 +24,6 @@ from mongoengine.queryset import ( from tests.utils import ( requires_mongodb_gte_44, requires_mongodb_lt_42, - requires_mongodb_lte_42, ) @@ -3120,7 +3119,7 @@ class TestQueryset(unittest.TestCase): freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True) assert freq == {"CRB": 0.5, None: 0.5} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): name = StringField() @@ -3149,7 +3148,7 @@ class TestQueryset(unittest.TestCase): ot = Person.objects.item_frequencies("extra.tag", map_reduce=True) assert ot == {None: 1.0, "friend": 1.0} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_0_values(self): class Test(Document): val = IntField() @@ -3164,7 +3163,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {0: 1} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_False_values(self): class Test(Document): val = BooleanField() @@ -3179,7 +3178,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {False: 1} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_normalize(self): class Test(Document): val = IntField() diff --git a/tests/utils.py b/tests/utils.py index 19596afa..52f56980 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -38,10 +38,6 @@ def requires_mongodb_lt_42(func): return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt) -def requires_mongodb_lte_42(func): - return _decorated_with_ver_requirement(func, (4, 2), oper=operator.le) - - def requires_mongodb_gte_44(func): return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge) From 34d273015c98967a6cfd9f65bea5ef48a65f6a68 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Thu, 4 Mar 2021 22:46:29 +0100 Subject: [PATCH 07/11] update changelog for 0.23.0 release --- docs/changelog.rst | 5 +++++ mongoengine/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d8f7dde7..64bf2775 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,4 +1,5 @@ + ========= Changelog ========= @@ -6,8 +7,12 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). + +Changes in 0.23.0 +=========== - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 - Add MongoDB 4.2 and 4.4 to CI +- Add support for allowDiskUse on querysets #2468 Changes in 0.22.1 ================= diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 68346399..bf38d8b0 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 22, 1) +VERSION = (0, 23, 0) def get_version(): From 58a3c6de03954686595e5e58aa2d1dd00c8f4ddc Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Mon, 8 Mar 2021 00:08:10 +0100 Subject: [PATCH 08/11] Add _lazy_load_ref methods so that the impact of lazy loading surfaces when profiling --- mongoengine/base/fields.py | 25 ++++++++++----- mongoengine/fields.py | 62 +++++++++++++++++++++----------------- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index ca01d00d..f1bc020c 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -267,6 +267,17 @@ class ComplexBaseField(BaseField): self.field = field super().__init__(**kwargs) + @staticmethod + def _lazy_load_refs(instance, name, ref_values, *, max_depth): + _dereference = _import_class("DeReference")() + documents = _dereference( + ref_values, + max_depth=max_depth, + instance=instance, + name=name, + ) + return documents + def __get__(self, instance, owner): """Descriptor to automatically dereference references.""" if instance is None: @@ -284,19 +295,15 @@ class ComplexBaseField(BaseField): or isinstance(self.field, (GenericReferenceField, ReferenceField)) ) - _dereference = _import_class("DeReference")() - if ( instance._initialised and dereference and instance._data.get(self.name) and not getattr(instance._data[self.name], "_dereferenced", False) ): - instance._data[self.name] = _dereference( - instance._data.get(self.name), - max_depth=1, - instance=instance, - name=self.name, + ref_values = instance._data.get(self.name) + instance._data[self.name] = self._lazy_load_refs( + ref_values=ref_values, instance=instance, name=self.name, max_depth=1 ) if hasattr(instance._data[self.name], "_dereferenced"): instance._data[self.name]._dereferenced = True @@ -322,7 +329,9 @@ class ComplexBaseField(BaseField): and isinstance(value, (BaseList, BaseDict)) and not value._dereferenced ): - value = _dereference(value, max_depth=1, instance=instance, name=self.name) + value = self._lazy_load_refs( + ref_values=value, instance=instance, name=self.name, max_depth=1 + ) value._dereferenced = True instance._data[self.name] = value diff --git a/mongoengine/fields.py b/mongoengine/fields.py index cfff2b47..e714dde9 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1194,6 +1194,14 @@ class ReferenceField(BaseField): self.document_type_obj = get_document(self.document_type_obj) return self.document_type_obj + @staticmethod + def _lazy_load_ref(ref_cls, dbref): + dereferenced_son = ref_cls._get_db().dereference(dbref) + if dereferenced_son is None: + raise DoesNotExist(f"Trying to dereference unknown document {dbref}") + + return ref_cls._from_son(dereferenced_son) + def __get__(self, instance, owner): """Descriptor to allow lazy dereferencing.""" if instance is None: @@ -1201,20 +1209,17 @@ class ReferenceField(BaseField): return self # Get value from document instance if available - value = instance._data.get(self.name) + ref_value = instance._data.get(self.name) auto_dereference = instance._fields[self.name]._auto_dereference # Dereference DBRefs - if auto_dereference and isinstance(value, DBRef): - if hasattr(value, "cls"): + if auto_dereference and isinstance(ref_value, DBRef): + if hasattr(ref_value, "cls"): # Dereference using the class type specified in the reference - cls = get_document(value.cls) + cls = get_document(ref_value.cls) else: cls = self.document_type - dereferenced = cls._get_db().dereference(value) - if dereferenced is None: - raise DoesNotExist("Trying to dereference unknown document %s" % value) - else: - instance._data[self.name] = cls._from_son(dereferenced) + + instance._data[self.name] = self._lazy_load_ref(cls, ref_value) return super().__get__(instance, owner) @@ -1353,6 +1358,14 @@ class CachedReferenceField(BaseField): self.document_type_obj = get_document(self.document_type_obj) return self.document_type_obj + @staticmethod + def _lazy_load_ref(ref_cls, dbref): + dereferenced_son = ref_cls._get_db().dereference(dbref) + if dereferenced_son is None: + raise DoesNotExist(f"Trying to dereference unknown document {dbref}") + + return ref_cls._from_son(dereferenced_son) + def __get__(self, instance, owner): if instance is None: # Document class being used rather than a document object @@ -1364,11 +1377,7 @@ class CachedReferenceField(BaseField): # Dereference DBRefs if auto_dereference and isinstance(value, DBRef): - dereferenced = self.document_type._get_db().dereference(value) - if dereferenced is None: - raise DoesNotExist("Trying to dereference unknown document %s" % value) - else: - instance._data[self.name] = self.document_type._from_son(dereferenced) + instance._data[self.name] = self._lazy_load_ref(self.document_type, value) return super().__get__(instance, owner) @@ -1493,6 +1502,14 @@ class GenericReferenceField(BaseField): value = value._class_name super()._validate_choices(value) + @staticmethod + def _lazy_load_ref(ref_cls, dbref): + dereferenced_son = ref_cls._get_db().dereference(dbref) + if dereferenced_son is None: + raise DoesNotExist(f"Trying to dereference unknown document {dbref}") + + return ref_cls._from_son(dereferenced_son) + def __get__(self, instance, owner): if instance is None: return self @@ -1500,12 +1517,9 @@ class GenericReferenceField(BaseField): value = instance._data.get(self.name) auto_dereference = instance._fields[self.name]._auto_dereference - if auto_dereference and isinstance(value, (dict, SON)): - dereferenced = self.dereference(value) - if dereferenced is None: - raise DoesNotExist("Trying to dereference unknown document %s" % value) - else: - instance._data[self.name] = dereferenced + if auto_dereference and isinstance(value, dict): + doc_cls = get_document(value["_cls"]) + instance._data[self.name] = self._lazy_load_ref(doc_cls, value["_ref"]) return super().__get__(instance, owner) @@ -1524,14 +1538,6 @@ class GenericReferenceField(BaseField): " saved to the database" ) - def dereference(self, value): - doc_cls = get_document(value["_cls"]) - reference = value["_ref"] - doc = doc_cls._get_db().dereference(reference) - if doc is not None: - doc = doc_cls._from_son(doc) - return doc - def to_mongo(self, document): if document is None: return None From e31f9150d26f2ea7c7ecb09445a0cb39c2a2fd4d Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Sun, 28 Feb 2021 14:32:46 +0100 Subject: [PATCH 09/11] improve connect() doc and put more emphasis on URI string --- docs/guide/connecting.rst | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index fea5e97b..d9b8bdbf 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -25,30 +25,35 @@ and :attr:`authentication_source` arguments should be provided:: connect('project1', username='webapp', password='pwd123', authentication_source='admin') -URI style connections are also supported -- just supply the URI as -the :attr:`host` to -:func:`~mongoengine.connect`:: +URI string connection is also supported and **is the recommended way to connect**. The URI string is +forwarded to the driver as is, so it follows the same scheme as the `MongoDB URI `_. +Just supply the URI as the :attr:`host` to :func:`~mongoengine.connect`:: + + connect(host="mongodb://user:password@hostname:port/db_name") + +URI string can be used to configure advanced parameters like ssl, replicaSet, etc:: + + connect(host="mongodb://user:password@hostname:port/db_name?ssl=true&replicaSet=globaldb") - connect('project1', host='mongodb://localhost/database_name') .. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host` -.. note:: Database, username and password from URI string overrides - corresponding parameters in :func:`~mongoengine.connect`: :: +.. note:: The URI string has precedence over keyword args so if you accidentally call :: connect( db='test', username='user', password='12345', - host='mongodb://admin:qwerty@localhost/production' + host='mongodb://admin:qwerty@localhost/my_db' ) - will establish connection to ``production`` database using - ``admin`` username and ``12345`` password. + it will ignore the db, username and password argument and establish the connection to ``my_db`` database using + ``admin`` username and ``qwerty`` password. .. note:: Calling :func:`~mongoengine.connect` without argument will establish a connection to the "test" database by default + Replica Sets ============ From 50d891cb7bfa22f335a2e8a2279cf5e57fd4a596 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Mon, 8 Mar 2021 21:51:25 +0100 Subject: [PATCH 10/11] more improvement to connect doc --- docs/changelog.rst | 3 +- docs/guide/connecting.rst | 97 ++++++++++++++++++++++++--------------- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64bf2775..036ad1d1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,9 +7,10 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- Improve connection doc #2481 Changes in 0.23.0 -=========== +================= - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 - Add MongoDB 4.2 and 4.4 to CI - Add support for allowDiskUse on querysets #2468 diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index d9b8bdbf..387151df 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -5,7 +5,7 @@ Connecting to MongoDB ===================== Connections in MongoEngine are registered globally and are identified with aliases. -If no `alias` is provided during the connection, it will use "default" as alias. +If no ``alias`` is provided during the connection, it will use "default" as alias. To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect` function. The first argument is the name of the database to connect to:: @@ -14,62 +14,87 @@ function. The first argument is the name of the database to connect to:: connect('project1') By default, MongoEngine assumes that the :program:`mongod` instance is running -on **localhost** on port **27017**. If MongoDB is running elsewhere, you should -provide the :attr:`host` and :attr:`port` arguments to -:func:`~mongoengine.connect`:: +on **localhost** on port **27017**. - connect('project1', host='192.168.1.35', port=12345) +If MongoDB is running elsewhere, you need to provide details on how to connect. There are two ways of +doing this. Using a connection string in URI format (**this is the preferred method**) or individual attributes +provided as keyword arguments. + +Connect with URI string +======================= + +When using a connection string in URI format you should specify the connection details +as the :attr:`host` to :func:`~mongoengine.connect`. In a web application context for instance, the URI +is typically read from the config file:: + + connect(host="mongodb://127.0.0.1:27017/my_db") + +If the database requires authentication, you can specify it in the +URI. As each database can have its own users configured, you need to tell MongoDB +where to look for the user you are working with, that's what the ``?authSource=admin`` bit +of the MongoDB connection string is for:: + + # Connects to 'my_db' database by authenticating + # with given credentials against the 'admin' database (by default as authSource isn't provided) + connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db") + + # Equivalent to previous connection but explicitly states that + # it should use admin as the authentication source database + connect(host="mongodb://my_user:my_password@hostname:port/my_db?authSource=admin") + + # Connects to 'my_db' database by authenticating + # with given credentials against that same database + connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=my_db") + +The URI string can also be used to configure advanced parameters like ssl, replicaSet, etc. For more +information or example about URI string, you can refer to the `official doc `_:: + + connect(host="mongodb://my_user:my_password@127.0.0.1:27017/my_db?authSource=admin&ssl=true&replicaSet=globaldb") + +.. note:: URI containing SRV records (e.g "mongodb+srv://server.example.com/") can be used as well + +Connect with keyword attributes +=============================== + +The second option for specifying the connection details is to provide the information as keyword +attributes to :func:`~mongoengine.connect`:: + + connect('my_db', host='127.0.0.1', port=27017) If the database requires authentication, :attr:`username`, :attr:`password` and :attr:`authentication_source` arguments should be provided:: - connect('project1', username='webapp', password='pwd123', authentication_source='admin') + connect('my_db', username='my_user', password='my_password', authentication_source='admin') -URI string connection is also supported and **is the recommended way to connect**. The URI string is -forwarded to the driver as is, so it follows the same scheme as the `MongoDB URI `_. -Just supply the URI as the :attr:`host` to :func:`~mongoengine.connect`:: +The set of attributes that :func:`~mongoengine.connect` recognizes includes but is not limited to: +:attr:`host`, :attr:`port`, :attr:`read_preference`, :attr:`username`, :attr:`password`, :attr:`authentication_source`, :attr:`authentication_mechanism`, +:attr:`replicaset`, :attr:`tls`, etc. Most of the parameters accepted by `pymongo.MongoClient `_ +can be used with :func:`~mongoengine.connect` and will simply be forwarded when instantiating the `pymongo.MongoClient`. - connect(host="mongodb://user:password@hostname:port/db_name") - -URI string can be used to configure advanced parameters like ssl, replicaSet, etc:: - - connect(host="mongodb://user:password@hostname:port/db_name?ssl=true&replicaSet=globaldb") - - -.. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host` - -.. note:: The URI string has precedence over keyword args so if you accidentally call :: +.. note:: Database, username and password from URI string overrides + corresponding parameters in :func:`~mongoengine.connect`, this should + obviously be avoided: :: connect( db='test', username='user', password='12345', - host='mongodb://admin:qwerty@localhost/my_db' + host='mongodb://admin:qwerty@localhost/production' ) - it will ignore the db, username and password argument and establish the connection to ``my_db`` database using - ``admin`` username and ``qwerty`` password. + will establish connection to ``production`` database using ``admin`` username and ``qwerty`` password. .. note:: Calling :func:`~mongoengine.connect` without argument will establish a connection to the "test" database by default +Read Preferences +================ -Replica Sets -============ - -MongoEngine supports connecting to replica sets:: - - from mongoengine import connect - - # Regular connect - connect('dbname', replicaset='rs-name') - - # MongoDB URI-style connect - connect(host='mongodb://localhost/dbname?replicaSet=rs-name') - -Read preferences are supported through the connection or via individual +As stated above, Read preferences are supported through the connection but also via individual queries by passing the read_preference :: + from pymongo import ReadPreference + Bar.objects().read_preference(ReadPreference.PRIMARY) Bar.objects(read_preference=ReadPreference.PRIMARY) From b3ce65453aef5e63668ec62a57b37f9ef3b340c3 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Mon, 8 Mar 2021 22:21:48 +0100 Subject: [PATCH 11/11] Fix one-to-many example that is actually a many to many --- docs/guide/defining-documents.rst | 6 +++--- mongoengine/fields.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index ed092907..a457de1f 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -290,12 +290,12 @@ as the constructor's argument:: content = StringField() -.. _one-to-many-with-listfields: +.. _many-to-many-with-listfields: -One to Many with ListFields +Many to Many with ListFields ''''''''''''''''''''''''''' -If you are implementing a one to many relationship via a list of references, +If you are implementing a many to many relationship via a list of references, then the references are stored as DBRefs and to query you need to pass an instance of the object to the query:: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e714dde9..276b02ce 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -915,7 +915,7 @@ class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. - If using with ReferenceFields see: :ref:`one-to-many-with-listfields` + If using with ReferenceFields see: :ref:`many-to-many-with-listfields` .. note:: Required means it cannot be empty - as the default for ListFields is []