From 8ef771912dbcd0cc56cfb7a1b5fefb6172518139 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Sun, 28 Feb 2021 14:07:15 +0100 Subject: [PATCH 1/6] fixing incompatibility with mongoengine aggregation to support mongo 4.4 --- mongoengine/mongodb_support.py | 1 + mongoengine/queryset/base.py | 9 +++------ tests/document/test_indexes.py | 5 ++++- tests/queryset/test_queryset.py | 18 +++++++++++++++--- tests/utils.py | 8 ++++++++ 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 522f064e..df51100d 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -8,6 +8,7 @@ from mongoengine.connection import get_connection # get_mongodb_version() MONGODB_34 = (3, 4) MONGODB_36 = (3, 6) +MONGODB_44 = (4, 4) def get_mongodb_version(): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index ae8cd407..47a5f733 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1355,21 +1355,18 @@ class BaseQuerySet: MapReduceDocument = _import_class("MapReduceDocument") - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - map_f_scope = {} if isinstance(map_f, Code): map_f_scope = map_f.scope map_f = str(map_f) - map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) + map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None) reduce_f_scope = {} if isinstance(reduce_f, Code): reduce_f_scope = reduce_f.scope reduce_f = str(reduce_f) reduce_f_code = queryset._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) + reduce_f = Code(reduce_f_code, reduce_f_scope or None) mr_args = {"query": queryset._query} @@ -1379,7 +1376,7 @@ class BaseQuerySet: finalize_f_scope = finalize_f.scope finalize_f = str(finalize_f) finalize_f_code = queryset._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) + finalize_f = Code(finalize_f_code, finalize_f_scope or None) mr_args["finalize"] = finalize_f if scope: diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index 55a56931..17643dd8 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -7,6 +7,7 @@ import pytest from mongoengine import * from mongoengine.connection import get_db +from mongoengine.mongodb_support import MONGODB_44, get_mongodb_version class TestIndexes(unittest.TestCase): @@ -452,9 +453,11 @@ class TestIndexes(unittest.TestCase): .get("stage") == "IXSCAN" ) + mongo_db = get_mongodb_version() + PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_44 else "PROJECTION_COVERED" assert ( query_plan.get("queryPlanner").get("winningPlan").get("stage") - == "PROJECTION" + == PROJECTION_STR ) query_plan = Test.objects(a=1).explain() diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 4d281c60..c346abde 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -21,7 +21,11 @@ from mongoengine.queryset import ( QuerySetManager, queryset_manager, ) -from tests.utils import requires_mongodb_gte_44 +from tests.utils import ( + requires_mongodb_gte_44, + requires_mongodb_lt_42, + requires_mongodb_lte_42, +) class db_ops_tracker(query_counter): @@ -1490,6 +1494,7 @@ class TestQueryset(unittest.TestCase): BlogPost.drop_collection() + @requires_mongodb_lt_42 def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js.""" @@ -1527,6 +1532,7 @@ class TestQueryset(unittest.TestCase): BlogPost.drop_collection() + @requires_mongodb_lt_42 def test_exec_js_field_sub(self): """Ensure that field substitutions occur properly in exec_js functions.""" @@ -3109,6 +3115,7 @@ class TestQueryset(unittest.TestCase): freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True) assert freq == {"CRB": 0.5, None: 0.5} + @requires_mongodb_lte_42 def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): name = StringField() @@ -3137,6 +3144,7 @@ class TestQueryset(unittest.TestCase): ot = Person.objects.item_frequencies("extra.tag", map_reduce=True) assert ot == {None: 1.0, "friend": 1.0} + @requires_mongodb_lte_42 def test_item_frequencies_with_0_values(self): class Test(Document): val = IntField() @@ -3151,6 +3159,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {0: 1} + @requires_mongodb_lte_42 def test_item_frequencies_with_False_values(self): class Test(Document): val = BooleanField() @@ -3165,6 +3174,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {False: 1} + @requires_mongodb_lte_42 def test_item_frequencies_normalize(self): class Test(Document): val = IntField() @@ -3551,7 +3561,8 @@ class TestQueryset(unittest.TestCase): Book.objects.create(title="The Stories", authors=[mark_twain, john_tolkien]) authors = Book.objects.distinct("authors") - assert authors == [mark_twain, john_tolkien] + authors_names = {author.name for author in authors} + assert authors_names == {mark_twain.name, john_tolkien.name} def test_distinct_ListField_EmbeddedDocumentField_EmbeddedDocumentField(self): class Continent(EmbeddedDocument): @@ -3588,7 +3599,8 @@ class TestQueryset(unittest.TestCase): assert country_list == [scotland, tibet] continent_list = Book.objects.distinct("authors.country.continent") - assert continent_list == [europe, asia] + continent_list_names = {c.continent_name for c in continent_list} + assert continent_list_names == {europe.continent_name, asia.continent_name} def test_distinct_ListField_ReferenceField(self): class Bar(Document): diff --git a/tests/utils.py b/tests/utils.py index adb0bdb4..19596afa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,6 +34,14 @@ def get_as_pymongo(doc): return doc.__class__.objects.as_pymongo().get(id=doc.id) +def requires_mongodb_lt_42(func): + return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt) + + +def requires_mongodb_lte_42(func): + return _decorated_with_ver_requirement(func, (4, 2), oper=operator.le) + + def requires_mongodb_gte_44(func): return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge) From b479bb7c6bdf4c8853555dad3f24a67d407db484 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 10:52:46 +0100 Subject: [PATCH 2/6] Fix tests for supporting Mongo4.4 for some reason results comes sorted differently in map reduce when no sort is specified --- tests/queryset/test_queryset.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index c346abde..a4dbabd5 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -2666,6 +2666,8 @@ class TestQueryset(unittest.TestCase): title = StringField(primary_key=True) tags = ListField(StringField()) + BlogPost.drop_collection() + post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"]) post2 = BlogPost(title="Post #2", tags=["django", "mongodb"]) post3 = BlogPost(title="Post #3", tags=["hitchcock films"]) @@ -2694,12 +2696,15 @@ class TestQueryset(unittest.TestCase): } """ - results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults") + results = BlogPost.objects.order_by("_id").map_reduce( + map_f, reduce_f, "myresults2" + ) results = list(results) - assert results[0].object == post1 - assert results[1].object == post2 - assert results[2].object == post3 + assert len(results) == 3 + assert results[0].object.id == post1.id + assert results[1].object.id == post2.id + assert results[2].object.id == post3.id BlogPost.drop_collection() @@ -2707,7 +2712,6 @@ class TestQueryset(unittest.TestCase): """ Test map/reduce custom output """ - register_connection("test2", "mongoenginetest2") class Family(Document): id = IntField(primary_key=True) @@ -2780,6 +2784,7 @@ class TestQueryset(unittest.TestCase): family.persons.push(person); family.totalAge += person.age; }); + family.persons.sort((a, b) => (a.age > b.age)) } }); @@ -2808,10 +2813,10 @@ class TestQueryset(unittest.TestCase): "_id": 1, "value": { "persons": [ - {"age": 21, "name": "Wilson Jr"}, - {"age": 45, "name": "Wilson Father"}, - {"age": 40, "name": "Eliana Costa"}, {"age": 17, "name": "Tayza Mariana"}, + {"age": 21, "name": "Wilson Jr"}, + {"age": 40, "name": "Eliana Costa"}, + {"age": 45, "name": "Wilson Father"}, ], "totalAge": 123, }, @@ -2821,9 +2826,9 @@ class TestQueryset(unittest.TestCase): "_id": 2, "value": { "persons": [ + {"age": 10, "name": "Igor Gabriel"}, {"age": 16, "name": "Isabella Luanna"}, {"age": 36, "name": "Sandra Mara"}, - {"age": 10, "name": "Igor Gabriel"}, ], "totalAge": 62, }, @@ -2833,8 +2838,8 @@ class TestQueryset(unittest.TestCase): "_id": 3, "value": { "persons": [ - {"age": 30, "name": "Arthur WA"}, {"age": 25, "name": "Paula Leonel"}, + {"age": 30, "name": "Arthur WA"}, ], "totalAge": 55, }, From 3b9a167022f73ae0bf9e21e3e1a2c5901e7564bd Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 10:57:18 +0100 Subject: [PATCH 3/6] Add Mongo 4.4 to ci --- .github/workflows/github-actions.yml | 11 ++++++----- docs/changelog.rst | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 70393ebc..02eeccd4 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -11,9 +11,10 @@ on: tags: - 'v[0-9]+\.[0-9]+\.[0-9]+*' env: - MONGODB_3_4: 3.4.19 MONGODB_3_6: 3.6.13 MONGODB_4_0: 4.0.13 + MONGODB_4_2: 4.2.12 + MONGODB_4_4: 4.4.4 PYMONGO_3_4: 3.4 PYMONGO_3_6: 3.6 @@ -47,14 +48,14 @@ jobs: MONGODB: [$MONGODB_4_0] PYMONGO: [$PYMONGO_3_11] include: - - python-version: 3.7 - MONGODB: $MONGODB_3_4 - PYMONGO: $PYMONGO_3_6 - python-version: 3.7 MONGODB: $MONGODB_3_6 PYMONGO: $PYMONGO_3_9 - python-version: 3.7 - MONGODB: $MONGODB_3_6 + MONGODB: MONGODB_4_2 + PYMONGO: $PYMONGO_3_6 + - python-version: 3.7 + MONGODB: $MONGODB_4_4 PYMONGO: $PYMONGO_3_11 steps: - uses: actions/checkout@v2 diff --git a/docs/changelog.rst b/docs/changelog.rst index 5898eb1f..aaf8f3cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 +- Add MongoDB 4.4 to CI Changes in 0.22.1 ================= From 0620ac5641b721e2205647d67718a1ae329d1bac Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 11:30:03 +0100 Subject: [PATCH 4/6] Fix mongo download link as convention changed in official repo with > 4.0 --- .github/workflows/github-actions.yml | 4 ++-- .github/workflows/install_mongo.sh | 7 +++++++ docs/changelog.rst | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 02eeccd4..4ee0fd37 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -11,8 +11,8 @@ on: tags: - 'v[0-9]+\.[0-9]+\.[0-9]+*' env: - MONGODB_3_6: 3.6.13 - MONGODB_4_0: 4.0.13 + MONGODB_3_6: 3.6.14 + MONGODB_4_0: 4.0.23 MONGODB_4_2: 4.2.12 MONGODB_4_4: 4.4.4 diff --git a/.github/workflows/install_mongo.sh b/.github/workflows/install_mongo.sh index a98440a8..5f486666 100644 --- a/.github/workflows/install_mongo.sh +++ b/.github/workflows/install_mongo.sh @@ -2,7 +2,14 @@ MONGODB=$1 +# Mongo > 4.0 follows different name convention for download links mongo_build=mongodb-linux-x86_64-${MONGODB} + +if [[ "$MONGODB" == *"4."* ]] && [[ ! "$MONGODB" == *"4.0"* ]]; then + echo "It's there." + mongo_build=mongodb-linux-x86_64-ubuntu2004-v${MONGODB}-latest +fi + wget http://fastdl.mongodb.org/linux/$mongo_build.tgz tar xzf $mongo_build.tgz ${PWD}/$mongo_build/bin/mongod --version diff --git a/docs/changelog.rst b/docs/changelog.rst index aaf8f3cc..d8f7dde7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471 -- Add MongoDB 4.4 to CI +- Add MongoDB 4.2 and 4.4 to CI Changes in 0.22.1 ================= From f2442071681bc29957682348a3e01e83741c4f97 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 3 Mar 2021 11:40:39 +0100 Subject: [PATCH 5/6] fix mongo download link for 4.2 4.4 explicitly --- .github/workflows/github-actions.yml | 6 +++--- .github/workflows/install_mongo.sh | 11 +++++++---- .github/workflows/start_mongo.sh | 3 ++- mongoengine/mongodb_support.py | 1 + tests/document/test_indexes.py | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 4ee0fd37..bfdfa3a4 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -13,8 +13,8 @@ on: env: MONGODB_3_6: 3.6.14 MONGODB_4_0: 4.0.23 - MONGODB_4_2: 4.2.12 - MONGODB_4_4: 4.4.4 + MONGODB_4_2: 4.2 + MONGODB_4_4: 4.4 PYMONGO_3_4: 3.4 PYMONGO_3_6: 3.6 @@ -52,7 +52,7 @@ jobs: MONGODB: $MONGODB_3_6 PYMONGO: $PYMONGO_3_9 - python-version: 3.7 - MONGODB: MONGODB_4_2 + MONGODB: $MONGODB_4_2 PYMONGO: $PYMONGO_3_6 - python-version: 3.7 MONGODB: $MONGODB_4_4 diff --git a/.github/workflows/install_mongo.sh b/.github/workflows/install_mongo.sh index 5f486666..136f5c20 100644 --- a/.github/workflows/install_mongo.sh +++ b/.github/workflows/install_mongo.sh @@ -5,11 +5,14 @@ MONGODB=$1 # Mongo > 4.0 follows different name convention for download links mongo_build=mongodb-linux-x86_64-${MONGODB} -if [[ "$MONGODB" == *"4."* ]] && [[ ! "$MONGODB" == *"4.0"* ]]; then - echo "It's there." - mongo_build=mongodb-linux-x86_64-ubuntu2004-v${MONGODB}-latest +if [[ "$MONGODB" == *"4.2"* ]]; then + mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest +elif [[ "$MONGODB" == *"4.4"* ]]; then + mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest fi wget http://fastdl.mongodb.org/linux/$mongo_build.tgz tar xzf $mongo_build.tgz -${PWD}/$mongo_build/bin/mongod --version + +mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*") +$mongodb_dir/bin/mongod --version diff --git a/.github/workflows/start_mongo.sh b/.github/workflows/start_mongo.sh index dc844dbd..800004c8 100644 --- a/.github/workflows/start_mongo.sh +++ b/.github/workflows/start_mongo.sh @@ -2,7 +2,8 @@ MONGODB=$1 -mongodb_dir=${PWD}/mongodb-linux-x86_64-${MONGODB} +mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*") + mkdir $mongodb_dir/data $mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork mongo --eval 'db.version();' # Make sure mongo is awake diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index df51100d..8c5b3e5d 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -8,6 +8,7 @@ from mongoengine.connection import get_connection # get_mongodb_version() MONGODB_34 = (3, 4) MONGODB_36 = (3, 6) +MONGODB_42 = (4, 2) MONGODB_44 = (4, 4) diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index 17643dd8..2de9307d 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -7,7 +7,7 @@ import pytest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.mongodb_support import MONGODB_44, get_mongodb_version +from mongoengine.mongodb_support import MONGODB_42, get_mongodb_version class TestIndexes(unittest.TestCase): @@ -454,7 +454,7 @@ class TestIndexes(unittest.TestCase): == "IXSCAN" ) mongo_db = get_mongodb_version() - PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_44 else "PROJECTION_COVERED" + PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED" assert ( query_plan.get("queryPlanner").get("winningPlan").get("stage") == PROJECTION_STR From b9b536133de9f777b6643527b2b5fbaa366d4efc Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Thu, 4 Mar 2021 00:10:08 +0100 Subject: [PATCH 6/6] fix test incompat for 4.2 --- tests/queryset/test_queryset.py | 9 ++++----- tests/utils.py | 4 ---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index a4dbabd5..01e6b568 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -24,7 +24,6 @@ from mongoengine.queryset import ( from tests.utils import ( requires_mongodb_gte_44, requires_mongodb_lt_42, - requires_mongodb_lte_42, ) @@ -3120,7 +3119,7 @@ class TestQueryset(unittest.TestCase): freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True) assert freq == {"CRB": 0.5, None: 0.5} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): name = StringField() @@ -3149,7 +3148,7 @@ class TestQueryset(unittest.TestCase): ot = Person.objects.item_frequencies("extra.tag", map_reduce=True) assert ot == {None: 1.0, "friend": 1.0} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_0_values(self): class Test(Document): val = IntField() @@ -3164,7 +3163,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {0: 1} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_with_False_values(self): class Test(Document): val = BooleanField() @@ -3179,7 +3178,7 @@ class TestQueryset(unittest.TestCase): ot = Test.objects.item_frequencies("val", map_reduce=False) assert ot == {False: 1} - @requires_mongodb_lte_42 + @requires_mongodb_lt_42 def test_item_frequencies_normalize(self): class Test(Document): val = IntField() diff --git a/tests/utils.py b/tests/utils.py index 19596afa..52f56980 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -38,10 +38,6 @@ def requires_mongodb_lt_42(func): return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt) -def requires_mongodb_lte_42(func): - return _decorated_with_ver_requirement(func, (4, 2), oper=operator.le) - - def requires_mongodb_gte_44(func): return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge)