From 3e0d84383e4a0f9c52d6840ff1d4a882e7eeb6cb Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 13:41:20 +0800 Subject: [PATCH 01/13] use insert_one, insert_many and remove deprecated one #1491 --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 29471463..4dec42ce 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ dev - Fix validation error instance in GenericEmbeddedDocumentField #1067 - Update cached fields when fields argument is given #1712 - Add a db parameter to register_connection for compatibility with connect +- Use insert_one, insert_many in Document.insert #1491 Changes in 0.15.0 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index b1d3342b..95b25d06 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -350,8 +350,14 @@ class BaseQuerySet(object): documents=docs, **signal_kwargs) raw = [doc.to_mongo() for doc in docs] + insert_func = self._collection.insert_many + if return_one: + raw = raw[0] + insert_func = self._collection.insert_one + try: - ids = self._collection.insert(raw, **write_concern) + inserted_result = insert_func(raw, **write_concern) + ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) From f9a887c8c6665a77dda33ca785acf2ece8158337 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 14:33:12 +0800 Subject: [PATCH 02/13] fix inserted_ids --- mongoengine/queryset/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 95b25d06..eec7df18 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -357,7 +357,8 @@ class BaseQuerySet(object): try: inserted_result = insert_func(raw, **write_concern) - ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids + ids = return_one and inserted_result.inserted_id or\ + inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) @@ -374,7 +375,6 @@ class BaseQuerySet(object): signals.post_bulk_insert.send( self._document, documents=docs, loaded=False, **signal_kwargs) return return_one and ids[0] or ids - documents = self.in_bulk(ids) results = [] for obj_id in ids: From 1d3f20b666bfe0ef8dcdea06a2341e460d13c486 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 14:41:25 +0800 Subject: [PATCH 03/13] fix style and ids need to be an array --- mongoengine/queryset/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 2f2d38dd..0634dceb 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -357,8 +357,7 @@ class BaseQuerySet(object): try: inserted_result = insert_func(raw, **write_concern) - ids = return_one and inserted_result.inserted_id or\ - inserted_result.inserted_ids + ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) From 1aebc9514564a9ba1c58cb52fa4552ace6d2066a Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 13:41:20 +0800 Subject: [PATCH 04/13] use insert_one, insert_many and remove deprecated one #1491 --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb31da21..e3d366b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ dev - Update cached fields when fields argument is given #1712 - Add a db parameter to register_connection for compatibility with connect - Use new update_one, update_many on document/queryset update #1491 +- Use insert_one, insert_many in Document.insert #1491 Changes in 0.15.0 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index f78ee882..ff7afa96 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -350,8 +350,14 @@ class BaseQuerySet(object): documents=docs, **signal_kwargs) raw = [doc.to_mongo() for doc in docs] + insert_func = self._collection.insert_many + if return_one: + raw = raw[0] + insert_func = self._collection.insert_one + try: - ids = self._collection.insert(raw, **write_concern) + inserted_result = insert_func(raw, **write_concern) + ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) From 5c0bd8a81090aa39a74b347adb7afde9edb202b6 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 14:33:12 +0800 Subject: [PATCH 05/13] fix inserted_ids --- mongoengine/queryset/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index ff7afa96..2f2d38dd 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -357,7 +357,8 @@ class BaseQuerySet(object): try: inserted_result = insert_func(raw, **write_concern) - ids = inserted_result.inserted_id if return_one else inserted_result.inserted_ids + ids = return_one and inserted_result.inserted_id or\ + inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) @@ -374,7 +375,6 @@ class BaseQuerySet(object): signals.post_bulk_insert.send( self._document, documents=docs, loaded=False, **signal_kwargs) return return_one and ids[0] or ids - documents = self.in_bulk(ids) results = [] for obj_id in ids: From 0fc55451c2aaad6a612d28ead0b58898a2236e40 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 14:41:25 +0800 Subject: [PATCH 06/13] fix style and ids need to be an array --- mongoengine/queryset/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 2f2d38dd..0634dceb 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -357,8 +357,7 @@ class BaseQuerySet(object): try: inserted_result = insert_func(raw, **write_concern) - ids = return_one and inserted_result.inserted_id or\ - inserted_result.inserted_ids + ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) From acba86993dd92b04fff289a151cb8882913bbf20 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 15:43:19 +0800 Subject: [PATCH 07/13] set_write_concern pymongo3 --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0634dceb..0a3f65fb 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -356,7 +356,7 @@ class BaseQuerySet(object): insert_func = self._collection.insert_one try: - inserted_result = insert_func(raw, **write_concern) + inserted_result = insert_func(raw, set_write_concern(write_concern)) ids = return_one and [inserted_result.inserted_id] or inserted_result.inserted_ids except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' From fa4ac95ecc245c59040664905c935d3c85b49429 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 17:49:49 +0800 Subject: [PATCH 08/13] catch bulkwriteerror --- mongoengine/queryset/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 008e7f15..391cc819 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -371,6 +371,11 @@ class BaseQuerySet(object): message = u'Tried to save duplicate unique keys (%s)' raise NotUniqueError(message % six.text_type(err)) raise OperationError(message % six.text_type(err)) + except pymongo.error.BulkWriteError as err: + # inserting documents that already have an _id field will + # give huge performance debt or raise + message = u'Document must not have _id value before bulk write (%s)' + raise NotUniqueError(message % sx.text_type(err)) if not load_bulk: signals.post_bulk_insert.send( From 78601d90c9fd13b20d1e6cfe2342ddb23a520e33 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 17:54:13 +0800 Subject: [PATCH 09/13] fix typo --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 391cc819..4c8309a7 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -375,7 +375,7 @@ class BaseQuerySet(object): # inserting documents that already have an _id field will # give huge performance debt or raise message = u'Document must not have _id value before bulk write (%s)' - raise NotUniqueError(message % sx.text_type(err)) + raise NotUniqueError(message % six.text_type(err)) if not load_bulk: signals.post_bulk_insert.send( From 94cda90a6ead7a64059328c58dafa6154436c951 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 17:56:19 +0800 Subject: [PATCH 10/13] fix syntax --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 4c8309a7..bba63da0 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -371,7 +371,7 @@ class BaseQuerySet(object): message = u'Tried to save duplicate unique keys (%s)' raise NotUniqueError(message % six.text_type(err)) raise OperationError(message % six.text_type(err)) - except pymongo.error.BulkWriteError as err: + except pymongo.errors.BulkWriteError as err: # inserting documents that already have an _id field will # give huge performance debt or raise message = u'Document must not have _id value before bulk write (%s)' From 088fd6334bfe8316c2df2466ad93ee568e3013c6 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 18:06:59 +0800 Subject: [PATCH 11/13] bulkwriteerror does not trigger --- mongoengine/queryset/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index bba63da0..647eafc6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -363,6 +363,11 @@ class BaseQuerySet(object): except pymongo.errors.DuplicateKeyError as err: message = 'Could not save document (%s)' raise NotUniqueError(message % six.text_type(err)) + except pymongo.errors.BulkWriteError as err: + # inserting documents that already have an _id field will + # give huge performance debt or raise + message = u'Document must not have _id value before bulk write (%s)' + raise NotUniqueError(message % six.text_type(err)) except pymongo.errors.OperationFailure as err: message = 'Could not save document (%s)' if re.match('^E1100[01] duplicate key', six.text_type(err)): @@ -371,11 +376,6 @@ class BaseQuerySet(object): message = u'Tried to save duplicate unique keys (%s)' raise NotUniqueError(message % six.text_type(err)) raise OperationError(message % six.text_type(err)) - except pymongo.errors.BulkWriteError as err: - # inserting documents that already have an _id field will - # give huge performance debt or raise - message = u'Document must not have _id value before bulk write (%s)' - raise NotUniqueError(message % six.text_type(err)) if not load_bulk: signals.post_bulk_insert.send( From 506168ab83a11dc22c706dd308f77f6233543d57 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 18:25:50 +0800 Subject: [PATCH 12/13] use write_concern class --- tests/queryset/queryset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 5c3f179f..4c0085d0 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -927,8 +927,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], - write_concern={"w": 0, 'continue_on_error': True}) + Blog.objects.insert([blog2, blog3], write_concern={"w": 0}) self.assertEqual(Blog.objects.count(), 3) def test_get_changed_fields_query_count(self): From fc5d9ae10033e7e083931d9a64b7137feadc6eb6 Mon Sep 17 00:00:00 2001 From: Erdenezul Batmunkh Date: Sun, 20 May 2018 18:39:22 +0800 Subject: [PATCH 13/13] pymongo3 does not support continue_on_error --- tests/queryset/queryset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 4c0085d0..3fa86a58 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -927,9 +927,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], write_concern={"w": 0}) - self.assertEqual(Blog.objects.count(), 3) - def test_get_changed_fields_query_count(self): """Make sure we don't perform unnecessary db operations when none of document's fields were updated.