From 5a038de1d52405e32602b238cb66b4c3f779fc7c Mon Sep 17 00:00:00 2001 From: Lars Butler Date: Mon, 5 Oct 2015 14:31:41 +0200 Subject: [PATCH 01/99] fields.ReferenceField: add integer values to `reverse_delete_rule` docs When I first tried to use the `reverse_delete_rule` feature of `ReferenceField`, I had to dig through the source code to find what the actual integer values were expected to be for DO_NOTHING, NULLIFY, CASCADE, DENY, and PULL (or at least, where these constants were defined so that I could import and use them). This patch adds the integer values for those constants (which are defined in mongoengine.queryset.base) to the docs so that users can easily choose the correct integer value. Note: A possible improvement on this change would be to include `mongoengine.queryset.base` module documentation in the generated docs, and then update the `ReferenceField` docs to link to the documentation of these constants (DO_NOTHING, NULLIFY, etc.). --- mongoengine/fields.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f5899311..ad3c468b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -863,12 +863,11 @@ class ReferenceField(BaseField): The options are: - * DO_NOTHING - don't do anything (default). - * NULLIFY - Updates the reference to null. - * CASCADE - Deletes the documents associated with the reference. - * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.fields.ListField` - of references + * DO_NOTHING (0) - don't do anything (default). + * NULLIFY (1) - Updates the reference to null. + * CASCADE (2) - Deletes the documents associated with the reference. + * DENY (3) - Prevent the deletion of the reference object. + * PULL (4) - Pull the reference from a :class:`~mongoengine.fields.ListField` of references Alternative syntax for registering delete rules (useful when implementing bi-directional delete rules) From 11024deaae1cb22cfef117dc65643a22eac050a7 Mon Sep 17 00:00:00 2001 From: Kirill Kuzminykh Date: Mon, 5 Oct 2015 22:40:44 +0300 Subject: [PATCH 02/99] Fixed detection of shared connections --- mongoengine/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 4abca1ab..cf2bd41c 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -126,6 +126,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): connection_settings.pop('name', None) connection_settings.pop('username', None) connection_settings.pop('password', None) + connection_settings.pop('authentication_source', None) if conn_settings == connection_settings and _connections.get(db_alias, None): connection = _connections[db_alias] break From e049cef00afb2fc1ae0e315674282a2d39464b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:54:58 -0400 Subject: [PATCH 03/99] Add arbitrary metadata capture to `BaseField`. Includes ability to detect and report conflicts. --- mongoengine/base/fields.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 304c084d..d92e9b3a 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -42,7 +42,8 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None, verbose_name=None, - help_text=None, null=False, sparse=False, custom_data=None): + help_text=None, null=False, sparse=False, custom_data=None + **kwargs): """ :param db_field: The database field to store this field in (defaults to the name of the field) @@ -70,6 +71,7 @@ class BaseField(object): :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values :param custom_data: (optional) Custom metadata for this field. + :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. """ self.db_field = (db_field or name) if not primary_key else '_id' @@ -89,6 +91,15 @@ class BaseField(object): self.sparse = sparse self._owner_document = None self.custom_data = custom_data + + conflicts = set(dir(self)).intersect(kwargs) + if conflicts: + raise TypeError("%s already has attribute(s): %s" % ( + self.__class__.__name__, + ', '.join(conflicts) + )) + + self.__dict__.update(kwargs) # Adjust the appropriate creation counter, and save our local copy. if self.db_field == '_id': From d133913c3dd561e3b0a0002489cad4be9973637f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:59:29 -0400 Subject: [PATCH 04/99] Remove now superfluous special cases. Removes `verbose_name`, `help_text`, and `custom_data`. All three are covered by the one metadata assignment and will continue working as expected. --- mongoengine/base/fields.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index d92e9b3a..3427a094 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -41,8 +41,7 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, - validation=None, choices=None, verbose_name=None, - help_text=None, null=False, sparse=False, custom_data=None + validation=None, choices=None, null=False, sparse=False, **kwargs): """ :param db_field: The database field to store this field in @@ -61,16 +60,10 @@ class BaseField(object): field. Generally this is deprecated in favour of the `FIELD.validate` method :param choices: (optional) The valid choices - :param verbose_name: (optional) The verbose name for the field. - Designed to be human readable and is often used when generating - model forms from the document model. - :param help_text: (optional) The help text for this field and is often - used when generating model forms from the document model. :param null: (optional) Is the field value can be null. If no and there is a default value then the default value is set :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values - :param custom_data: (optional) Custom metadata for this field. :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. """ self.db_field = (db_field or name) if not primary_key else '_id' @@ -85,12 +78,9 @@ class BaseField(object): self.primary_key = primary_key self.validation = validation self.choices = choices - self.verbose_name = verbose_name - self.help_text = help_text self.null = null self.sparse = sparse self._owner_document = None - self.custom_data = custom_data conflicts = set(dir(self)).intersect(kwargs) if conflicts: From 3f3747a2fe660d08bb8af4f079fb2f4b0c3540eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:59:46 -0400 Subject: [PATCH 05/99] Minor formatting tweaks and additional comments. --- mongoengine/base/fields.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 3427a094..9f054a2b 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -82,13 +82,14 @@ class BaseField(object): self.sparse = sparse self._owner_document = None + # Detect and report conflicts between metadata and base properties. conflicts = set(dir(self)).intersect(kwargs) if conflicts: raise TypeError("%s already has attribute(s): %s" % ( - self.__class__.__name__, - ', '.join(conflicts) - )) - + self.__class__.__name__, ', '.join(conflicts) )) + + # Assign metadata to the instance + # This efficient method is available because no __slots__ are defined. self.__dict__.update(kwargs) # Adjust the appropriate creation counter, and save our local copy. From a57f28ac832041ccb5c21cb7713c9723a3fd672f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 22:41:58 -0400 Subject: [PATCH 06/99] Correction for local monkeypatch. --- mongoengine/base/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 9f054a2b..d14ecce0 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -83,7 +83,7 @@ class BaseField(object): self._owner_document = None # Detect and report conflicts between metadata and base properties. - conflicts = set(dir(self)).intersect(kwargs) + conflicts = set(dir(self)) & set(kwargs) if conflicts: raise TypeError("%s already has attribute(s): %s" % ( self.__class__.__name__, ', '.join(conflicts) )) From 50b271c868c6b1fac95eb650d1b4b2592540e236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 22:51:03 -0400 Subject: [PATCH 07/99] Arbitrary metadata documentation. --- docs/guide/defining-documents.rst | 10 +++++----- mongoengine/base/fields.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 8f7382ee..c3ad208e 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -172,11 +172,11 @@ arguments can be set on all fields: class Shirt(Document): size = StringField(max_length=3, choices=SIZE) -:attr:`help_text` (Default: None) - Optional help text to output with the field -- used by form libraries - -:attr:`verbose_name` (Default: None) - Optional human-readable name for the field -- used by form libraries +:attr:`**kwargs` (Optional) + You can supply additional metadata as arbitrary additional keyword + arguments. You can not override existing attributes, however. Common + choices include `help_text` and `verbose_name`, commonly used by form and + widget libraries. List fields diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index d14ecce0..b1024526 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -64,7 +64,11 @@ class BaseField(object): then the default value is set :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values - :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. + :param **kwargs: (optional) Arbitrary indirection-free metadata for + this field can be supplied as additional keyword arguments and + accessed as attributes of the field. Must not conflict with any + existing attributes. Common metadata includes `verbose_name` and + `help_text`. """ self.db_field = (db_field or name) if not primary_key else '_id' From 159b0828286378543b3d5437b149d041eadfb53b Mon Sep 17 00:00:00 2001 From: reallistic Date: Thu, 24 Sep 2015 16:31:38 -0700 Subject: [PATCH 08/99] Recursively create mongo query for embeddeddocument elemMatch --- docs/changelog.rst | 1 + mongoengine/queryset/transform.py | 21 +++++++++++++-------- tests/queryset/queryset.py | 9 +++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b340aab0..bb626f33 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Changes in 0.10.1 - DEV - Fix Document.reload for DynamicDocument. #1050 - StrictDict & SemiStrictDict are shadowed at init time. #1105 - Remove test dependencies (nose and rednose) from install dependencies list. #1079 +- Recursively build query when using elemMatch operator. #1130 Changes in 0.10.0 ================= diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 03f3acf0..1f18c429 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -26,12 +26,12 @@ MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS + STRING_OPERATORS + CUSTOM_OPERATORS) -def query(_doc_cls=None, **query): +def query(_doc_cls=None, **kwargs): """Transform a query from Django-style format to Mongo format. """ mongo_query = {} merge_query = defaultdict(list) - for key, value in sorted(query.items()): + for key, value in sorted(kwargs.items()): if key == "__raw__": mongo_query.update(value) continue @@ -105,13 +105,18 @@ def query(_doc_cls=None, **query): if op: if op in GEO_OPERATORS: value = _geo_operator(field, op, value) - elif op in CUSTOM_OPERATORS: - if op in ('elem_match', 'match'): - value = field.prepare_query_value(op, value) - value = {"$elemMatch": value} + elif op in ('match', 'elemMatch'): + ListField = _import_class('ListField') + EmbeddedDocumentField = _import_class('EmbeddedDocumentField') + if (isinstance(value, dict) and isinstance(field, ListField) and + isinstance(field.field, EmbeddedDocumentField)): + value = query(field.field.document_type, **value) else: - NotImplementedError("Custom method '%s' has not " - "been implemented" % op) + value = field.prepare_query_value(op, value) + value = {"$elemMatch": value} + elif op in CUSTOM_OPERATORS: + NotImplementedError("Custom method '%s' has not " + "been implemented" % op) elif op not in STRING_OPERATORS: value = {'$' + op: value} diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 944c6fc1..3eff7cea 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4116,6 +4116,15 @@ class QuerySetTest(unittest.TestCase): ak = list(Bar.objects(foo__match=Foo(shape="square", color="purple"))) self.assertEqual([b1], ak) + ak = list( + Bar.objects(foo__elemMatch={'shape': "square", "color__exists": True})) + self.assertEqual([b1, b2], ak) + + ak = list( + Bar.objects(foo__match={'shape': "square", "color__exists": True})) + self.assertEqual([b1, b2], ak) + + def test_upsert_includes_cls(self): """Upserts should include _cls information for inheritable classes """ From 959740a585e23a71d56da05d046444f4d8810317 Mon Sep 17 00:00:00 2001 From: Axel Haustant Date: Mon, 19 Oct 2015 16:33:40 +0200 Subject: [PATCH 09/99] Fix false positive test on _instance --- tests/document/instance.py | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 7a393416..8caa5675 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -7,6 +7,7 @@ import os import pickle import unittest import uuid +import weakref from datetime import datetime from bson import DBRef, ObjectId @@ -30,6 +31,8 @@ TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), __all__ = ("InstanceTest",) + + class InstanceTest(unittest.TestCase): def setUp(self): @@ -63,6 +66,14 @@ class InstanceTest(unittest.TestCase): list(self.Person._get_collection().find().sort("id")), sorted(docs, key=lambda doc: doc["_id"])) + def assertHasInstance(self, field, instance): + self.assertTrue(hasattr(field, "_instance")) + self.assertIsNotNone(field._instance) + if isinstance(field._instance, weakref.ProxyType): + self.assertTrue(field._instance.__eq__(instance)) + else: + self.assertEqual(field._instance, instance) + def test_capped_collection(self): """Ensure that capped collections work properly. """ @@ -608,10 +619,12 @@ class InstanceTest(unittest.TestCase): embedded_field = EmbeddedDocumentField(Embedded) Doc.drop_collection() - Doc(embedded_field=Embedded(string="Hi")).save() + doc = Doc(embedded_field=Embedded(string="Hi")) + self.assertHasInstance(doc.embedded_field, doc) + doc.save() doc = Doc.objects.get() - self.assertEqual(doc, doc.embedded_field._instance) + self.assertHasInstance(doc.embedded_field, doc) def test_embedded_document_complex_instance(self): """Ensure that embedded documents in complex fields can reference @@ -623,10 +636,12 @@ class InstanceTest(unittest.TestCase): embedded_field = ListField(EmbeddedDocumentField(Embedded)) Doc.drop_collection() - Doc(embedded_field=[Embedded(string="Hi")]).save() + doc = Doc(embedded_field=[Embedded(string="Hi")]) + self.assertHasInstance(doc.embedded_field[0], doc) + doc.save() doc = Doc.objects.get() - self.assertEqual(doc, doc.embedded_field[0]._instance) + self.assertHasInstance(doc.embedded_field[0], doc) def test_instance_is_set_on_setattr(self): @@ -639,11 +654,28 @@ class InstanceTest(unittest.TestCase): Account.drop_collection() acc = Account() acc.email = Email(email='test@example.com') - self.assertTrue(hasattr(acc._data["email"], "_instance")) + self.assertHasInstance(acc._data["email"], acc) acc.save() acc1 = Account.objects.first() - self.assertTrue(hasattr(acc1._data["email"], "_instance")) + self.assertHasInstance(acc1._data["email"], acc1) + + def test_instance_is_set_on_setattr_on_embedded_document_list(self): + + class Email(EmbeddedDocument): + email = EmailField() + + class Account(Document): + emails = EmbeddedDocumentListField(Email) + + Account.drop_collection() + acc = Account() + acc.emails = [Email(email='test@example.com')] + self.assertHasInstance(acc._data["emails"][0], acc) + acc.save() + + acc1 = Account.objects.first() + self.assertHasInstance(acc1._data["emails"][0], acc1) def test_document_clean(self): class TestDocument(Document): From 6399de0b51dedc0c52a9c0480df785a20ce38c50 Mon Sep 17 00:00:00 2001 From: Axel Haustant Date: Mon, 19 Oct 2015 16:39:00 +0200 Subject: [PATCH 10/99] Fix _instance on list of EmbeddedDocuments --- mongoengine/base/fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 304c084d..4167720b 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -135,6 +135,10 @@ class BaseField(object): EmbeddedDocument = _import_class('EmbeddedDocument') if isinstance(value, EmbeddedDocument): value._instance = weakref.proxy(instance) + elif isinstance(value, (list, tuple)): + for v in value: + if isinstance(v, EmbeddedDocument): + v._instance = weakref.proxy(instance) instance._data[self.name] = value def error(self, message="", errors=None, field_name=None): From 9fe99979fe25e0ed94431ff51d7375a6da8ab9f6 Mon Sep 17 00:00:00 2001 From: Axel Haustant Date: Mon, 19 Oct 2015 18:04:15 +0200 Subject: [PATCH 11/99] Fix tests on Python 2.6 (assertIsNotNone does not exists) --- tests/document/instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 8caa5675..56e6765a 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -68,7 +68,7 @@ class InstanceTest(unittest.TestCase): def assertHasInstance(self, field, instance): self.assertTrue(hasattr(field, "_instance")) - self.assertIsNotNone(field._instance) + self.assertTrue(field._instance is not None) if isinstance(field._instance, weakref.ProxyType): self.assertTrue(field._instance.__eq__(instance)) else: From 44f92d41691284e552609bc535d99dc781dd06e4 Mon Sep 17 00:00:00 2001 From: abonhomme Date: Fri, 23 Oct 2015 11:07:26 -0400 Subject: [PATCH 12/99] docstring correction Corrected the docstring for `mongoengine.queryset.base.update_one()` --- mongoengine/queryset/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0e183889..0867e0e2 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -472,7 +472,8 @@ class BaseQuerySet(object): raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, upsert=False, write_concern=None, **update): - """Perform an atomic update on first field matched by the query. + """Perform an atomic update on the fields of the first document + matched by the query. :param upsert: Any existing document with that "_id" is overwritten. :param write_concern: Extra keyword arguments are passed down which From 2c282f955025c0c8d339c5ae539032ea3f544f84 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 8 Nov 2015 12:18:12 +0200 Subject: [PATCH 13/99] Added changelog entry for #1131. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb626f33..51f0967f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Changes in 0.10.1 - DEV - StrictDict & SemiStrictDict are shadowed at init time. #1105 - Remove test dependencies (nose and rednose) from install dependencies list. #1079 - Recursively build query when using elemMatch operator. #1130 +- Fix instance back references for lists of embedded documents. #1131 Changes in 0.10.0 ================= From fb4e9c37728d330dc01baadc9fb2ab5c96d4566d Mon Sep 17 00:00:00 2001 From: Paul-Armand Verhaegen Date: Tue, 10 Nov 2015 20:43:49 +0100 Subject: [PATCH 14/99] fix for reloading of strict with special fields --- mongoengine/document.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d2d9c5f..0cdfeeb6 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -594,12 +594,17 @@ class Document(BaseDocument): for field in obj._data: if not fields or field in fields: try: - setattr(self, field, self._reload(field, obj[field])) - except KeyError: - # If field is removed from the database while the object - # is in memory, a reload would cause a KeyError - # i.e. obj.update(unset__field=1) followed by obj.reload() - delattr(self, field) + setattr(self, field, self._reload(field, obj[field])) + except (KeyError, AttributeError): + try: + # If field is a special field, e.g. items is stored as _reserved_items, + # an KeyError is thrown. So try to retrieve the field from _data + setattr(self, field, self._reload(field, obj._data.get(field))) + except KeyError: + # If field is removed from the database while the object + # is in memory, a reload would cause a KeyError + # i.e. obj.update(unset__field=1) followed by obj.reload() + delattr(self, field) self._changed_fields = obj._changed_fields self._created = False From 6e745e9882edd0cf459166b289515cf4ee3145ef Mon Sep 17 00:00:00 2001 From: Paul-Armand Verhaegen Date: Tue, 10 Nov 2015 21:13:24 +0100 Subject: [PATCH 15/99] fixed wrong indentation style --- mongoengine/document.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 0cdfeeb6..bdae0a38 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -594,13 +594,13 @@ class Document(BaseDocument): for field in obj._data: if not fields or field in fields: try: - setattr(self, field, self._reload(field, obj[field])) + setattr(self, field, self._reload(field, obj[field])) except (KeyError, AttributeError): - try: - # If field is a special field, e.g. items is stored as _reserved_items, - # an KeyError is thrown. So try to retrieve the field from _data - setattr(self, field, self._reload(field, obj._data.get(field))) - except KeyError: + try: + # If field is a special field, e.g. items is stored as _reserved_items, + # an KeyError is thrown. So try to retrieve the field from _data + setattr(self, field, self._reload(field, obj._data.get(field))) + except KeyError: # If field is removed from the database while the object # is in memory, a reload would cause a KeyError # i.e. obj.update(unset__field=1) followed by obj.reload() From 3c8906494fb57e2b1c47b40091dea1b9eef24f4f Mon Sep 17 00:00:00 2001 From: Paul-Armand Verhaegen Date: Sun, 15 Nov 2015 15:31:22 +0100 Subject: [PATCH 16/99] Added #1156 to changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51f0967f..c9766c84 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Changes in 0.10.1 - DEV - Remove test dependencies (nose and rednose) from install dependencies list. #1079 - Recursively build query when using elemMatch operator. #1130 - Fix instance back references for lists of embedded documents. #1131 +- Fix for reloading of strict with special fields. #1156 Changes in 0.10.0 ================= From ed8174fe367c2c714ccc5addf7d7b02bff63592d Mon Sep 17 00:00:00 2001 From: Paul-Armand Verhaegen Date: Sun, 15 Nov 2015 15:32:26 +0100 Subject: [PATCH 17/99] Added Paul-Armand Verhaegen to contributor list --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4d5e69a3..df6ef611 100644 --- a/AUTHORS +++ b/AUTHORS @@ -229,3 +229,4 @@ that much better: * Emile Caron (https://github.com/emilecaron) * Amit Lichtenberg (https://github.com/amitlicht) * Lars Butler (https://github.com/larsbutler) + * Paul-Armand Verhaegen (https://github.com/paularmand) From cceef33fef18886082f35f847677bab67ed56a59 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 17 Nov 2015 14:22:10 +0000 Subject: [PATCH 18/99] Fixed a couple of documentation typos --- mongoengine/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d2d9c5f..251f5bb3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -217,7 +217,7 @@ class Document(BaseDocument): Returns True if the document has been updated or False if the document in the database doesn't match the query. - .. note:: All unsaved changes that has been made to the document are + .. note:: All unsaved changes that have been made to the document are rejected if the method returns True. :param query: the update will be performed only if the document in the @@ -403,7 +403,7 @@ class Document(BaseDocument): def cascade_save(self, *args, **kwargs): """Recursively saves any references / - generic references on an objects""" + generic references on the document""" _refs = kwargs.get('_refs', []) or [] ReferenceField = _import_class('ReferenceField') From 5c464c3f5a6a927a976c671d51d73b56a5f0a62f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 18 Nov 2015 14:07:39 +0200 Subject: [PATCH 19/99] Bumped version to 0.10.1 --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51f0967f..cd349793 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.10.1 - DEV +Changes in 0.10.1 ======================= - Fix infinite recursion with CASCADE delete rules under specific conditions. #1046 - Fix CachedReferenceField bug when loading cached docs as DBRef but failing to save them. #1047 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 474c2154..eefaf08d 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ import errors __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 10, 0) +VERSION = (0, 10, 1) def get_version(): From deb5677a5729a8213285f0637fc2d700c45301a7 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:14:45 -0500 Subject: [PATCH 20/99] Allow shard key to be in an embedded document (#551) --- mongoengine/document.py | 15 +++++++++++--- tests/document/instance.py | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d2d9c5f..bd2e7c5b 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -341,8 +341,12 @@ class Document(BaseDocument): select_dict['_id'] = object_id shard_key = self.__class__._meta.get('shard_key', tuple()) for k in shard_key: - actual_key = self._db_field_map.get(k, k) - select_dict[actual_key] = doc[actual_key] + path = self._lookup_field(k.split('.')) + actual_key = [p.db_field for p in path] + val = doc + for ak in actual_key: + val = val[ak] + select_dict['.'.join(actual_key)] = val def is_new_object(last_error): if last_error is not None: @@ -444,7 +448,12 @@ class Document(BaseDocument): select_dict = {'pk': self.pk} shard_key = self.__class__._meta.get('shard_key', tuple()) for k in shard_key: - select_dict[k] = getattr(self, k) + path = self._lookup_field(k.split('.')) + actual_key = [p.db_field for p in path] + val = self + for ak in actual_key: + val = getattr(val, ak) + select_dict['__'.join(actual_key)] = val return select_dict def update(self, **kwargs): diff --git a/tests/document/instance.py b/tests/document/instance.py index 56e6765a..5494ac6b 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -484,6 +484,20 @@ class InstanceTest(unittest.TestCase): doc.reload() Animal.drop_collection() + def test_reload_sharded_nested(self): + class SuperPhylum(EmbeddedDocument): + name = StringField() + + class Animal(Document): + superphylum = EmbeddedDocumentField(SuperPhylum) + meta = {'shard_key': ('superphylum.name',)} + + Animal.drop_collection() + doc = Animal(superphylum=SuperPhylum(name='Deuterostomia')) + doc.save() + doc.reload() + Animal.drop_collection() + def test_reload_referencing(self): """Ensures reloading updates weakrefs correctly """ @@ -2715,6 +2729,32 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_shard_key_in_embedded_document(self): + class Foo(EmbeddedDocument): + foo = StringField() + + class Bar(Document): + meta = { + 'shard_key': ('foo.foo',) + } + foo = EmbeddedDocumentField(Foo) + bar = StringField() + + foo_doc = Foo(foo='hello') + bar_doc = Bar(foo=foo_doc, bar='world') + bar_doc.save() + + self.assertTrue(bar_doc.id is not None) + + bar_doc.bar = 'baz' + bar_doc.save() + + def change_shard_key(): + bar_doc.foo.foo = 'something' + bar_doc.save() + + self.assertRaises(OperationError, change_shard_key) + def test_shard_key_primary(self): class LogEntry(Document): machine = StringField(primary_key=True) From f6d3bd8ccb692dd98b0ea45a5cd4f28a9068eff1 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:15:27 -0500 Subject: [PATCH 21/99] Update changelog for #551 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cd349793..6275d6d9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.10.2 +================= +- Allow shard key to point to a field in an embedded document. #551 + Changes in 0.10.1 ======================= - Fix infinite recursion with CASCADE delete rules under specific conditions. #1046 From 899e56e5b81e56f703b07df57a7e010f4c0eff32 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:15:40 -0500 Subject: [PATCH 22/99] Add gmacon to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4d5e69a3..411e274d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -229,3 +229,4 @@ that much better: * Emile Caron (https://github.com/emilecaron) * Amit Lichtenberg (https://github.com/amitlicht) * Lars Butler (https://github.com/larsbutler) + * George Macon (https://github.com/gmacon) From 0a1ba7c434cc4d437b7992a44b9dc73627f293a9 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Sat, 21 Nov 2015 10:25:11 +0100 Subject: [PATCH 23/99] Add SaveConditionError to __all__ --- mongoengine/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 2c5c2946..15830b5c 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -6,7 +6,7 @@ from mongoengine.python_support import txt_type __all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError', 'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError', 'OperationError', 'NotUniqueError', 'FieldDoesNotExist', - 'ValidationError') + 'ValidationError', 'SaveConditionError') class NotRegistered(Exception): From 19cbb442ee0755abde120a91150310824873a7af Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 23 Nov 2015 13:57:15 +0200 Subject: [PATCH 24/99] Added #1129 to the changelog. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6275d6d9..5809314c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 +- Allow arbirary metadata in fields. #1129 Changes in 0.10.1 ======================= From 013227323d8725e6ffd2e13aecfd0f3a9165d276 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 14:29:25 +0000 Subject: [PATCH 25/99] ReferenceFields can now reference abstract Document types A class that inherits from an abstract Document type is stored in the database as a reference with a 'cls' field that is the class name of the document being stored. Fixes #837 --- mongoengine/fields.py | 25 ++++++++++++++++----- tests/fields/fields.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f5899311..13538c89 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -928,9 +928,14 @@ class ReferenceField(BaseField): self._auto_dereference = instance._fields[self.name]._auto_dereference # Dereference DBRefs if self._auto_dereference and isinstance(value, DBRef): - value = self.document_type._get_db().dereference(value) + if hasattr(value, 'cls'): + # Dereference using the class type specified in the reference + cls = get_document(value.cls) + else: + cls = self.document_type + value = cls._get_db().dereference(value) if value is not None: - instance._data[self.name] = self.document_type._from_son(value) + instance._data[self.name] = cls._from_son(value) return super(ReferenceField, self).__get__(instance, owner) @@ -940,22 +945,30 @@ class ReferenceField(BaseField): return document.id return document - id_field_name = self.document_type._meta['id_field'] - id_field = self.document_type._fields[id_field_name] - if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk if id_ is None: self.error('You can only reference documents once they have' ' been saved to the database') + + # Use the attributes from the document instance, so that they + # override the attributes of this field's document type + cls = document else: id_ = document + cls = self.document_type + + id_field_name = cls._meta['id_field'] + id_field = cls._fields[id_field_name] id_ = id_field.to_mongo(id_) if self.dbref: - collection = self.document_type._get_collection_name() + collection = cls._get_collection_name() return DBRef(collection, id_) + elif self.document_type._meta.get('abstract'): + collection = cls._get_collection_name() + return DBRef(collection, id_, cls=cls._class_name) return id_ diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7ef298fc..860a5749 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2281,6 +2281,56 @@ class FieldTest(unittest.TestCase): Member.drop_collection() BlogPost.drop_collection() + def test_reference_class_with_abstract_parent(self): + """Ensure that a class with an abstract parent can be referenced. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Sister(Sibling): + pass + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + Sister.drop_collection() + Brother.drop_collection() + + sister = Sister(name="Alice") + sister.save() + brother = Brother(name="Bob", sibling=sister) + brother.save() + + self.assertEquals(Brother.objects[0].sibling.name, sister.name) + + Sister.drop_collection() + Brother.drop_collection() + + def test_reference_abstract_class(self): + """Ensure that an abstract class instance cannot be used in the + reference of that abstract class. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Sister(Sibling): + pass + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + Sister.drop_collection() + Brother.drop_collection() + + sister = Sibling(name="Alice") + brother = Brother(name="Bob", sibling=sister) + self.assertRaises(ValidationError, brother.save) + + Sister.drop_collection() + Brother.drop_collection() + def test_generic_reference(self): """Ensure that a GenericReferenceField properly dereferences items. """ From f96e68cd11c72ab321bc1b7923bdfffd29f3bb11 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 15:02:19 +0000 Subject: [PATCH 26/99] Made type inheritance a validation check for abstract references --- mongoengine/fields.py | 8 ++++++++ tests/fields/fields.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 13538c89..4c8c8498 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -996,6 +996,14 @@ class ReferenceField(BaseField): self.error('You can only reference documents once they have been ' 'saved to the database') + if self.document_type._meta.get('abstract') and \ + not isinstance(value, self.document_type): + self.error('%s is not an instance of abstract reference' + ' type %s' % (value._class_name, + self.document_type._class_name) + ) + + def lookup_member(self, member_name): return self.document_type._fields.get(member_name) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 860a5749..15daecb9 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2331,6 +2331,31 @@ class FieldTest(unittest.TestCase): Sister.drop_collection() Brother.drop_collection() + def test_abstract_reference_base_type(self): + """Ensure that an an abstract reference fails validation when given a + Document that does not inherit from the abstract type. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + class Mother(Document): + name = StringField() + + Brother.drop_collection() + Mother.drop_collection() + + mother = Mother(name="Carol") + mother.save() + brother = Brother(name="Bob", sibling=mother) + self.assertRaises(ValidationError, brother.save) + + Brother.drop_collection() + Mother.drop_collection() + def test_generic_reference(self): """Ensure that a GenericReferenceField properly dereferences items. """ From aa9d5969301964c3e25fe5205b4d25ce1a9dd296 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 15:03:15 +0000 Subject: [PATCH 27/99] Updated documentation for abstract reference changes --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 411e274d..fbca84e4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -230,3 +230,4 @@ that much better: * Amit Lichtenberg (https://github.com/amitlicht) * Lars Butler (https://github.com/larsbutler) * George Macon (https://github.com/gmacon) + * Ashley Whetter (https://github.com/AWhetter) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5809314c..37720431 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 +- ReferenceFields now support abstract document types. #837 Changes in 0.10.1 ======================= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4c8c8498..6ef13640 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -896,6 +896,10 @@ class ReferenceField(BaseField): or as the :class:`~pymongo.objectid.ObjectId`.id . :param reverse_delete_rule: Determines what to do when the referring object is deleted + + .. note :: + A reference to an abstract document type is always stored as a + :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`. """ if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): From 04497aec36232c893755698911edff31c044f192 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Wed, 11 Nov 2015 09:24:31 +0000 Subject: [PATCH 28/99] Fixed setting dbref to True on abstract reference fields causing the reference to be stored incorrectly --- mongoengine/fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 6ef13640..f89d7d8b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -967,12 +967,12 @@ class ReferenceField(BaseField): id_field = cls._fields[id_field_name] id_ = id_field.to_mongo(id_) - if self.dbref: - collection = cls._get_collection_name() - return DBRef(collection, id_) - elif self.document_type._meta.get('abstract'): + if self.document_type._meta.get('abstract'): collection = cls._get_collection_name() return DBRef(collection, id_, cls=cls._class_name) + elif self.dbref: + collection = cls._get_collection_name() + return DBRef(collection, id_) return id_ From 3c0b00e42d63a3f51842b364a5f69f353d10b466 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 23 Nov 2015 15:40:16 +0200 Subject: [PATCH 29/99] Added python 3.5 to the build. --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 34702192..a0d75551 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - '3.2' - '3.3' - '3.4' +- '3.5' - pypy - pypy3 env: diff --git a/tox.ini b/tox.ini index e6aa7c81..2f84d668 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27,py32,py33,py34,pypy,pypy3}-{mg27,mg28} +envlist = {py26,py27,py32,py33,py34,py35,pypy,pypy3}-{mg27,mg28} #envlist = {py26,py27,py32,py33,py34,pypy,pypy3}-{mg27,mg28,mg30,mgdev} [testenv] From a7aead5138f8792e4030328e23531ee7d4024744 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 24 Jun 2015 22:34:38 -0700 Subject: [PATCH 30/99] re-create the cursor object whenever we apply read_preference --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 1 + tests/queryset/queryset.py | 48 +++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 37720431..1ea4d19f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changes in 0.10.2 - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 - ReferenceFields now support abstract document types. #837 +- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 Changes in 0.10.1 ======================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0867e0e2..8880c2e3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -930,6 +930,7 @@ class BaseQuerySet(object): validate_read_preference('read_preference', read_preference) queryset = self.clone() queryset._read_preference = read_preference + queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_preference return queryset def scalar(self, *fields): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3eff7cea..8726801e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4165,7 +4165,11 @@ class QuerySetTest(unittest.TestCase): def test_read_preference(self): class Bar(Document): - pass + txt = StringField() + + meta = { + 'indexes': [ 'txt' ] + } Bar.drop_collection() bars = list(Bar.objects(read_preference=ReadPreference.PRIMARY)) @@ -4177,9 +4181,51 @@ class QuerySetTest(unittest.TestCase): error_class = TypeError self.assertRaises(error_class, Bar.objects, read_preference='Primary') + # read_preference as a kwarg bars = Bar.objects(read_preference=ReadPreference.SECONDARY_PREFERRED) self.assertEqual( bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference as a query set method + bars = Bar.objects.read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after skip + bars = Bar.objects.skip(1) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after limit + bars = Bar.objects.limit(1) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after order_by + bars = Bar.objects.order_by('txt') \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after hint + bars = Bar.objects.hint([('txt', 1)]) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) def test_json_simple(self): From 54975de0f387ad02a75134a4275401f684b855d7 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Sun, 5 Jul 2015 15:59:50 -0700 Subject: [PATCH 31/99] fix read_preference for PyMongo 3+ --- mongoengine/queryset/base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 8880c2e3..ac4764d3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1444,8 +1444,16 @@ class BaseQuerySet(object): def _cursor(self): if self._cursor_obj is None: - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) + # In PyMongo 3+, we define the read preference on a collection + # level, not a cursor level. Thus, we need to get a cloned + # collection object using `with_options` first. + if IS_PYMONGO_3 and self._read_preference is not None: + self._cursor_obj = self._collection\ + .with_options(read_preference=self._read_preference)\ + .find(self._query, **self._cursor_args) + else: + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) # Apply where clauses to cursor if self._where_clause: where_clause = self._sub_js_fields(self._where_clause) From f9284d20cae1b2596c164493f7bac0f240ade4d5 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Nov 2015 07:00:09 +0200 Subject: [PATCH 32/99] Moved #1042 to the next version in the changelog. --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1ea4d19f..a0c49900 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,12 +2,15 @@ Changelog ========= +Changes in 0.10.3 +================= +- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 + Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 - ReferenceFields now support abstract document types. #837 -- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 Changes in 0.10.1 ======================= From a6e996d92142866a1d18570fe60d91eca806e14d Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Nov 2015 07:06:54 +0200 Subject: [PATCH 33/99] Added #1165 to the changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0c49900..c9285416 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.10.4 - DEV +======================= +- SaveConditionError is now importable from the top level package. #1165 + Changes in 0.10.3 ================= - Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 From b7b28390df2f0684b4a1af7c91dbaef60a8fdab5 Mon Sep 17 00:00:00 2001 From: srossiter Date: Tue, 24 Nov 2015 12:46:38 +0000 Subject: [PATCH 34/99] Added upsert_one method on BaseQuerySet and modified test_upsert_one --- AUTHORS | 1 + mongoengine/__init__.py | 2 +- mongoengine/queryset/base.py | 27 +++++++++++++++++++++++++++ tests/queryset/queryset.py | 13 +++++++++++-- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index fbca84e4..dd04aee1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -231,3 +231,4 @@ that much better: * Lars Butler (https://github.com/larsbutler) * George Macon (https://github.com/gmacon) * Ashley Whetter (https://github.com/AWhetter) + * Steven Rossiter (https://github.com/BeardedSteve) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index eefaf08d..09fb26dc 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ import errors __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 10, 1) +VERSION = (0, 10, 2) def get_version(): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index ac4764d3..5c44db9c 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -471,6 +471,33 @@ class BaseQuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) + + def upsert_one(self, write_concern=None, **update): + """Perform an atomic upsert on the fields of the first document + matched by the query. + + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param update: Django-style update keyword arguments + + :returns the new or overwritten document + + .. versionadded:: 10.0.2 + """ + + update = self.update(multi=False, upsert=True, write_concern=write_concern, + full_result=True,**update) + + if update['updatedExisting']: + document = self.get() + else: + document = self._document.objects.with_id(update['upserted']) + return document + def update_one(self, upsert=False, write_concern=None, **update): """Perform an atomic update on the fields of the first document matched by the query. diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 8726801e..87c14757 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -680,12 +680,21 @@ class QuerySetTest(unittest.TestCase): def test_upsert_one(self): self.Person.drop_collection() - self.Person.objects(name="Bob", age=30).update_one(upsert=True) + bob = self.Person.objects(name="Bob", age=30).upsert_one() - bob = self.Person.objects.first() self.assertEqual("Bob", bob.name) self.assertEqual(30, bob.age) + bob.name = "Bobby" + bob.save() + + bobby = self.Person.objects(name="Bobby", age=30).upsert_one() + + self.assertEqual("Bobby", bobby.name) + self.assertEqual(30, bobby.age) + self.assertEqual(bob.id, bobby.id) + + def test_set_on_insert(self): self.Person.drop_collection() From 164e2b2678a7d88deb00aa353662e69ad7a6100c Mon Sep 17 00:00:00 2001 From: srossiter Date: Tue, 24 Nov 2015 12:53:09 +0000 Subject: [PATCH 35/99] Docstring change and rename variable to avoid clash with kwargs --- mongoengine/queryset/base.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 5c44db9c..ce4174b9 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -473,8 +473,7 @@ class BaseQuerySet(object): def upsert_one(self, write_concern=None, **update): - """Perform an atomic upsert on the fields of the first document - matched by the query. + """Overwrite or add the first document matched by the query. :param write_concern: Extra keyword arguments are passed down which will be used as options for the resultant @@ -486,16 +485,16 @@ class BaseQuerySet(object): :returns the new or overwritten document - .. versionadded:: 10.0.2 + .. versionadded:: 0.10.2 """ - update = self.update(multi=False, upsert=True, write_concern=write_concern, + atomic_update = self.update(multi=False, upsert=True, write_concern=write_concern, full_result=True,**update) - if update['updatedExisting']: + if atomic_update['updatedExisting']: document = self.get() else: - document = self._document.objects.with_id(update['upserted']) + document = self._document.objects.with_id(atomic_update['upserted']) return document def update_one(self, upsert=False, write_concern=None, **update): From fc3db7942d2a2cd0016776268ebc8263c9b44dab Mon Sep 17 00:00:00 2001 From: srossiter Date: Tue, 24 Nov 2015 12:56:59 +0000 Subject: [PATCH 36/99] updated changelog and version tuple --- docs/changelog.rst | 1 + mongoengine/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c9285416..7776a99b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.4 - DEV ======================= - SaveConditionError is now importable from the top level package. #1165 +- upsert_one method added. #1157 Changes in 0.10.3 ================= diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 09fb26dc..14a7c1ce 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ import errors __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 10, 2) +VERSION = (0, 10, 4) def get_version(): From 2af8342fea1702a247de9587a9494e48bf73b9af Mon Sep 17 00:00:00 2001 From: hhstore Date: Thu, 26 Nov 2015 12:01:42 +0800 Subject: [PATCH 37/99] bugfix - two small bugs. --- docs/code/tumblelog.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/code/tumblelog.py b/docs/code/tumblelog.py index 0e40e899..c10160ea 100644 --- a/docs/code/tumblelog.py +++ b/docs/code/tumblelog.py @@ -17,6 +17,10 @@ class Post(Document): tags = ListField(StringField(max_length=30)) comments = ListField(EmbeddedDocumentField(Comment)) + # bugfix + meta = {'allow_inheritance': True} + + class TextPost(Post): content = StringField() @@ -45,7 +49,8 @@ print 'ALL POSTS' print for post in Post.objects: print post.title - print '=' * post.title.count() + #print '=' * post.title.count() + print "=" * 20 if isinstance(post, TextPost): print post.content From 3c18f79ea45996e715c4d9af394c7d23947c2ffe Mon Sep 17 00:00:00 2001 From: Paul-Armand Verhaegen Date: Fri, 27 Nov 2015 23:45:25 +0100 Subject: [PATCH 38/99] Added test for reloading of strict with special fields #1156 --- tests/document/instance.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/document/instance.py b/tests/document/instance.py index 56e6765a..fe8ed1bd 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -557,6 +557,28 @@ class InstanceTest(unittest.TestCase): except Exception: self.assertFalse("Threw wrong exception") + def test_reload_of_non_strict_with_special_field_name(self): + """Ensures reloading works for documents with meta strict == False + """ + class Post(Document): + meta = { + 'strict': False + } + title = StringField() + items = ListField() + + Post.drop_collection() + + Post._get_collection().insert_one({ + "title": "Items eclipse", + "items": ["more lorem", "even more ipsum"] + }) + + post = Post.objects.first() + post.reload() + self.assertEqual(post.title, "Items eclipse") + self.assertEqual(post.items, ["more lorem", "even more ipsum"]) + def test_dictionary_access(self): """Ensure that dictionary-style field access works properly. """ From 23f07fde5e88c42c42f26ca48edd10456c7babbf Mon Sep 17 00:00:00 2001 From: RussellLuo Date: Sun, 8 Nov 2015 22:11:47 +0800 Subject: [PATCH 39/99] Add support for mocking MongoEngine based on mongomock Using `mongomock://` scheme in URI enables the mocking. Fix #1045. --- mongoengine/connection.py | 21 ++++++++++++++++++--- tests/test_connection.py | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 4abca1ab..2ea572df 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -54,8 +54,11 @@ def register_connection(alias, name=None, host=None, port=None, } # Handle uri style connections - if "://" in conn_settings['host']: - uri_dict = uri_parser.parse_uri(conn_settings['host']) + conn_host = conn_settings['host'] + if conn_host.startswith('mongomock://'): + conn_settings['is_mock'] = True + elif '://' in conn_host: + uri_dict = uri_parser.parse_uri(conn_host) conn_settings.update({ 'name': uri_dict.get('database') or name, 'username': uri_dict.get('username'), @@ -106,7 +109,19 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings.pop('password', None) conn_settings.pop('authentication_source', None) - connection_class = MongoClient + is_mock = conn_settings.pop('is_mock', None) + if is_mock: + # Use MongoClient from mongomock + try: + import mongomock + except ImportError: + raise RuntimeError('You need mongomock installed ' + 'to mock MongoEngine.') + connection_class = mongomock.MongoClient + else: + # Use MongoClient from pymongo + connection_class = MongoClient + if 'replicaSet' in conn_settings: # Discard port since it can't be used on MongoReplicaSetClient conn_settings.pop('port', None) diff --git a/tests/test_connection.py b/tests/test_connection.py index 1b7b7a22..4466ee73 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -8,6 +8,7 @@ try: import unittest2 as unittest except ImportError: import unittest +from nose.plugins.skip import SkipTest import pymongo from bson.tz_util import utc @@ -51,6 +52,22 @@ class ConnectionTest(unittest.TestCase): conn = get_connection('testdb') self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) + def test_connect_in_mocking(self): + """Ensure that the connect() method works properly in mocking. + """ + try: + import mongomock + except ImportError: + raise SkipTest('you need mongomock installed to run this testcase') + + connect('mongoenginetest', host='mongomock://localhost') + conn = get_connection() + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect('mongoenginetest2', host='mongomock://localhost', alias='testdb') + conn = get_connection('testdb') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + def test_disconnect(self): """Ensure that the disconnect() method works properly """ @@ -151,7 +168,7 @@ class ConnectionTest(unittest.TestCase): self.assertRaises(ConnectionError, get_db, 'test1') # Authentication succeeds with "authSource" - test_conn2 = connect( + connect( 'mongoenginetest', alias='test2', host=('mongodb://username2:password@localhost/' 'mongoenginetest?authSource=admin') From a81d6d124b7f6f3dc17794fae32fe0f3959158b6 Mon Sep 17 00:00:00 2001 From: RussellLuo Date: Sun, 6 Dec 2015 11:11:46 +0800 Subject: [PATCH 40/99] Update AUTHORS and add changelog entry for #1151 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 66e181c9..b087f41c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -233,3 +233,4 @@ that much better: * Ashley Whetter (https://github.com/AWhetter) * Paul-Armand Verhaegen (https://github.com/paularmand) * Steven Rossiter (https://github.com/BeardedSteve) + * Luo Peng (https://github.com/RussellLuo) diff --git a/docs/changelog.rst b/docs/changelog.rst index c354be18..13b14a0f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.5 ================= - Fix for reloading of strict with special fields. #1156 +- Add support for mocking MongoEngine based on mongomock. #1151 Changes in 0.10.4 ================= From 713af133a09ba6c5b98dcc743ead3e1decc86124 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 6 Dec 2015 07:54:25 +0200 Subject: [PATCH 41/99] Moved #1151 changelog entry to the correct version. --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 13b14a0f..353a2a0c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,10 +2,13 @@ Changelog ========= +Changes in 0.10.6 - Dev +======================= +- Add support for mocking MongoEngine based on mongomock. #1151 + Changes in 0.10.5 ================= - Fix for reloading of strict with special fields. #1156 -- Add support for mocking MongoEngine based on mongomock. #1151 Changes in 0.10.4 ================= From a3886702a396158cc8bf6e4c34a5428eca236c24 Mon Sep 17 00:00:00 2001 From: moonso Date: Sun, 6 Dec 2015 11:02:26 +0100 Subject: [PATCH 42/99] Added page for documenting mongomock. Updated docs/guide/index.rst --- docs/guide/index.rst | 1 + docs/guide/mongomock.rst | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/guide/mongomock.rst diff --git a/docs/guide/index.rst b/docs/guide/index.rst index c4077888..46eb7af2 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -13,3 +13,4 @@ User Guide gridfs signals text-indexes + mongomock diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst new file mode 100644 index 00000000..9471e94e --- /dev/null +++ b/docs/guide/mongomock.rst @@ -0,0 +1,21 @@ +============================== +Use mongomock for testing +============================== + +`mongomock `_ is a package to do just +what the name implies, mocking a mongo database. + +To use with mongoengine, simply specify mongomock when connecting with +mongoengine: + +.. code-block:: python + + connect('mongoenginetest', host='mongomock://localhost') + conn = get_connection() + +or with an alias: + +.. code-block:: python + + connect('mongoenginetest', host='mongomock://localhost', alias='testdb') + conn = get_connection('testdb) From 04e8b83d4586a94ba3db5e342ba060e16e388b7e Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 11:05:30 +0000 Subject: [PATCH 43/99] Fixed being unable to run tests on Windows --- docs/changelog.rst | 1 + tox.ini | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 353a2a0c..0dcb7e94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.6 - Dev ======================= - Add support for mocking MongoEngine based on mongomock. #1151 +- Fixed not being able to run tests on Windows. #1153 Changes in 0.10.5 ================= diff --git a/tox.ini b/tox.ini index 2f84d668..124c8843 100644 --- a/tox.ini +++ b/tox.ini @@ -12,3 +12,6 @@ deps = mg28: PyMongo>=2.8,<3.0 mg30: PyMongo>=3.0 mgdev: https://github.com/mongodb/mongo-python-driver/tarball/master +setenv = + PYTHON_EGG_CACHE = {envdir}/python-eggs +passenv = windir From 9c264611cf9af99057ac31d3f863fa300da8e2d7 Mon Sep 17 00:00:00 2001 From: bitdivision Date: Wed, 9 Dec 2015 11:40:22 +0000 Subject: [PATCH 44/99] Add EmbeddedDocumentListField to user guide The 'defining a document' section currently doesn't include EmbeddedDocumentListField. Only EmbeddedDocumentField --- docs/guide/defining-documents.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index c3ad208e..1a5fbfc9 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -75,6 +75,7 @@ are as follows: * :class:`~mongoengine.fields.DynamicField` * :class:`~mongoengine.fields.EmailField` * :class:`~mongoengine.fields.EmbeddedDocumentField` +* :class:`~mongoengine.fields.EmbeddedDocumentListField` * :class:`~mongoengine.fields.FileField` * :class:`~mongoengine.fields.FloatField` * :class:`~mongoengine.fields.GenericEmbeddedDocumentField` From 00221e341008913cae7c309eb9ba7487fcffd4de Mon Sep 17 00:00:00 2001 From: George Macon Date: Wed, 16 Sep 2015 15:27:57 -0400 Subject: [PATCH 45/99] Allow sparse compound indexes --- docs/changelog.rst | 1 + mongoengine/base/document.py | 4 ---- tests/document/indexes.py | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0dcb7e94..df984059 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changes in 0.10.6 - Dev ======================= - Add support for mocking MongoEngine based on mongomock. #1151 - Fixed not being able to run tests on Windows. #1153 +- Allow creation of sparse compound indexes. #1114 Changes in 0.10.5 ================= diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 6353162a..05b3ad95 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -839,10 +839,6 @@ class BaseDocument(object): if index_list: spec['fields'] = index_list - if spec.get('sparse', False) and len(spec['fields']) > 1: - raise ValueError( - 'Sparse indexes can only have one field in them. ' - 'See https://jira.mongodb.org/browse/SERVER-2193') return spec diff --git a/tests/document/indexes.py b/tests/document/indexes.py index ccc6cf44..7ee017a6 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -863,6 +863,20 @@ class IndexesTest(unittest.TestCase): self.assertTrue([('provider_ids.foo', 1)] in info) self.assertTrue([('provider_ids.bar', 1)] in info) + def test_sparse_compound_indexes(self): + + class MyDoc(Document): + provider_ids = DictField() + meta = { + "indexes": [{'fields': ("provider_ids.foo", "provider_ids.bar"), + 'sparse': True}], + } + + info = MyDoc.objects._collection.index_information() + self.assertEqual([('provider_ids.foo', 1), ('provider_ids.bar', 1)], + info['provider_ids.foo_1_provider_ids.bar_1']['key']) + self.assertTrue(info['provider_ids.foo_1_provider_ids.bar_1']['sparse']) + def test_text_indexes(self): class Book(Document): From 35d3d3de724af4dec758be1c02fb56f0600b16b8 Mon Sep 17 00:00:00 2001 From: David Bordeynik Date: Tue, 15 Dec 2015 22:26:30 +0200 Subject: [PATCH 46/99] fix-#1187: count on ListField of EmbeddedDocumentField fails --- docs/changelog.rst | 1 + mongoengine/fields.py | 2 +- tests/queryset/queryset.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df984059..857835ae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changes in 0.10.6 - Dev - Add support for mocking MongoEngine based on mongomock. #1151 - Fixed not being able to run tests on Windows. #1153 - Allow creation of sparse compound indexes. #1114 +- count on ListField of EmbeddedDocumentField fails. #1187 Changes in 0.10.5 ================= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3887fd84..07209d0b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -697,7 +697,7 @@ class ListField(ComplexBaseField): def prepare_query_value(self, op, value): if self.field: - if op in ('set', 'unset') and ( + if op in ('set', 'unset', None) and ( not isinstance(value, basestring) and not isinstance(value, BaseDocument) and hasattr(value, '__iter__')): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 87c14757..39ec064c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3613,6 +3613,15 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(MyDoc.objects.count(), 10) self.assertEqual(MyDoc.objects.none().count(), 0) + def test_count_list_embedded(self): + class B(EmbeddedDocument): + c = StringField() + + class A(Document): + b = ListField(EmbeddedDocumentField(B)) + + self.assertEqual(A.objects(b=[{'c': 'c'}]).count(), 0) + def test_call_after_limits_set(self): """Ensure that re-filtering after slicing works """ @@ -4888,5 +4897,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(1, Doc.objects(item__type__="axe").count()) + if __name__ == '__main__': unittest.main() From acc7448dc50eef6f90edf5b16e992f0b7635466d Mon Sep 17 00:00:00 2001 From: Nick Pjetrovic Date: Thu, 24 Dec 2015 18:30:46 -0500 Subject: [PATCH 47/99] Fix pre_bulk_insert signal --- mongoengine/queryset/base.py | 4 +-- tests/test_signals.py | 51 +++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index ce4174b9..489a4994 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -296,7 +296,6 @@ class BaseQuerySet(object): return_one = True docs = [docs] - raw = [] for doc in docs: if not isinstance(doc, self._document): msg = ("Some documents inserted aren't instances of %s" @@ -305,9 +304,10 @@ class BaseQuerySet(object): if doc.pk and not doc._created: msg = "Some documents have ObjectIds use doc.update() instead" raise OperationError(msg) - raw.append(doc.to_mongo()) signals.pre_bulk_insert.send(self._document, documents=docs) + + raw = [doc.to_mongo() for doc in docs] try: ids = self._collection.insert(raw, **write_concern) except pymongo.errors.DuplicateKeyError, err: diff --git a/tests/test_signals.py b/tests/test_signals.py index 8672925c..78305b7d 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -118,6 +118,35 @@ class SignalTests(unittest.TestCase): self.ExplicitId = ExplicitId ExplicitId.drop_collection() + class Post(Document): + title = StringField() + content = StringField() + active = BooleanField(default=False) + + def __unicode__(self): + return self.title + + @classmethod + def pre_bulk_insert(cls, sender, documents, **kwargs): + signal_output.append('pre_bulk_insert signal, %s' % + [(doc, {'active': documents[n].active}) + for n, doc in enumerate(documents)]) + + # make changes here, this is just an example - + # it could be anything that needs pre-validation or looks-ups before bulk bulk inserting + for document in documents: + if not document.active: + document.active = True + + @classmethod + def post_bulk_insert(cls, sender, documents, **kwargs): + signal_output.append('post_bulk_insert signal, %s' % + [(doc, {'active': documents[n].active}) + for n, doc in enumerate(documents)]) + + self.Post = Post + Post.drop_collection() + # Save up the number of connected signals so that we can check at the # end that all the signals we register get properly unregistered self.pre_signals = ( @@ -147,6 +176,9 @@ class SignalTests(unittest.TestCase): signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId) + signals.pre_bulk_insert.connect(Post.pre_bulk_insert, sender=Post) + signals.post_bulk_insert.connect(Post.post_bulk_insert, sender=Post) + def tearDown(self): signals.pre_init.disconnect(self.Author.pre_init) signals.post_init.disconnect(self.Author.post_init) @@ -163,6 +195,9 @@ class SignalTests(unittest.TestCase): signals.post_save.disconnect(self.ExplicitId.post_save) + signals.pre_bulk_insert.disconnect(self.Post.pre_bulk_insert) + signals.post_bulk_insert.disconnect(self.Post.post_bulk_insert) + # Check that all our signals got disconnected properly. post_signals = ( len(signals.pre_init.receivers), @@ -199,7 +234,7 @@ class SignalTests(unittest.TestCase): a.save() self.get_signal_output(lambda: None) # eliminate signal output a1 = self.Author.objects(name='Bill Shakespeare')[0] - + self.assertEqual(self.get_signal_output(create_author), [ "pre_init signal, Author", "{'name': 'Bill Shakespeare'}", @@ -306,6 +341,20 @@ class SignalTests(unittest.TestCase): ei.switch_db("testdb-1", keep_created=False) self.assertEqual(self.get_signal_output(ei.save), ['Is created']) + def test_signals_bulk_insert(self): + def bulk_set_active_post(): + posts = [ + self.Post(title='Post 1'), + self.Post(title='Post 2'), + self.Post(title='Post 3') + ] + self.Post.objects.insert(posts) + + results = self.get_signal_output(bulk_set_active_post) + self.assertEqual(results, [ + "pre_bulk_insert signal, [(, {'active': False}), (, {'active': False}), (, {'active': False})]", + "post_bulk_insert signal, [(, {'active': True}), (, {'active': True}), (, {'active': True})]" + ]) if __name__ == '__main__': unittest.main() From 3d80e549cb52939bfbdc9a1d4382bc5b80d4ddb3 Mon Sep 17 00:00:00 2001 From: Ruslan Nassonov Date: Fri, 25 Dec 2015 15:52:35 +0500 Subject: [PATCH 48/99] fix missing quote in /docs/guide/mongomock.rst --- docs/guide/mongomock.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst index 9471e94e..1d5227ec 100644 --- a/docs/guide/mongomock.rst +++ b/docs/guide/mongomock.rst @@ -18,4 +18,4 @@ or with an alias: .. code-block:: python connect('mongoenginetest', host='mongomock://localhost', alias='testdb') - conn = get_connection('testdb) + conn = get_connection('testdb') From 8c81f7ece99517703249051dc8462b50c81478a7 Mon Sep 17 00:00:00 2001 From: Zephor Date: Wed, 6 Jan 2016 12:00:48 +0800 Subject: [PATCH 49/99] add highlight for python code --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f4c92d5f..08e95029 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,9 @@ Optional Dependencies Examples ======== -Some simple examples of what MongoEngine code looks like:: +Some simple examples of what MongoEngine code looks like: + +.. code :: python class BlogPost(Document): title = StringField(required=True, max_length=200) From 0284975f3fe408bec3a64afba66471b2973b5010 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Tue, 19 Jan 2016 15:34:38 +0100 Subject: [PATCH 50/99] Correct test_reload_of_non_strict_with_special_field_name for pymongo<2.9 --- tests/document/instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index deb4e6ac..f463ec31 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -583,7 +583,7 @@ class InstanceTest(unittest.TestCase): Post.drop_collection() - Post._get_collection().insert_one({ + Post._get_collection().insert({ "title": "Items eclipse", "items": ["more lorem", "even more ipsum"] }) From 13897db6d3a953077a3fc4c4d555141e959d42da Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Wed, 20 Jan 2016 11:06:45 +0100 Subject: [PATCH 51/99] Fix mongomock url prefix error during connection --- mongoengine/connection.py | 5 +++++ tests/test_connection.py | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index cde3ccd4..4055a9b6 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -38,8 +38,11 @@ def register_connection(alias, name=None, host=None, port=None, :param username: username to authenticate with :param password: password to authenticate with :param authentication_source: database to authenticate against + :param is_mock: explicitly use mongomock for this connection + (can also be done by using `mongomock://` as db host prefix) :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver + .. versionchanged:: 0.10.6 - added mongomock support """ global _connection_settings @@ -57,6 +60,8 @@ def register_connection(alias, name=None, host=None, port=None, conn_host = conn_settings['host'] if conn_host.startswith('mongomock://'): conn_settings['is_mock'] = True + # `mongomock://` is not a valid url prefix and must be replaced by `mongodb://` + conn_settings['host'] = conn_host.replace('mongomock://', 'mongodb://', 1) elif '://' in conn_host: uri_dict = uri_parser.parse_uri(conn_host) conn_settings.update({ diff --git a/tests/test_connection.py b/tests/test_connection.py index 4466ee73..b2f7406e 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -64,8 +64,28 @@ class ConnectionTest(unittest.TestCase): conn = get_connection() self.assertTrue(isinstance(conn, mongomock.MongoClient)) - connect('mongoenginetest2', host='mongomock://localhost', alias='testdb') - conn = get_connection('testdb') + connect('mongoenginetest2', host='mongomock://localhost', alias='testdb2') + conn = get_connection('testdb2') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect('mongoenginetest3', host='mongodb://localhost', is_mock=True, alias='testdb3') + conn = get_connection('testdb3') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect('mongoenginetest4', is_mock=True, alias='testdb4') + conn = get_connection('testdb4') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect(host='mongodb://localhost:27017/mongoenginetest5', is_mock=True, alias='testdb5') + conn = get_connection('testdb5') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect(host='mongomock://localhost:27017/mongoenginetest6', alias='testdb6') + conn = get_connection('testdb6') + self.assertTrue(isinstance(conn, mongomock.MongoClient)) + + connect(host='mongomock://localhost:27017/mongoenginetest7', is_mock=True, alias='testdb7') + conn = get_connection('testdb7') self.assertTrue(isinstance(conn, mongomock.MongoClient)) def test_disconnect(self): From 7b7165f5d8ab9cc29c29a02fbdd547113edb30b9 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Wed, 20 Jan 2016 11:48:31 +0100 Subject: [PATCH 52/99] Force pip version to 7.1.2 in tox for py32 (support dropped for latter versions) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 124c8843..eae0767c 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = mg28: PyMongo>=2.8,<3.0 mg30: PyMongo>=3.0 mgdev: https://github.com/mongodb/mongo-python-driver/tarball/master + py32: pip==7.1.2 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs passenv = windir From f4fa39c70ede0599d317e3b30f3b96da3f1d4249 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Wed, 20 Jan 2016 13:07:06 +0100 Subject: [PATCH 53/99] Revert "Force pip version to 7.1.2 in tox for py32 (support dropped for latter versions)" This reverts commit 7b7165f5d8ab9cc29c29a02fbdd547113edb30b9. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index eae0767c..124c8843 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ deps = mg28: PyMongo>=2.8,<3.0 mg30: PyMongo>=3.0 mgdev: https://github.com/mongodb/mongo-python-driver/tarball/master - py32: pip==7.1.2 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs passenv = windir From feb5eed8a5db8799bfd4c86cda40d2f0b63071cd Mon Sep 17 00:00:00 2001 From: Bastien Date: Thu, 21 Jan 2016 16:59:37 +0100 Subject: [PATCH 54/99] fixed minor typo in docstring --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index c2ad027e..f7800197 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1109,7 +1109,7 @@ class BaseQuerySet(object): Can only do direct simple mappings and cannot map across :class:`~mongoengine.fields.ReferenceField` or :class:`~mongoengine.fields.GenericReferenceField` for more complex - counting a manual map reduce call would is required. + counting a manual map reduce call is required. If the field is a :class:`~mongoengine.fields.ListField`, the items within each list will be counted individually. From b4af8ec751a17fe6c819c2e129e89e7005a1d0ae Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Fri, 22 Jan 2016 08:38:42 +0100 Subject: [PATCH 55/99] Fix travis for python 3.2 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0d75551..0af5c269 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,8 @@ install: - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk -- travis_retry pip install tox>=1.9 coveralls +# virtualenv>=14.0.0 has dropped Python 3.2 support +- travis_retry pip install "virtualenv<14.0.0" "tox>=1.9" coveralls - travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test script: - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage From a20d40618f5c40e63e98efd57f0733592a9fe064 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Mon, 25 Jan 2016 01:42:19 +0100 Subject: [PATCH 56/99] Bump to v0.10.6 --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df984059..b92b9cec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.10.6 - Dev -======================= +Changes in 0.10.6 +================= - Add support for mocking MongoEngine based on mongomock. #1151 - Fixed not being able to run tests on Windows. #1153 - Allow creation of sparse compound indexes. #1114 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 791e741e..65250b62 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ import errors __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 10, 5) +VERSION = (0, 10, 6) def get_version(): From 6c9e1799c703fa0ab0f568c6da08b3990679ab64 Mon Sep 17 00:00:00 2001 From: Bryan Bennett Date: Wed, 20 Jan 2016 16:42:57 -0500 Subject: [PATCH 57/99] MongoEngine/mongoengine #1217: Curry **kwargs through to_mongo on fields --- mongoengine/base/document.py | 21 +++++++--------- mongoengine/base/fields.py | 22 +++++++++-------- mongoengine/fields.py | 46 ++++++++++++++++++------------------ tests/document/instance.py | 13 ++++++++++ tests/fields/fields.py | 2 +- 5 files changed, 58 insertions(+), 46 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 05b3ad95..a890081a 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -325,20 +325,17 @@ class BaseDocument(object): if value is not None: - if isinstance(field, EmbeddedDocumentField): - if fields: - key = '%s.' % field_name - embedded_fields = [ - i.replace(key, '') for i in fields - if i.startswith(key)] + if fields: + key = '%s.' % field_name + embedded_fields = [ + i.replace(key, '') for i in fields + if i.startswith(key)] - else: - embedded_fields = [] - - value = field.to_mongo(value, use_db_field=use_db_field, - fields=embedded_fields) else: - value = field.to_mongo(value) + embedded_fields = [] + + value = field.to_mongo(value, use_db_field=use_db_field, + fields=embedded_fields) # Handle self generating fields if value is None and field._auto_gen: diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index b7b8d0de..4b2d0e8f 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -158,7 +158,7 @@ class BaseField(object): """ return value - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): """Convert a Python type to a MongoDB-compatible type. """ return self.to_python(value) @@ -325,7 +325,7 @@ class ComplexBaseField(BaseField): key=operator.itemgetter(0))] return value_dict - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): """Convert a Python type to a MongoDB-compatible type. """ Document = _import_class("Document") @@ -337,9 +337,10 @@ class ComplexBaseField(BaseField): if hasattr(value, 'to_mongo'): if isinstance(value, Document): - return GenericReferenceField().to_mongo(value) + return GenericReferenceField().to_mongo( + value, **kwargs) cls = value.__class__ - val = value.to_mongo() + val = value.to_mongo(**kwargs) # If it's a document that is not inherited add _cls if isinstance(value, EmbeddedDocument): val['_cls'] = cls.__name__ @@ -354,7 +355,7 @@ class ComplexBaseField(BaseField): return value if self.field: - value_dict = dict([(key, self.field.to_mongo(item)) + value_dict = dict([(key, self.field.to_mongo(item, **kwargs)) for key, item in value.iteritems()]) else: value_dict = {} @@ -373,19 +374,20 @@ class ComplexBaseField(BaseField): meta.get('allow_inheritance', ALLOW_INHERITANCE) is True) if not allow_inheritance and not self.field: - value_dict[k] = GenericReferenceField().to_mongo(v) + value_dict[k] = GenericReferenceField().to_mongo( + v, **kwargs) else: collection = v._get_collection_name() value_dict[k] = DBRef(collection, v.pk) elif hasattr(v, 'to_mongo'): cls = v.__class__ - val = v.to_mongo() + val = v.to_mongo(**kwargs) # If it's a document that is not inherited add _cls if isinstance(v, (Document, EmbeddedDocument)): val['_cls'] = cls.__name__ value_dict[k] = val else: - value_dict[k] = self.to_mongo(v) + value_dict[k] = self.to_mongo(v, **kwargs) if is_list: # Convert back to a list return [v for _, v in sorted(value_dict.items(), @@ -443,7 +445,7 @@ class ObjectIdField(BaseField): pass return value - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): if not isinstance(value, ObjectId): try: return ObjectId(unicode(value)) @@ -618,7 +620,7 @@ class GeoJsonBaseField(BaseField): if errors: return "Invalid MultiPolygon:\n%s" % ", ".join(errors) - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): if isinstance(value, dict): return value return SON([("type", self._type), ("coordinates", value)]) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3887fd84..e5e25457 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -325,7 +325,7 @@ class DecimalField(BaseField): return value return value.quantize(decimal.Decimal(".%s" % ("0" * self.precision)), rounding=self.rounding) - def to_mongo(self, value, use_db_field=True): + def to_mongo(self, value, **kwargs): if value is None: return value if self.force_string: @@ -388,7 +388,7 @@ class DateTimeField(BaseField): if not isinstance(new_value, (datetime.datetime, datetime.date)): self.error(u'cannot parse date "%s"' % value) - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): if value is None: return value if isinstance(value, datetime.datetime): @@ -511,7 +511,7 @@ class ComplexDateTimeField(StringField): except: return original_value - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): value = self.to_python(value) return self._convert_from_datetime(value) @@ -546,11 +546,10 @@ class EmbeddedDocumentField(BaseField): return self.document_type._from_son(value, _auto_dereference=self._auto_dereference) return value - def to_mongo(self, value, use_db_field=True, fields=[]): + def to_mongo(self, value, **kwargs): if not isinstance(value, self.document_type): return value - return self.document_type.to_mongo(value, use_db_field, - fields=fields) + return self.document_type.to_mongo(value, **kwargs) def validate(self, value, clean=True): """Make sure that the document instance is an instance of the @@ -600,11 +599,11 @@ class GenericEmbeddedDocumentField(BaseField): value.validate(clean=clean) - def to_mongo(self, document, use_db_field=True): + def to_mongo(self, document, **kwargs): if document is None: return None - data = document.to_mongo(use_db_field) + data = document.to_mongo(**kwargs) if '_cls' not in data: data['_cls'] = document._class_name return data @@ -616,7 +615,7 @@ class DynamicField(BaseField): Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): """Convert a Python type to a MongoDB compatible type. """ @@ -625,7 +624,7 @@ class DynamicField(BaseField): if hasattr(value, 'to_mongo'): cls = value.__class__ - val = value.to_mongo() + val = value.to_mongo(**kwargs) # If we its a document thats not inherited add _cls if isinstance(value, Document): val = {"_ref": value.to_dbref(), "_cls": cls.__name__} @@ -643,7 +642,7 @@ class DynamicField(BaseField): data = {} for k, v in value.iteritems(): - data[k] = self.to_mongo(v) + data[k] = self.to_mongo(v, **kwargs) value = data if is_list: # Convert back to a list @@ -755,8 +754,8 @@ class SortedListField(ListField): self._order_reverse = kwargs.pop('reverse') super(SortedListField, self).__init__(field, **kwargs) - def to_mongo(self, value): - value = super(SortedListField, self).to_mongo(value) + def to_mongo(self, value, **kwargs): + value = super(SortedListField, self).to_mongo(value, **kwargs) if self._ordering is not None: return sorted(value, key=itemgetter(self._ordering), reverse=self._order_reverse) @@ -942,7 +941,7 @@ class ReferenceField(BaseField): return super(ReferenceField, self).__get__(instance, owner) - def to_mongo(self, document): + def to_mongo(self, document, **kwargs): if isinstance(document, DBRef): if not self.dbref: return document.id @@ -965,7 +964,7 @@ class ReferenceField(BaseField): id_field_name = cls._meta['id_field'] id_field = cls._fields[id_field_name] - id_ = id_field.to_mongo(id_) + id_ = id_field.to_mongo(id_, **kwargs) if self.document_type._meta.get('abstract'): collection = cls._get_collection_name() return DBRef(collection, id_, cls=cls._class_name) @@ -1088,7 +1087,7 @@ class CachedReferenceField(BaseField): return super(CachedReferenceField, self).__get__(instance, owner) - def to_mongo(self, document): + def to_mongo(self, document, **kwargs): id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] @@ -1103,10 +1102,11 @@ class CachedReferenceField(BaseField): # TODO: should raise here or will fail next statement value = SON(( - ("_id", id_field.to_mongo(id_)), + ("_id", id_field.to_mongo(id_, **kwargs)), )) - value.update(dict(document.to_mongo(fields=self.fields))) + kwargs['fields'] = self.fields + value.update(dict(document.to_mongo(**kwargs))) return value def prepare_query_value(self, op, value): @@ -1222,7 +1222,7 @@ class GenericReferenceField(BaseField): doc = doc_cls._from_son(doc) return doc - def to_mongo(self, document, use_db_field=True): + def to_mongo(self, document, **kwargs): if document is None: return None @@ -1241,7 +1241,7 @@ class GenericReferenceField(BaseField): else: id_ = document - id_ = id_field.to_mongo(id_) + id_ = id_field.to_mongo(id_, **kwargs) collection = document._get_collection_name() ref = DBRef(collection, id_) return SON(( @@ -1270,7 +1270,7 @@ class BinaryField(BaseField): value = bin_type(value) return super(BinaryField, self).__set__(instance, value) - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): return Binary(value) def validate(self, value): @@ -1495,7 +1495,7 @@ class FileField(BaseField): db_alias=db_alias, collection_name=collection_name) - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): # Store the GridFS file id in MongoDB if isinstance(value, self.proxy_class) and value.grid_id is not None: return value.grid_id @@ -1845,7 +1845,7 @@ class UUIDField(BaseField): return original_value return value - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): if not self._binary: return unicode(value) elif isinstance(value, basestring): diff --git a/tests/document/instance.py b/tests/document/instance.py index f463ec31..61f13897 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -679,6 +679,19 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertHasInstance(doc.embedded_field[0], doc) + def test_embedded_document_complex_instance_no_use_db_field(self): + """Ensure that use_db_field is propagated to list of Emb Docs + """ + class Embedded(EmbeddedDocument): + string = StringField(db_field='s') + + class Doc(Document): + embedded_field = ListField(EmbeddedDocumentField(Embedded)) + + d = Doc(embedded_field=[Embedded(string="Hi")]).to_mongo( + use_db_field=False).to_dict() + self.assertEqual(d['embedded_field'], [{'string': 'Hi'}]) + def test_instance_is_set_on_setattr(self): class Email(EmbeddedDocument): diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 15daecb9..dbba6f9f 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -3380,7 +3380,7 @@ class FieldTest(unittest.TestCase): def __init__(self, **kwargs): super(EnumField, self).__init__(**kwargs) - def to_mongo(self, value): + def to_mongo(self, value, **kwargs): return value def to_python(self, value): From 8f8217e9284375331ed64914d4db3b5190a8fb54 Mon Sep 17 00:00:00 2001 From: Bryan Bennett Date: Wed, 20 Jan 2016 16:58:25 -0500 Subject: [PATCH 58/99] Add Bryan Bennett to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index b087f41c..ad070b30 100644 --- a/AUTHORS +++ b/AUTHORS @@ -234,3 +234,4 @@ that much better: * Paul-Armand Verhaegen (https://github.com/paularmand) * Steven Rossiter (https://github.com/BeardedSteve) * Luo Peng (https://github.com/RussellLuo) + * Bryan Bennett (https://github.com/bbenne10) From c65fd0e47752ed7b713288bba749aee6a6654695 Mon Sep 17 00:00:00 2001 From: Bryan Bennett Date: Wed, 20 Jan 2016 16:58:33 -0500 Subject: [PATCH 59/99] Note changes for #1217 in Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b92b9cec..363bdbfb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changes in 0.10.6 - Add support for mocking MongoEngine based on mongomock. #1151 - Fixed not being able to run tests on Windows. #1153 - Allow creation of sparse compound indexes. #1114 +- Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances Changes in 0.10.5 ================= From 7a568dc118fff0668807d334e9f5cffd3b8166ce Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Tue, 26 Jan 2016 15:54:57 +0100 Subject: [PATCH 60/99] Add version 0.10.7 - DEV in changelog.rst --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 363bdbfb..7682598b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,12 +2,15 @@ Changelog ========= +Changes in 0.10.7 - DEV +======================= +- Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances + Changes in 0.10.6 ================= - Add support for mocking MongoEngine based on mongomock. #1151 - Fixed not being able to run tests on Windows. #1153 - Allow creation of sparse compound indexes. #1114 -- Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances Changes in 0.10.5 ================= From 9f8327926d3fa4f7559834a899bfc998371cfeb1 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Thu, 28 Jan 2016 18:18:51 +0100 Subject: [PATCH 61/99] Improve a bit queryset's test_elem_match --- tests/queryset/queryset.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 87c14757..a9a2536a 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4114,6 +4114,10 @@ class QuerySetTest(unittest.TestCase): Foo(shape="circle", color="purple", thick=False)]) b2.save() + b3 = Bar(foo=[Foo(shape="square", thick=True), + Foo(shape="circle", color="purple", thick=False)]) + b3.save() + ak = list( Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) @@ -4133,6 +4137,13 @@ class QuerySetTest(unittest.TestCase): Bar.objects(foo__match={'shape': "square", "color__exists": True})) self.assertEqual([b1, b2], ak) + ak = list( + Bar.objects(foo__elemMatch={'shape': "square", "color__exists": False})) + self.assertEqual([b3], ak) + + ak = list( + Bar.objects(foo__match={'shape': "square", "color__exists": False})) + self.assertEqual([b3], ak) def test_upsert_includes_cls(self): """Upserts should include _cls information for inheritable classes From a643933d163b6dd3a1952bb171259210fadf4b31 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Sat, 30 Jan 2016 11:58:37 +0100 Subject: [PATCH 62/99] Fix cascade delete mixing among collections --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 6 ++-- tests/document/instance.py | 56 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7682598b..6fb7e6aa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.7 - DEV ======================= - Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances +- Fixed cascade delete mixing among collections #1224 Changes in 0.10.6 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 454d781a..de796de5 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -403,8 +403,10 @@ class BaseQuerySet(object): rule = doc._meta['delete_rules'][rule_entry] if rule == CASCADE: cascade_refs = set() if cascade_refs is None else cascade_refs - for ref in queryset: - cascade_refs.add(ref.id) + # Handle recursive reference + if doc._collection == document_cls._collection: + for ref in queryset: + cascade_refs.add(ref.id) ref_q = document_cls.objects(**{field_name + '__in': self, 'id__nin': cascade_refs}) ref_q_count = ref_q.count() if ref_q_count > 0: diff --git a/tests/document/instance.py b/tests/document/instance.py index 61f13897..4cfe8b8e 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1906,6 +1906,62 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(BlogPost.objects.count(), 0) + def test_reverse_delete_rule_with_custom_id_field(self): + """Ensure that a referenced document with custom primary key + is also deleted upon deletion. + """ + class User(Document): + name = StringField(primary_key=True) + + class Book(Document): + author = ReferenceField(User, reverse_delete_rule=CASCADE) + reviewer = ReferenceField(User, reverse_delete_rule=NULLIFY) + + User.drop_collection() + Book.drop_collection() + + user = User(name='Mike').save() + reviewer = User(name='John').save() + book = Book(author=user, reviewer=reviewer).save() + + reviewer.delete() + self.assertEqual(Book.objects.count(), 1) + self.assertEqual(Book.objects.get().reviewer, None) + + user.delete() + self.assertEqual(Book.objects.count(), 0) + + def test_reverse_delete_rule_with_shared_id_among_collections(self): + """Ensure that cascade delete rule doesn't mix id among collections. + """ + class User(Document): + id = IntField(primary_key=True) + + class Book(Document): + id = IntField(primary_key=True) + author = ReferenceField(User, reverse_delete_rule=CASCADE) + + User.drop_collection() + Book.drop_collection() + + user_1 = User(id=1).save() + user_2 = User(id=2).save() + book_1 = Book(id=1, author=user_2).save() + book_2 = Book(id=2, author=user_1).save() + + user_2.delete() + # Deleting user_2 should also delete book_1 but not book_2 + self.assertEqual(Book.objects.count(), 1) + self.assertEqual(Book.objects.get(), book_2) + + user_3 = User(id=3).save() + book_3 = Book(id=3, author=user_3).save() + + user_3.delete() + # Deleting user_3 should also delete book_3 + self.assertEqual(Book.objects.count(), 1) + self.assertEqual(Book.objects.get(), book_2) + def test_reverse_delete_rule_with_document_inheritance(self): """Ensure that a referenced document is also deleted upon deletion of a child document. From b32006441847298405f5be7711477a88163ae064 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Tue, 9 Feb 2016 14:28:55 +0100 Subject: [PATCH 63/99] Add signal_kwargs arg for save/delete/bulk insert --- docs/changelog.rst | 1 + mongoengine/document.py | 26 +++++-- mongoengine/queryset/base.py | 15 +++- tests/test_signals.py | 145 +++++++++++++++++++++++++++-------- 4 files changed, 142 insertions(+), 45 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6fb7e6aa..3794e9e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changes in 0.10.7 - DEV ======================= - Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances - Fixed cascade delete mixing among collections #1224 +- Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 Changes in 0.10.6 ================= diff --git a/mongoengine/document.py b/mongoengine/document.py index d76c393b..8211d95e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -250,7 +250,7 @@ class Document(BaseDocument): def save(self, force_insert=False, validate=True, clean=True, write_concern=None, cascade=None, cascade_kwargs=None, - _refs=None, save_condition=None, **kwargs): + _refs=None, save_condition=None, signal_kwargs=None, **kwargs): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -276,6 +276,8 @@ class Document(BaseDocument): :param save_condition: only perform save if matching record in db satisfies condition(s) (e.g. version number). Raises :class:`OperationError` if the conditions are not satisfied + :parm signal_kwargs: (optional) kwargs dictionary to be passed to + the signal calls. .. versionchanged:: 0.5 In existing documents it only saves changed fields using @@ -297,8 +299,11 @@ class Document(BaseDocument): :class:`OperationError` exception raised if save_condition fails. .. versionchanged:: 0.10.1 :class: save_condition failure now raises a `SaveConditionError` + .. versionchanged:: 0.10.7 + Add signal_kwargs argument """ - signals.pre_save.send(self.__class__, document=self) + signal_kwargs = signal_kwargs or {} + signals.pre_save.send(self.__class__, document=self, **signal_kwargs) if validate: self.validate(clean=clean) @@ -311,7 +316,7 @@ class Document(BaseDocument): created = ('_id' not in doc or self._created or force_insert) signals.pre_save_post_validation.send(self.__class__, document=self, - created=created) + created=created, **signal_kwargs) try: collection = self._get_collection() @@ -400,7 +405,8 @@ class Document(BaseDocument): if created or id_field not in self._meta.get('shard_key', []): self[id_field] = self._fields[id_field].to_python(object_id) - signals.post_save.send(self.__class__, document=self, created=created) + signals.post_save.send(self.__class__, document=self, + created=created, **signal_kwargs) self._clear_changed_fields() self._created = False return self @@ -476,18 +482,24 @@ class Document(BaseDocument): # Need to add shard key to query, or you get an error return self._qs.filter(**self._object_key).update_one(**kwargs) - def delete(self, **write_concern): + def delete(self, signal_kwargs=None, **write_concern): """Delete the :class:`~mongoengine.Document` from the database. This will only take effect if the document has been previously saved. + :parm signal_kwargs: (optional) kwargs dictionary to be passed to + the signal calls. :param write_concern: Extra keyword arguments are passed down which will be used as options for the resultant ``getLastError`` command. For example, ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. + + .. versionchanged:: 0.10.7 + Add signal_kwargs argument """ - signals.pre_delete.send(self.__class__, document=self) + signal_kwargs = signal_kwargs or {} + signals.pre_delete.send(self.__class__, document=self, **signal_kwargs) # Delete FileFields separately FileField = _import_class('FileField') @@ -501,7 +513,7 @@ class Document(BaseDocument): except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) - signals.post_delete.send(self.__class__, document=self) + signals.post_delete.send(self.__class__, document=self, **signal_kwargs) def switch_db(self, db_alias, keep_created=True): """ diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index de796de5..f301e160 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -266,7 +266,8 @@ class BaseQuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True, write_concern=None): + def insert(self, doc_or_docs, load_bulk=True, + write_concern=None, signal_kwargs=None): """bulk insert documents :param doc_or_docs: a document or list of documents to be inserted @@ -279,11 +280,15 @@ class BaseQuerySet(object): ``insert(..., {w: 2, fsync: True})`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. + :parm signal_kwargs: (optional) kwargs dictionary to be passed to + the signal calls. By default returns document instances, set ``load_bulk`` to False to return just ``ObjectIds`` .. versionadded:: 0.5 + .. versionchanged:: 0.10.7 + Add signal_kwargs argument """ Document = _import_class('Document') @@ -305,7 +310,9 @@ class BaseQuerySet(object): msg = "Some documents have ObjectIds use doc.update() instead" raise OperationError(msg) - signals.pre_bulk_insert.send(self._document, documents=docs) + signal_kwargs = signal_kwargs or {} + signals.pre_bulk_insert.send(self._document, + documents=docs, **signal_kwargs) raw = [doc.to_mongo() for doc in docs] try: @@ -324,7 +331,7 @@ class BaseQuerySet(object): if not load_bulk: signals.post_bulk_insert.send( - self._document, documents=docs, loaded=False) + self._document, documents=docs, loaded=False, **signal_kwargs) return return_one and ids[0] or ids documents = self.in_bulk(ids) @@ -332,7 +339,7 @@ class BaseQuerySet(object): for obj_id in ids: results.append(documents.get(obj_id)) signals.post_bulk_insert.send( - self._document, documents=results, loaded=True) + self._document, documents=results, loaded=True, **signal_kwargs) return return_one and results[0] or results def count(self, with_limit_and_skip=False): diff --git a/tests/test_signals.py b/tests/test_signals.py index 78305b7d..23da7cd4 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -25,6 +25,8 @@ class SignalTests(unittest.TestCase): connect(db='mongoenginetest') class Author(Document): + # Make the id deterministic for easier testing + id = SequenceField(primary_key=True) name = StringField() def __unicode__(self): @@ -33,7 +35,7 @@ class SignalTests(unittest.TestCase): @classmethod def pre_init(cls, sender, document, *args, **kwargs): signal_output.append('pre_init signal, %s' % cls.__name__) - signal_output.append(str(kwargs['values'])) + signal_output.append(kwargs['values']) @classmethod def post_init(cls, sender, document, **kwargs): @@ -43,48 +45,55 @@ class SignalTests(unittest.TestCase): @classmethod def pre_save(cls, sender, document, **kwargs): signal_output.append('pre_save signal, %s' % document) + signal_output.append(kwargs) @classmethod def pre_save_post_validation(cls, sender, document, **kwargs): signal_output.append('pre_save_post_validation signal, %s' % document) - if 'created' in kwargs: - if kwargs['created']: - signal_output.append('Is created') - else: - signal_output.append('Is updated') + if kwargs.pop('created', False): + signal_output.append('Is created') + else: + signal_output.append('Is updated') + signal_output.append(kwargs) @classmethod def post_save(cls, sender, document, **kwargs): dirty_keys = document._delta()[0].keys() + document._delta()[1].keys() signal_output.append('post_save signal, %s' % document) signal_output.append('post_save dirty keys, %s' % dirty_keys) - if 'created' in kwargs: - if kwargs['created']: - signal_output.append('Is created') - else: - signal_output.append('Is updated') + if kwargs.pop('created', False): + signal_output.append('Is created') + else: + signal_output.append('Is updated') + signal_output.append(kwargs) @classmethod def pre_delete(cls, sender, document, **kwargs): signal_output.append('pre_delete signal, %s' % document) + signal_output.append(kwargs) @classmethod def post_delete(cls, sender, document, **kwargs): signal_output.append('post_delete signal, %s' % document) + signal_output.append(kwargs) @classmethod def pre_bulk_insert(cls, sender, documents, **kwargs): signal_output.append('pre_bulk_insert signal, %s' % documents) + signal_output.append(kwargs) @classmethod def post_bulk_insert(cls, sender, documents, **kwargs): signal_output.append('post_bulk_insert signal, %s' % documents) - if kwargs.get('loaded', False): + if kwargs.pop('loaded', False): signal_output.append('Is loaded') else: signal_output.append('Not loaded') + signal_output.append(kwargs) + self.Author = Author Author.drop_collection() + Author.id.set_next_value(0) class Another(Document): @@ -96,10 +105,12 @@ class SignalTests(unittest.TestCase): @classmethod def pre_delete(cls, sender, document, **kwargs): signal_output.append('pre_delete signal, %s' % document) + signal_output.append(kwargs) @classmethod def post_delete(cls, sender, document, **kwargs): signal_output.append('post_delete signal, %s' % document) + signal_output.append(kwargs) self.Another = Another Another.drop_collection() @@ -137,12 +148,18 @@ class SignalTests(unittest.TestCase): for document in documents: if not document.active: document.active = True + signal_output.append(kwargs) @classmethod def post_bulk_insert(cls, sender, documents, **kwargs): signal_output.append('post_bulk_insert signal, %s' % [(doc, {'active': documents[n].active}) for n, doc in enumerate(documents)]) + if kwargs.pop('loaded', False): + signal_output.append('Is loaded') + else: + signal_output.append('Not loaded') + signal_output.append(kwargs) self.Post = Post Post.drop_collection() @@ -237,63 +254,118 @@ class SignalTests(unittest.TestCase): self.assertEqual(self.get_signal_output(create_author), [ "pre_init signal, Author", - "{'name': 'Bill Shakespeare'}", + {'name': 'Bill Shakespeare'}, "post_init signal, Bill Shakespeare, document._created = True", ]) a1 = self.Author(name='Bill Shakespeare') self.assertEqual(self.get_signal_output(a1.save), [ "pre_save signal, Bill Shakespeare", + {}, "pre_save_post_validation signal, Bill Shakespeare", "Is created", + {}, "post_save signal, Bill Shakespeare", "post_save dirty keys, ['name']", - "Is created" + "Is created", + {} ]) a1.reload() a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ "pre_save signal, William Shakespeare", + {}, "pre_save_post_validation signal, William Shakespeare", "Is updated", + {}, "post_save signal, William Shakespeare", "post_save dirty keys, ['name']", - "Is updated" + "Is updated", + {} ]) self.assertEqual(self.get_signal_output(a1.delete), [ 'pre_delete signal, William Shakespeare', + {}, 'post_delete signal, William Shakespeare', + {} ]) - signal_output = self.get_signal_output(load_existing_author) - # test signal_output lines separately, because of random ObjectID after object load - self.assertEqual(signal_output[0], + self.assertEqual(self.get_signal_output(load_existing_author), [ "pre_init signal, Author", - ) - self.assertEqual(signal_output[2], - "post_init signal, Bill Shakespeare, document._created = False", - ) + {'id': 2, 'name': 'Bill Shakespeare'}, + "post_init signal, Bill Shakespeare, document._created = False" + ]) - - signal_output = self.get_signal_output(bulk_create_author_with_load) - - # The output of this signal is not entirely deterministic. The reloaded - # object will have an object ID. Hence, we only check part of the output - self.assertEqual(signal_output[3], "pre_bulk_insert signal, []" - ) - self.assertEqual(signal_output[-2:], - ["post_bulk_insert signal, []", - "Is loaded",]) + self.assertEqual(self.get_signal_output(bulk_create_author_with_load), [ + 'pre_init signal, Author', + {'name': 'Bill Shakespeare'}, + 'post_init signal, Bill Shakespeare, document._created = True', + 'pre_bulk_insert signal, []', + {}, + 'pre_init signal, Author', + {'id': 3, 'name': 'Bill Shakespeare'}, + 'post_init signal, Bill Shakespeare, document._created = False', + 'post_bulk_insert signal, []', + 'Is loaded', + {} + ]) self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [ "pre_init signal, Author", - "{'name': 'Bill Shakespeare'}", + {'name': 'Bill Shakespeare'}, "post_init signal, Bill Shakespeare, document._created = True", "pre_bulk_insert signal, []", + {}, "post_bulk_insert signal, []", "Not loaded", + {} + ]) + + def test_signal_kwargs(self): + """ Make sure signal_kwargs is passed to signals calls. """ + + def live_and_let_die(): + a = self.Author(name='Bill Shakespeare') + a.save(signal_kwargs={'live': True, 'die': False}) + a.delete(signal_kwargs={'live': False, 'die': True}) + + self.assertEqual(self.get_signal_output(live_and_let_die), [ + "pre_init signal, Author", + {'name': 'Bill Shakespeare'}, + "post_init signal, Bill Shakespeare, document._created = True", + "pre_save signal, Bill Shakespeare", + {'die': False, 'live': True}, + "pre_save_post_validation signal, Bill Shakespeare", + "Is created", + {'die': False, 'live': True}, + "post_save signal, Bill Shakespeare", + "post_save dirty keys, ['name']", + "Is created", + {'die': False, 'live': True}, + 'pre_delete signal, Bill Shakespeare', + {'die': True, 'live': False}, + 'post_delete signal, Bill Shakespeare', + {'die': True, 'live': False} + ]) + + def bulk_create_author(): + a1 = self.Author(name='Bill Shakespeare') + self.Author.objects.insert([a1], signal_kwargs={'key': True}) + + self.assertEqual(self.get_signal_output(bulk_create_author), [ + 'pre_init signal, Author', + {'name': 'Bill Shakespeare'}, + 'post_init signal, Bill Shakespeare, document._created = True', + 'pre_bulk_insert signal, []', + {'key': True}, + 'pre_init signal, Author', + {'id': 2, 'name': 'Bill Shakespeare'}, + 'post_init signal, Bill Shakespeare, document._created = False', + 'post_bulk_insert signal, []', + 'Is loaded', + {'key': True} ]) def test_queryset_delete_signals(self): @@ -302,7 +374,9 @@ class SignalTests(unittest.TestCase): self.Another(name='Bill Shakespeare').save() self.assertEqual(self.get_signal_output(self.Another.objects.delete), [ 'pre_delete signal, Bill Shakespeare', + {}, 'post_delete signal, Bill Shakespeare', + {} ]) def test_signals_with_explicit_doc_ids(self): @@ -353,7 +427,10 @@ class SignalTests(unittest.TestCase): results = self.get_signal_output(bulk_set_active_post) self.assertEqual(results, [ "pre_bulk_insert signal, [(, {'active': False}), (, {'active': False}), (, {'active': False})]", - "post_bulk_insert signal, [(, {'active': True}), (, {'active': True}), (, {'active': True})]" + {}, + "post_bulk_insert signal, [(, {'active': True}), (, {'active': True}), (, {'active': True})]", + 'Is loaded', + {} ]) if __name__ == '__main__': From 8f505c2dcc36c57d57fcba8e8bb4f52da4424344 Mon Sep 17 00:00:00 2001 From: hhstore Date: Wed, 17 Feb 2016 10:55:31 +0800 Subject: [PATCH 64/99] fix a small bug - ReferenceField() comment give a wrong demo . --- mongoengine/fields.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e5e25457..1300b0e4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -877,7 +877,7 @@ class ReferenceField(BaseField): content = StringField() foo = ReferenceField('Foo') - Bar.register_delete_rule(Foo, 'bar', NULLIFY) + Bar.register_delete_rule(Foo, 'foo', NULLIFY) .. note :: `reverse_delete_rule` does not trigger pre / post delete signals to be @@ -1013,7 +1013,7 @@ class ReferenceField(BaseField): class CachedReferenceField(BaseField): """ A referencefield with cache fields to purpose pseudo-joins - + .. versionadded:: 0.9 """ @@ -1707,17 +1707,17 @@ class SequenceField(BaseField): :param collection_name: Name of the counter collection (default 'mongoengine.counters') :param sequence_name: Name of the sequence in the collection (default 'ClassName.counter') :param value_decorator: Any callable to use as a counter (default int) - + Use any callable as `value_decorator` to transform calculated counter into any value suitable for your needs, e.g. string or hexadecimal representation of the default integer counter value. - + .. note:: - - In case the counter is defined in the abstract document, it will be - common to all inherited documents and the default sequence name will + + In case the counter is defined in the abstract document, it will be + common to all inherited documents and the default sequence name will be the class name of the abstract document. - + .. versionadded:: 0.5 .. versionchanged:: 0.8 added `value_decorator` """ From c499133bbe1aee8ed3d6adb024eb762ca47a2e5f Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Fri, 19 Feb 2016 00:11:30 +0100 Subject: [PATCH 65/99] Add missing drop_collection in tests fields --- tests/fields/fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index dbba6f9f..bf2b5dd5 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1551,6 +1551,8 @@ class FieldTest(unittest.TestCase): name = StringField() preferences = EmbeddedDocumentField(PersonPreferences) + Person.drop_collection() + person = Person(name='Test User') person.preferences = 'My Preferences' self.assertRaises(ValidationError, person.validate) @@ -1583,6 +1585,8 @@ class FieldTest(unittest.TestCase): content = StringField() author = EmbeddedDocumentField(User) + BlogPost.drop_collection() + post = BlogPost(content='What I did today...') post.author = PowerUser(name='Test User', power=47) post.save() From adce9e6220616ee030041da7ee6c12f78a133132 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Fri, 19 Feb 2016 01:58:15 +0100 Subject: [PATCH 66/99] Raise OperationError in drop_collection if no collection is set --- docs/changelog.rst | 1 + mongoengine/document.py | 12 +++++++++++- tests/fields/fields.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3794e9e3..069872ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changes in 0.10.7 - DEV - Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances - Fixed cascade delete mixing among collections #1224 - Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 +- Raise `OperationError` when trying to do a `drop_collection` on document with no collection set. Changes in 0.10.6 ================= diff --git a/mongoengine/document.py b/mongoengine/document.py index 8211d95e..52353523 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -679,10 +679,20 @@ class Document(BaseDocument): def drop_collection(cls): """Drops the entire collection associated with this :class:`~mongoengine.Document` type from the database. + + Raises :class:`OperationError` if the document has no collection set + (i.g. if it is `abstract`) + + .. versionchanged:: 0.10.7 + :class:`OperationError` exception raised if no collection available """ + col_name = cls._get_collection_name() + if not col_name: + raise OperationError('Document %s has no collection defined ' + '(is it abstract ?)' % cls) cls._collection = None db = cls._get_db() - db.drop_collection(cls._get_collection_name()) + db.drop_collection(col_name) @classmethod def create_index(cls, keys, background=False, **kwargs): diff --git a/tests/fields/fields.py b/tests/fields/fields.py index bf2b5dd5..eb2f46c7 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2285,6 +2285,16 @@ class FieldTest(unittest.TestCase): Member.drop_collection() BlogPost.drop_collection() + def test_drop_abstract_document(self): + """Ensure that an abstract document cannot be dropped given it + has no underlying collection. + """ + class AbstractDoc(Document): + name = StringField() + meta = {"abstract": True} + + self.assertRaises(OperationError, AbstractDoc.drop_collection) + def test_reference_class_with_abstract_parent(self): """Ensure that a class with an abstract parent can be referenced. """ From 9e9703183fd54997eea3c5a42d9e999c38975b23 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Fri, 19 Feb 2016 02:16:37 +0100 Subject: [PATCH 67/99] Add test for nested list in EmbeddedDocument --- mongoengine/base/fields.py | 3 +-- tests/fields/fields.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 4b2d0e8f..be50ac40 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -285,8 +285,6 @@ class ComplexBaseField(BaseField): def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ - Document = _import_class('Document') - if isinstance(value, basestring): return value @@ -306,6 +304,7 @@ class ComplexBaseField(BaseField): value_dict = dict([(key, self.field.to_python(item)) for key, item in value.items()]) else: + Document = _import_class('Document') value_dict = {} for k, v in value.items(): if isinstance(v, Document): diff --git a/tests/fields/fields.py b/tests/fields/fields.py index eb2f46c7..757f66a8 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1593,6 +1593,31 @@ class FieldTest(unittest.TestCase): self.assertEqual(47, BlogPost.objects.first().author.power) + def test_embedded_document_inheritance_with_list(self): + """Ensure that nested list of subclassed embedded documents is + handled correctly. + """ + + class Group(EmbeddedDocument): + name = StringField() + content = ListField(StringField()) + + class Basedoc(Document): + groups = ListField(EmbeddedDocumentField(Group)) + meta = {'abstract': True} + + class User(Basedoc): + doctype = StringField(require=True, default='userdata') + + User.drop_collection() + + content = ['la', 'le', 'lu'] + group = Group(name='foo', content=content) + foobar = User(groups=[group]) + foobar.save() + + self.assertEqual(content, User.objects.first().groups[0].content) + def test_reference_validation(self): """Ensure that invalid docment objects cannot be assigned to reference fields. From ddedc1ee92efe73e7d11f455bd8ee9cec122ca4a Mon Sep 17 00:00:00 2001 From: Konstantin Gukov Date: Tue, 23 Feb 2016 23:40:23 +0500 Subject: [PATCH 68/99] Fixed too broad exception clauses in the project --- mongoengine/base/fields.py | 16 ++++++++-------- mongoengine/document.py | 2 +- mongoengine/fields.py | 16 ++++++++-------- mongoengine/queryset/base.py | 2 +- mongoengine/queryset/transform.py | 10 +++++++--- setup.py | 9 +++++---- tests/document/inheritance.py | 2 +- 7 files changed, 31 insertions(+), 26 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index be50ac40..a803657d 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -133,7 +133,7 @@ class BaseField(object): if (self.name not in instance._data or instance._data[self.name] != value): instance._mark_as_changed(self.name) - except: + except Exception: # Values cant be compared eg: naive and tz datetimes # So mark it as changed instance._mark_as_changed(self.name) @@ -440,7 +440,7 @@ class ObjectIdField(BaseField): try: if not isinstance(value, ObjectId): value = ObjectId(value) - except: + except Exception: pass return value @@ -459,7 +459,7 @@ class ObjectIdField(BaseField): def validate(self, value): try: ObjectId(unicode(value)) - except: + except Exception: self.error('Invalid Object ID') @@ -511,7 +511,7 @@ class GeoJsonBaseField(BaseField): # Quick and dirty validator try: value[0][0][0] - except: + except (TypeError, IndexError): return "Invalid Polygon must contain at least one valid linestring" errors = [] @@ -535,7 +535,7 @@ class GeoJsonBaseField(BaseField): # Quick and dirty validator try: value[0][0] - except: + except (TypeError, IndexError): return "Invalid LineString must contain at least one valid point" errors = [] @@ -566,7 +566,7 @@ class GeoJsonBaseField(BaseField): # Quick and dirty validator try: value[0][0] - except: + except (TypeError, IndexError): return "Invalid MultiPoint must contain at least one valid point" errors = [] @@ -585,7 +585,7 @@ class GeoJsonBaseField(BaseField): # Quick and dirty validator try: value[0][0][0] - except: + except (TypeError, IndexError): return "Invalid MultiLineString must contain at least one valid linestring" errors = [] @@ -607,7 +607,7 @@ class GeoJsonBaseField(BaseField): # Quick and dirty validator try: value[0][0][0][0] - except: + except (TypeError, IndexError): return "Invalid MultiPolygon must contain at least one valid Polygon" errors = [] diff --git a/mongoengine/document.py b/mongoengine/document.py index 52353523..2fac15b0 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -981,7 +981,7 @@ class MapReduceDocument(object): if not isinstance(self.key, id_field_type): try: self.key = id_field_type(self.key) - except: + except Exception: raise Exception("Could not cast key as %s" % id_field_type.__name__) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e5e25457..b57f93b6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -65,7 +65,7 @@ class StringField(BaseField): return value try: value = value.decode('utf-8') - except: + except Exception: pass return value @@ -194,7 +194,7 @@ class IntField(BaseField): def validate(self, value): try: value = int(value) - except: + except Exception: self.error('%s could not be converted to int' % value) if self.min_value is not None and value < self.min_value: @@ -228,7 +228,7 @@ class LongField(BaseField): def validate(self, value): try: value = long(value) - except: + except Exception: self.error('%s could not be converted to long' % value) if self.min_value is not None and value < self.min_value: @@ -508,7 +508,7 @@ class ComplexDateTimeField(StringField): original_value = value try: return self._convert_from_string(value) - except: + except Exception: return original_value def to_mongo(self, value, **kwargs): @@ -1370,7 +1370,7 @@ class GridFSProxy(object): if self.gridout is None: self.gridout = self.fs.get(self.grid_id) return self.gridout - except: + except Exception: # File has been deleted return None @@ -1408,7 +1408,7 @@ class GridFSProxy(object): else: try: return gridout.read(size) - except: + except Exception: return "" def delete(self): @@ -1473,7 +1473,7 @@ class FileField(BaseField): if grid_file: try: grid_file.delete() - except: + except Exception: pass # Create a new proxy object as we don't already have one @@ -1841,7 +1841,7 @@ class UUIDField(BaseField): if not isinstance(value, basestring): value = unicode(value) return uuid.UUID(value) - except: + except Exception: return original_value return value diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index f301e160..4bc63597 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1705,7 +1705,7 @@ class BaseQuerySet(object): key = key.replace('__', '.') try: key = self._document._translate_field_name(key) - except: + except Exception: pass key_list.append((key, direction)) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 1f18c429..13302afa 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -364,20 +364,24 @@ def _infer_geometry(value): "type and coordinates keys") elif isinstance(value, (list, set)): # TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon? + # TODO: should both TypeError and IndexError be alike interpreted? + try: value[0][0][0] return {"$geometry": {"type": "Polygon", "coordinates": value}} - except: + except (TypeError, IndexError): pass + try: value[0][0] return {"$geometry": {"type": "LineString", "coordinates": value}} - except: + except (TypeError, IndexError): pass + try: value[0] return {"$geometry": {"type": "Point", "coordinates": value}} - except: + except (TypeError, IndexError): pass raise InvalidQueryError("Invalid $geometry data. Can be either a dictionary " diff --git a/setup.py b/setup.py index 7384e04d..6aa924c8 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,12 @@ except ImportError: DESCRIPTION = 'MongoEngine is a Python Object-Document ' + \ 'Mapper for working with MongoDB.' -LONG_DESCRIPTION = None + try: - LONG_DESCRIPTION = open('README.rst').read() -except: - pass + with open('README.rst') as fin: + LONG_DESCRIPTION = fin.read() +except Exception: + LONG_DESCRIPTION = None def get_version(version_tuple): diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index 7673a103..957938be 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -411,7 +411,7 @@ class InheritanceTest(unittest.TestCase): try: class MyDocument(DateCreatedDocument, DateUpdatedDocument): pass - except: + except Exception: self.assertTrue(False, "Couldn't create MyDocument class") def test_abstract_documents(self): From fed58f3920d8d2cf9b3f4e337231054f8583253f Mon Sep 17 00:00:00 2001 From: Konstantin Gukov Date: Tue, 23 Feb 2016 22:59:20 +0500 Subject: [PATCH 69/99] Added support for long values in FloatFields --- mongoengine/fields.py | 10 +++++++--- tests/fields/fields.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ba505fb5..edf6515b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -260,10 +260,14 @@ class FloatField(BaseField): return value def validate(self, value): - if isinstance(value, int): - value = float(value) + if isinstance(value, (int, long)): + try: + value = float(value) + except OverflowError: + self.error('The value is too large to be converted to float') + if not isinstance(value, float): - self.error('FloatField only accepts float values') + self.error('FloatField only accepts float, int and long values') if self.min_value is not None and value < self.min_value: self.error('Float value is too small') diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 757f66a8..3b048094 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -399,20 +399,36 @@ class FieldTest(unittest.TestCase): class Person(Document): height = FloatField(min_value=0.1, max_value=3.5) + class BigPerson(Document): + height = FloatField() + person = Person() person.height = 1.89 person.validate() person.height = '2.0' self.assertRaises(ValidationError, person.validate) + person.height = 0.01 self.assertRaises(ValidationError, person.validate) + person.height = 4.0 self.assertRaises(ValidationError, person.validate) person_2 = Person(height='something invalid') self.assertRaises(ValidationError, person_2.validate) + big_person = BigPerson() + + big_person.height = 1L + big_person.validate() + + big_person.height = 2 ** 500 + big_person.validate() + + big_person.height = 2 ** 100000 # Too big for a float value + self.assertRaises(ValidationError, big_person.validate) + def test_decimal_validation(self): """Ensure that invalid values cannot be assigned to decimal fields. """ From 66b233eaea1f3c9ff0544413489000f562769204 Mon Sep 17 00:00:00 2001 From: Konstantin Gukov Date: Sun, 6 Mar 2016 16:10:02 +0500 Subject: [PATCH 70/99] Added the six module to test int/long support --- mongoengine/fields.py | 6 ++++-- requirements.txt | 3 ++- setup.py | 2 +- tests/fields/fields.py | 7 +++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index edf6515b..9d01d81c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,6 +8,8 @@ import uuid import warnings from operator import itemgetter +import six + try: import dateutil except ImportError: @@ -260,14 +262,14 @@ class FloatField(BaseField): return value def validate(self, value): - if isinstance(value, (int, long)): + if isinstance(value, six.integer_types): try: value = float(value) except OverflowError: self.error('The value is too large to be converted to float') if not isinstance(value, float): - self.error('FloatField only accepts float, int and long values') + self.error('FloatField only accepts float and integer values') if self.min_value is not None and value < self.min_value: self.error('Float value is too small') diff --git a/requirements.txt b/requirements.txt index 03935868..b6a5b06c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -pymongo>=2.7.1 nose +pymongo>=2.7.1 +six==1.10.0 diff --git a/setup.py b/setup.py index 6aa924c8..e34d834a 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo>=2.7.1'], + install_requires=['pymongo>=2.7.1', 'six'], test_suite='nose.collector', **extra_opts ) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 3b048094..1c1f49ad 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- import sys + +import six from nose.plugins.skip import SkipTest sys.path[0:0] = [""] @@ -420,8 +422,9 @@ class FieldTest(unittest.TestCase): big_person = BigPerson() - big_person.height = 1L - big_person.validate() + for value, value_type in enumerate(six.integer_types): + big_person.height = value_type(value) + big_person.validate() big_person.height = 2 ** 500 big_person.validate() From 7748e68440df2e3ad2d48ec0b55b9647c4de014d Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 10 Mar 2016 12:19:11 +0200 Subject: [PATCH 71/99] Adjust changelog for #1188. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3c0b6422..69555b95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,7 @@ Changes in 0.10.7 - DEV - Fixed cascade delete mixing among collections #1224 - Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 - Raise `OperationError` when trying to do a `drop_collection` on document with no collection set. +- count on ListField of EmbeddedDocumentField fails. #1187 Changes in 0.10.6 ================= From 7cc1a4eba04d33039e2422b39865ea72c30dee54 Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:34:22 +0100 Subject: [PATCH 72/99] Fix long fields stored as int32 in Python 3. issue #1253 --- AUTHORS | 1 + mongoengine/fields.py | 5 ++++- tests/fields/fields.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index ad070b30..758c52ea 100644 --- a/AUTHORS +++ b/AUTHORS @@ -235,3 +235,4 @@ that much better: * Steven Rossiter (https://github.com/BeardedSteve) * Luo Peng (https://github.com/RussellLuo) * Bryan Bennett (https://github.com/bbenne10) + * Gilb's Gilb's (https://github.com/gilbsgilbs) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 6f69465c..35d5f35a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -19,7 +19,7 @@ else: import pymongo import gridfs -from bson import Binary, DBRef, SON, ObjectId +from bson import Binary, DBRef, SON, ObjectId, Int64 from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, @@ -227,6 +227,9 @@ class LongField(BaseField): pass return value + def to_mongo(self, value, **kwargs): + return Int64(value) + def validate(self, value): try: value = long(value) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 1c1f49ad..2695e849 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -21,6 +21,7 @@ except ImportError: from decimal import Decimal from bson import Binary, DBRef, ObjectId +from bson.int64 import Int64 from mongoengine import * from mongoengine.connection import get_db @@ -4114,6 +4115,19 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): self.assertTrue(hasattr(CustomData.c_field, 'custom_data')) self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a']) + def test_long_field_is_stored_as_long(self): + """ + Tests that long fields are stored as long in mongo, even if long value + is small enough to be an int. + """ + class TestDocument(Document): + some_long = LongField() + + doc = TestDocument(some_long=42).save() + db = get_db() + self.assertTrue(isinstance(get_db().test_document.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(doc.some_long, (int, long,))) + if __name__ == '__main__': unittest.main() From cef4e313e11d594735520799a6e10bffbcafe4fe Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:36:31 +0100 Subject: [PATCH 73/99] Update changelog for #1253 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 69555b95..c34094a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Changes in 0.10.7 - DEV - Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 - Raise `OperationError` when trying to do a `drop_collection` on document with no collection set. - count on ListField of EmbeddedDocumentField fails. #1187 +- Fixed long fields stored as int32 in Python 3. #1253 Changes in 0.10.6 ================= From 87a2358a65fbf701506fb661005fa55d492aa5a2 Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:42:23 +0100 Subject: [PATCH 74/99] Fix unused variable. issue #1253 --- tests/fields/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 2695e849..bd0af17a 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -4125,7 +4125,7 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): doc = TestDocument(some_long=42).save() db = get_db() - self.assertTrue(isinstance(get_db().test_document.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(db.test_document.find()[0]['some_long'], Int64)) self.assertTrue(isinstance(doc.some_long, (int, long,))) From d651d0d472205c629f0b3d3de792b8609c2a6917 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Tue, 15 Mar 2016 20:50:34 +0100 Subject: [PATCH 75/99] Fix tests and imports. issue #1253 --- mongoengine/fields.py | 7 ++++++- tests/fields/fields.py | 31 +++++++++++++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 35d5f35a..206edcff 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -19,7 +19,12 @@ else: import pymongo import gridfs -from bson import Binary, DBRef, SON, ObjectId, Int64 +from bson import Binary, DBRef, SON, ObjectId + +try: + from bson.int64 import Int64 +except ImportError: + Int64 = long from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, diff --git a/tests/fields/fields.py b/tests/fields/fields.py index bd0af17a..c09cbf29 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -21,7 +21,10 @@ except ImportError: from decimal import Decimal from bson import Binary, DBRef, ObjectId -from bson.int64 import Int64 +try: + from bson.int64 import Int64 +except ImportError: + Int64 = long from mongoengine import * from mongoengine.connection import get_db @@ -3606,6 +3609,19 @@ class FieldTest(unittest.TestCase): self.assertRaises(FieldDoesNotExist, test) + def test_long_field_is_considered_as_int64(self): + """ + Tests that long fields are stored as long in mongo, even if long value + is small enough to be an int. + """ + class TestLongFieldConsideredAsInt64(Document): + some_long = LongField() + + doc = TestLongFieldConsideredAsInt64(some_long=42).save() + db = get_db() + self.assertTrue(isinstance(db.test_long_field_considered_as_int64.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(doc.some_long, (int, long,))) + class EmbeddedDocumentListFieldTestCase(unittest.TestCase): @@ -4115,19 +4131,6 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): self.assertTrue(hasattr(CustomData.c_field, 'custom_data')) self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a']) - def test_long_field_is_stored_as_long(self): - """ - Tests that long fields are stored as long in mongo, even if long value - is small enough to be an int. - """ - class TestDocument(Document): - some_long = LongField() - - doc = TestDocument(some_long=42).save() - db = get_db() - self.assertTrue(isinstance(db.test_document.find()[0]['some_long'], Int64)) - self.assertTrue(isinstance(doc.some_long, (int, long,))) - if __name__ == '__main__': unittest.main() From 39eec59c9025887d118a0027e7f338a872f94e64 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Tue, 15 Mar 2016 22:28:31 +0100 Subject: [PATCH 76/99] Fix test failing randomly because of concurrency. --- tests/document/indexes.py | 53 +++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 7ee017a6..1b4c4d96 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -11,6 +11,7 @@ from datetime import datetime from mongoengine import * from mongoengine.connection import get_db, get_connection +from mongoengine.context_managers import switch_db __all__ = ("IndexesTest", ) @@ -822,33 +823,29 @@ class IndexesTest(unittest.TestCase): name = StringField(required=True) term = StringField(required=True) - class Report(Document): + class ReportEmbedded(Document): key = EmbeddedDocumentField(CompoundKey, primary_key=True) text = StringField() - Report.drop_collection() - my_key = CompoundKey(name="n", term="ok") - report = Report(text="OK", key=my_key).save() + report = ReportEmbedded(text="OK", key=my_key).save() self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, report.to_mongo()) - self.assertEqual(report, Report.objects.get(pk=my_key)) + self.assertEqual(report, ReportEmbedded.objects.get(pk=my_key)) def test_compound_key_dictfield(self): - class Report(Document): + class ReportDictField(Document): key = DictField(primary_key=True) text = StringField() - Report.drop_collection() - my_key = {"name": "n", "term": "ok"} - report = Report(text="OK", key=my_key).save() + report = ReportDictField(text="OK", key=my_key).save() self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, report.to_mongo()) - self.assertEqual(report, Report.objects.get(pk=my_key)) + self.assertEqual(report, ReportDictField.objects.get(pk=my_key)) def test_string_indexes(self): @@ -909,26 +906,38 @@ class IndexesTest(unittest.TestCase): Issue #812 """ + # Use a new connection and database since dropping the database could + # cause concurrent tests to fail. + connection = connect(db='tempdatabase', + alias='test_indexes_after_database_drop') + class BlogPost(Document): title = StringField() slug = StringField(unique=True) - BlogPost.drop_collection() + meta = {'db_alias': 'test_indexes_after_database_drop'} - # Create Post #1 - post1 = BlogPost(title='test1', slug='test') - post1.save() + try: + BlogPost.drop_collection() - # Drop the Database - self.connection.drop_database(BlogPost._get_db().name) + # Create Post #1 + post1 = BlogPost(title='test1', slug='test') + post1.save() - # Re-create Post #1 - post1 = BlogPost(title='test1', slug='test') - post1.save() + # Drop the Database + connection.drop_database('tempdatabase') + + # Re-create Post #1 + post1 = BlogPost(title='test1', slug='test') + post1.save() + + # Create Post #2 + post2 = BlogPost(title='test2', slug='test') + self.assertRaises(NotUniqueError, post2.save) + finally: + # Drop the temporary database at the end + connection.drop_database('tempdatabase') - # Create Post #2 - post2 = BlogPost(title='test2', slug='test') - self.assertRaises(NotUniqueError, post2.save) def test_index_dont_send_cls_option(self): """ From d9b3a9fb60ecff7b250a5db3cfecdc406bfc15d5 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Fri, 18 Mar 2016 19:51:09 +0100 Subject: [PATCH 77/99] Use six integer types instead of explicit types, since six is now a dependency of the project. --- mongoengine/fields.py | 1 - tests/fields/fields.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 206edcff..c807809a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,7 +20,6 @@ else: import pymongo import gridfs from bson import Binary, DBRef, SON, ObjectId - try: from bson.int64 import Int64 except ImportError: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index c09cbf29..9af87f7f 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -12,6 +12,7 @@ import uuid import math import itertools import re +import six try: import dateutil @@ -3620,7 +3621,7 @@ class FieldTest(unittest.TestCase): doc = TestLongFieldConsideredAsInt64(some_long=42).save() db = get_db() self.assertTrue(isinstance(db.test_long_field_considered_as_int64.find()[0]['some_long'], Int64)) - self.assertTrue(isinstance(doc.some_long, (int, long,))) + self.assertTrue(isinstance(doc.some_long, six.integer_types)) class EmbeddedDocumentListFieldTestCase(unittest.TestCase): From e34100bab4678b41b38568c933a318b50c7a280d Mon Sep 17 00:00:00 2001 From: Gilb's Date: Fri, 18 Mar 2016 23:43:23 +0100 Subject: [PATCH 78/99] Another attempt to fix random fails of test `test_compound_key_dictfield`. --- tests/document/indexes.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 1b4c4d96..e13d8b84 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -5,21 +5,23 @@ import sys sys.path[0:0] = [""] import pymongo +from random import randint from nose.plugins.skip import SkipTest from datetime import datetime from mongoengine import * from mongoengine.connection import get_db, get_connection -from mongoengine.context_managers import switch_db __all__ = ("IndexesTest", ) class IndexesTest(unittest.TestCase): + _MAX_RAND = 10 ** 10 def setUp(self): - self.connection = connect(db='mongoenginetest') + self.db_name = 'mongoenginetest_IndexesTest_' + str(randint(0, self._MAX_RAND)) + self.connection = connect(db=self.db_name) self.db = get_db() class Person(Document): @@ -33,10 +35,7 @@ class IndexesTest(unittest.TestCase): self.Person = Person def tearDown(self): - for collection in self.db.collection_names(): - if 'system.' in collection: - continue - self.db.drop_collection(collection) + self.connection.drop_database(self.db) def test_indexes_document(self): """Ensure that indexes are used when meta[indexes] is specified for From 4650e5e8fb206812c783c6458a30b24b708a6a03 Mon Sep 17 00:00:00 2001 From: Younes JAAIDI Date: Fri, 25 Mar 2016 12:42:00 +0100 Subject: [PATCH 79/99] Fix MapField in order to handle unicode keys. --- mongoengine/dereference.py | 4 +++- tests/fields/fields.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 7fcc2ad2..0428095a 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,5 +1,7 @@ from bson import DBRef, SON +from mongoengine.python_support import txt_type + from base import ( BaseDict, BaseList, EmbeddedDocumentList, TopLevelDocumentMetaclass, get_document @@ -226,7 +228,7 @@ class DeReference(object): data[k]._data[field_name] = self.object_map.get( (v['_ref'].collection, v['_ref'].id), v) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: - item_name = "{0}.{1}.{2}".format(name, k, field_name) + item_name = txt_type("{0}.{1}.{2}").format(name, k, field_name) data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: item_name = '%s.%s' % (name, k) if name else name diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 15daecb9..4211496c 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1515,6 +1515,29 @@ class FieldTest(unittest.TestCase): actions__friends__operation='drink', actions__friends__object='beer').count()) + def test_map_field_unicode(self): + + class Info(EmbeddedDocument): + description = StringField() + value_list = ListField(field=StringField()) + + class BlogPost(Document): + info_dict = MapField(field=EmbeddedDocumentField(Info)) + + BlogPost.drop_collection() + + tree = BlogPost(info_dict={ + u"éééé": { + 'description': u"VALUE: éééé" + } + }) + + tree.save() + + self.assertEqual(BlogPost.objects.get(id=tree.id).info_dict[u"éééé"].description, u"VALUE: éééé") + + BlogPost.drop_collection() + def test_embedded_db_field(self): class Embedded(EmbeddedDocument): From 3b1509f30723f80cb8badedc863c4df47179473a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sat, 26 Mar 2016 09:13:25 +0300 Subject: [PATCH 80/99] Added changelog entry for #1267. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c34094a7..90de9556 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Changes in 0.10.7 - DEV - Raise `OperationError` when trying to do a `drop_collection` on document with no collection set. - count on ListField of EmbeddedDocumentField fails. #1187 - Fixed long fields stored as int32 in Python 3. #1253 +- MapField now handles unicodes keys correctly. #1267 Changes in 0.10.6 ================= From 109202329f88144c42e9b5e2e5da3707c30ac22d Mon Sep 17 00:00:00 2001 From: Madiyar Aitbayev Date: Thu, 31 Mar 2016 02:33:13 +0100 Subject: [PATCH 81/99] Handles unicode correctly EmbeddedDocumentListField --- mongoengine/base/datastructures.py | 2 +- tests/fields/fields.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index e4d2b392..59dacdb7 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -210,7 +210,7 @@ class EmbeddedDocumentList(BaseList): def __match_all(cls, i, kwargs): items = kwargs.items() return all([ - getattr(i, k) == v or str(getattr(i, k)) == v for k, v in items + getattr(i, k) == v or unicode(getattr(i, k)) == v for k, v in items ]) @classmethod diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7575e8c6..6279774b 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -4033,6 +4033,17 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): # modified self.assertEqual(number, 2) + def test_unicode(self): + """ + Tests that unicode strings handled correctly + """ + post = self.BlogPost(comments=[ + self.Comments(author='user1', message=u'сообщение'), + self.Comments(author='user2', message=u'хабарлама') + ]).save() + self.assertEqual(post.comments.get(message=u'сообщение').author, + 'user1') + def test_save(self): """ Tests the save method of a List of Embedded Documents. From 00430491caeaab6c2693dd437d5f0b25d7809fb4 Mon Sep 17 00:00:00 2001 From: Neurostack Date: Tue, 29 Mar 2016 11:46:03 -0600 Subject: [PATCH 82/99] Fixed bug accessing ListField (BaseList) with negative indices If you __setitem__ in BaseList with a negative index and then try to save this, you will get an error like: OperationError: Could not save document (cannot use the part (shape of signal.shape.-1) to traverse the element ({shape: [ 0 ]})). To fix this I rectify negative list indices in BaseList _mark_as_changed as the appropriate positive index. This fixes the above error. --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base/datastructures.py | 3 ++- tests/fields/fields.py | 13 +++++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 758c52ea..758554c0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -236,3 +236,4 @@ that much better: * Luo Peng (https://github.com/RussellLuo) * Bryan Bennett (https://github.com/bbenne10) * Gilb's Gilb's (https://github.com/gilbsgilbs) + * Joshua Nedrud (https://github.com/Neurostack) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90de9556..3fa9eb74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Changes in 0.10.7 - DEV - count on ListField of EmbeddedDocumentField fails. #1187 - Fixed long fields stored as int32 in Python 3. #1253 - MapField now handles unicodes keys correctly. #1267 +- ListField now handles negative indicies correctly. #1270 Changes in 0.10.6 ================= diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index e4d2b392..ed6a46cc 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -199,7 +199,8 @@ class BaseList(list): def _mark_as_changed(self, key=None): if hasattr(self._instance, '_mark_as_changed'): if key: - self._instance._mark_as_changed('%s.%s' % (self._name, key)) + self._instance._mark_as_changed('%s.%s' % (self._name, + key % len(self))) else: self._instance._mark_as_changed(self._name) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7575e8c6..b87c4366 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1160,6 +1160,19 @@ class FieldTest(unittest.TestCase): simple = simple.reload() self.assertEqual(simple.widgets, [4]) + def test_list_field_with_negative_indices(self): + + class Simple(Document): + widgets = ListField() + + simple = Simple(widgets=[1, 2, 3, 4]).save() + simple.widgets[-1] = 5 + self.assertEqual(['widgets.3'], simple._changed_fields) + simple.save() + + simple = simple.reload() + self.assertEqual(simple.widgets, [1, 2, 3, 5]) + def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From 3f30808104810e61ef34d382b5f5cdfccfb50be9 Mon Sep 17 00:00:00 2001 From: Shu Shen Date: Thu, 7 Apr 2016 09:56:21 -0700 Subject: [PATCH 83/99] Fix AttributeError when creating EmbeddedDocument When an EmbeddedDocument is initialized with positional arguments, the document attempts to read _auto_id_field attribute which may not exist and would throw an AttributeError exception and fail the initialization. This change and the test is based on the discussion in issue #681 and PR #777 with a number of community members. --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base/document.py | 2 +- tests/document/instance.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 758554c0..59660cbb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -237,3 +237,4 @@ that much better: * Bryan Bennett (https://github.com/bbenne10) * Gilb's Gilb's (https://github.com/gilbsgilbs) * Joshua Nedrud (https://github.com/Neurostack) + * Shu Shen (https://github.com/shushen) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3fa9eb74..c2a0a527 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Changes in 0.10.7 - DEV - Fixed long fields stored as int32 in Python 3. #1253 - MapField now handles unicodes keys correctly. #1267 - ListField now handles negative indicies correctly. #1270 +- Fixed AttributeError when initializing EmbeddedDocument with positional args. #681 Changes in 0.10.6 ================= diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a890081a..5641426d 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -51,7 +51,7 @@ class BaseDocument(object): # We only want named arguments. field = iter(self._fields_ordered) # If its an automatic id field then skip to the first defined field - if self._auto_id_field: + if getattr(self, '_auto_id_field', False): next(field) for value in args: name = next(field) diff --git a/tests/document/instance.py b/tests/document/instance.py index 4cfe8b8e..978488f1 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2928,6 +2928,20 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) + def test_positional_creation_embedded(self): + """Ensure that embedded document may be created using positional arguments. + """ + job = self.Job("Test Job", 4) + self.assertEqual(job.name, "Test Job") + self.assertEqual(job.years, 4) + + def test_mixed_creation_embedded(self): + """Ensure that embedded document may be created using mixed arguments. + """ + job = self.Job("Test Job", years=4) + self.assertEqual(job.name, "Test Job") + self.assertEqual(job.years, 4) + def test_mixed_creation_dynamic(self): """Ensure that document may be created using mixed arguments. """ From 7d0ec33b54d97622a0e845cf0382b33ae40476f5 Mon Sep 17 00:00:00 2001 From: vahan Date: Sun, 1 May 2016 22:59:39 -0400 Subject: [PATCH 84/99] * fixed the bug where dynamic doc has indx inside dict field --- mongoengine/base/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 05b3ad95..63dbf0db 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -974,7 +974,7 @@ class BaseDocument(object): if hasattr(getattr(field, 'field', None), 'lookup_member'): new_field = field.field.lookup_member(field_name) elif cls._dynamic and (isinstance(field, DynamicField) or - getattr(getattr(field, 'document_type'), '_dynamic')): + getattr(getattr(field, 'document_type', None), '_dynamic', None)): new_field = DynamicField(db_field=field_name) else: # Look up subfield on the previous field or raise From f750796444fbc25b4420d11aa46954b0b0a31d04 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 4 May 2016 17:11:38 -0700 Subject: [PATCH 85/99] fix typo --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 6e5f7220..5121463b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -38,7 +38,7 @@ class QuerySet(BaseQuerySet): def __len__(self): """Since __len__ is called quite frequently (for example, as part of - list(qs) we populate the result cache and cache the length. + list(qs)), we populate the result cache and cache the length. """ if self._len is not None: return self._len From 1f223aa7e649340b824ad04b74d6bbe89daf6cae Mon Sep 17 00:00:00 2001 From: xiaost Date: Thu, 26 May 2016 00:29:41 +0800 Subject: [PATCH 86/99] Fix no_cursor_timeout with pymongo3 --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 4bc63597..67b1af90 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1462,7 +1462,7 @@ class BaseQuerySet(object): msg = "The snapshot option is not anymore available with PyMongo 3+" warnings.warn(msg, DeprecationWarning) cursor_args = { - 'no_cursor_timeout': self._timeout + 'no_cursor_timeout': not self._timeout } if self._loaded_fields: cursor_args[fields_name] = self._loaded_fields.as_dict() From be0aee95f2009a818b3166fc5b92158994ed6305 Mon Sep 17 00:00:00 2001 From: xiaost Date: Fri, 3 Jun 2016 01:27:39 +0800 Subject: [PATCH 87/99] Update changelog for #1304 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 59660cbb..8a983f86 100644 --- a/AUTHORS +++ b/AUTHORS @@ -238,3 +238,4 @@ that much better: * Gilb's Gilb's (https://github.com/gilbsgilbs) * Joshua Nedrud (https://github.com/Neurostack) * Shu Shen (https://github.com/shushen) + * xiaost7 (https://github.com/xiaost7) diff --git a/docs/changelog.rst b/docs/changelog.rst index c2a0a527..8337c757 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Changes in 0.10.7 - DEV - MapField now handles unicodes keys correctly. #1267 - ListField now handles negative indicies correctly. #1270 - Fixed AttributeError when initializing EmbeddedDocument with positional args. #681 +- Fixed no_cursor_timeout error with pymongo 3.0+ #1304 Changes in 0.10.6 ================= From 4544afe4224d7a1b3278d77e8c3ca58d3c8bcadd Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Thu, 16 Jun 2016 21:21:10 +0100 Subject: [PATCH 88/99] Convert readthedocs links for their .org -> .io migration for hosted projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per [their blog post of the 27th April](https://blog.readthedocs.com/securing-subdomains/) ‘Securing subdomains’: > Starting today, Read the Docs will start hosting projects from subdomains on the domain readthedocs.io, instead of on readthedocs.org. This change addresses some security concerns around site cookies while hosting user generated data on the same domain as our dashboard. Test Plan: Manually visited all the links I’ve modified. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 08e95029..c94ec45c 100644 --- a/README.rst +++ b/README.rst @@ -19,10 +19,10 @@ MongoEngine About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. -Documentation available at http://mongoengine-odm.rtfd.org - there is currently -a `tutorial `_, a `user guide -`_ and an `API reference -`_. +Documentation available at https://mongoengine-odm.readthedocs.io - there is currently +a `tutorial `_, a `user guide +`_ and an `API reference +`_. Installation ============ From c21dcf14deea22f24a18a8860fd0394052350e8c Mon Sep 17 00:00:00 2001 From: vahan Date: Fri, 24 Jun 2016 13:45:42 -0400 Subject: [PATCH 89/99] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8337c757..a5b0dd2f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.10.7 - DEV ======================= +- Fixed the bug where dynamic doc has index inside a dict field #1278 - Fixed not being able to specify `use_db_field=False` on `ListField(EmbeddedDocumentField)` instances - Fixed cascade delete mixing among collections #1224 - Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 From 21974f7288a1f0aa7a62dd3e40f25ea3a2ea7da7 Mon Sep 17 00:00:00 2001 From: Anentropic Date: Wed, 29 Jun 2016 14:24:33 +0100 Subject: [PATCH 90/99] better description for upsert arg on some methods --- mongoengine/queryset/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 4bc63597..ea988732 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -434,7 +434,7 @@ class BaseQuerySet(object): full_result=False, **update): """Perform an atomic update on the fields matched by the query. - :param upsert: Any existing document with that "_id" is overwritten. + :param upsert: insert if document doesn't exist (default ``False``) :param multi: Update multiple documents. :param write_concern: Extra keyword arguments are passed down which will be used as options for the resultant @@ -480,7 +480,6 @@ class BaseQuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def upsert_one(self, write_concern=None, **update): """Overwrite or add the first document matched by the query. @@ -498,7 +497,7 @@ class BaseQuerySet(object): """ atomic_update = self.update(multi=False, upsert=True, write_concern=write_concern, - full_result=True,**update) + full_result=True, **update) if atomic_update['updatedExisting']: document = self.get() @@ -510,7 +509,7 @@ class BaseQuerySet(object): """Perform an atomic update on the fields of the first document matched by the query. - :param upsert: Any existing document with that "_id" is overwritten. + :param upsert: insert if document doesn't exist (default ``False``) :param write_concern: Extra keyword arguments are passed down which will be used as options for the resultant ``getLastError`` command. For example, From df9ed835caba83742b3ddd4995b72bc9541ccc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 2 Jul 2016 22:51:54 +0200 Subject: [PATCH 91/99] fixes in unit tests --- tests/document/instance.py | 4 ++-- tests/fixtures.py | 2 +- tests/migration/__init__.py | 5 ++++- ...ect_id.py => referencefield_dbref_to_object_id.py} | 0 tests/test_dereference.py | 11 ++++++++--- 5 files changed, 15 insertions(+), 7 deletions(-) rename tests/migration/{refrencefield_dbref_to_object_id.py => referencefield_dbref_to_object_id.py} (100%) diff --git a/tests/document/instance.py b/tests/document/instance.py index 978488f1..ba01db4c 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -13,7 +13,7 @@ from datetime import datetime from bson import DBRef, ObjectId from tests import fixtures from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest, - PickleDyanmicEmbedded, PickleDynamicTest) + PickleDynamicEmbedded, PickleDynamicTest) from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, @@ -2317,7 +2317,7 @@ class InstanceTest(unittest.TestCase): pickle_doc = PickleDynamicTest( name="test", number=1, string="One", lists=['1', '2']) - pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar") + pickle_doc.embedded = PickleDynamicEmbedded(foo="Bar") pickled_doc = pickle.dumps(pickle_doc) # make sure pickling works even before the doc is saved pickle_doc.save() diff --git a/tests/fixtures.py b/tests/fixtures.py index b3bf73e8..d8eb8487 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -26,7 +26,7 @@ class NewDocumentPickleTest(Document): new_field = StringField() -class PickleDyanmicEmbedded(DynamicEmbeddedDocument): +class PickleDynamicEmbedded(DynamicEmbeddedDocument): date = DateTimeField(default=datetime.now) diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index 6fc83e02..ef62d876 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,8 +1,11 @@ +import unittest + from convert_to_new_inheritance_model import * from decimalfield_as_float import * -from refrencefield_dbref_to_object_id import * +from referencefield_dbref_to_object_id import * from turn_off_inheritance import * from uuidfield_to_binary import * + if __name__ == '__main__': unittest.main() diff --git a/tests/migration/refrencefield_dbref_to_object_id.py b/tests/migration/referencefield_dbref_to_object_id.py similarity index 100% rename from tests/migration/refrencefield_dbref_to_object_id.py rename to tests/migration/referencefield_dbref_to_object_id.py diff --git a/tests/test_dereference.py b/tests/test_dereference.py index e1ae3740..11bdd612 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -12,9 +12,13 @@ from mongoengine.context_managers import query_counter class FieldTest(unittest.TestCase): - def setUp(self): - connect(db='mongoenginetest') - self.db = get_db() + @classmethod + def setUpClass(cls): + cls.db = connect(db='mongoenginetest') + + @classmethod + def tearDownClass(cls): + cls.db.drop_database('mongoenginetest') def test_list_item_dereference(self): """Ensure that DBRef items in ListFields are dereferenced. @@ -304,6 +308,7 @@ class FieldTest(unittest.TestCase): User.drop_collection() Post.drop_collection() + SimpleList.drop_collection() u1 = User.objects.create(name='u1') u2 = User.objects.create(name='u2') From 16fea6f00941c75163b9a4f5f10affedb4916101 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Sun, 10 Jul 2016 13:21:12 -0700 Subject: [PATCH 92/99] replace QuerySet.sum/average implementations with aggregate_sum/average + tweaks --- mongoengine/queryset/base.py | 141 +++++++---------------------------- tests/queryset/queryset.py | 132 +++++++------------------------- 2 files changed, 53 insertions(+), 220 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 97bf5d5a..4b8461fe 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1237,66 +1237,28 @@ class BaseQuerySet(object): def sum(self, field): """Sum over the values of the specified field. - :param field: the field to sum over; use dot-notation to refer to + :param field: the field to sum over; use dot notation to refer to embedded document fields - - .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work - with sharding. """ - map_func = """ - function() { - var path = '{{~%(field)s}}'.split('.'), - field = this; - - for (p in path) { - if (typeof field != 'undefined') - field = field[path[p]]; - else - break; - } - - if (field && field.constructor == Array) { - field.forEach(function(item) { - emit(1, item||0); - }); - } else if (typeof field != 'undefined') { - emit(1, field||0); - } - } - """ % dict(field=field) - - reduce_func = Code(""" - function(key, values) { - var sum = 0; - for (var i in values) { - sum += values[i]; - } - return sum; - } - """) - - for result in self.map_reduce(map_func, reduce_func, output='inline'): - return result.value - else: - return 0 - - def aggregate_sum(self, field): - """Sum over the values of the specified field. - - :param field: the field to sum over; use dot-notation to refer to - embedded document fields - - This method is more performant than the regular `sum`, because it uses - the aggregation framework instead of map-reduce. - """ - result = self._document._get_collection().aggregate([ + pipeline = [ {'$match': self._query}, {'$group': {'_id': 'sum', 'total': {'$sum': '$' + field}}} - ]) + ] + + # if we're performing a sum over a list field, we sum up all the + # elements in the list, hence we need to $unwind the arrays first + ListField = _import_class('ListField') + field_parts = field.split('.') + field_instances = self._document._lookup_field(field_parts) + if isinstance(field_instances[-1], ListField): + pipeline.insert(1, {'$unwind': '$' + field}) + + result = self._document._get_collection().aggregate(pipeline) if IS_PYMONGO_3: result = list(result) else: result = result.get('result') + if result: return result[0]['total'] return 0 @@ -1304,71 +1266,24 @@ class BaseQuerySet(object): def average(self, field): """Average over the values of the specified field. - :param field: the field to average over; use dot-notation to refer to + :param field: the field to average over; use dot notation to refer to embedded document fields - - .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work - with sharding. """ - map_func = """ - function() { - var path = '{{~%(field)s}}'.split('.'), - field = this; - - for (p in path) { - if (typeof field != 'undefined') - field = field[path[p]]; - else - break; - } - - if (field && field.constructor == Array) { - field.forEach(function(item) { - emit(1, {t: item||0, c: 1}); - }); - } else if (typeof field != 'undefined') { - emit(1, {t: field||0, c: 1}); - } - } - """ % dict(field=field) - - reduce_func = Code(""" - function(key, values) { - var out = {t: 0, c: 0}; - for (var i in values) { - var value = values[i]; - out.t += value.t; - out.c += value.c; - } - return out; - } - """) - - finalize_func = Code(""" - function(key, value) { - return value.t / value.c; - } - """) - - for result in self.map_reduce(map_func, reduce_func, - finalize_f=finalize_func, output='inline'): - return result.value - else: - return 0 - - def aggregate_average(self, field): - """Average over the values of the specified field. - - :param field: the field to average over; use dot-notation to refer to - embedded document fields - - This method is more performant than the regular `average`, because it - uses the aggregation framework instead of map-reduce. - """ - result = self._document._get_collection().aggregate([ + pipeline = [ {'$match': self._query}, {'$group': {'_id': 'avg', 'total': {'$avg': '$' + field}}} - ]) + ] + + # if we're performing an average over a list field, we average out + # all the elements in the list, hence we need to $unwind the arrays + # first + ListField = _import_class('ListField') + field_parts = field.split('.') + field_instances = self._document._lookup_field(field_parts) + if isinstance(field_instances[-1], ListField): + pipeline.insert(1, {'$unwind': '$' + field}) + + result = self._document._get_collection().aggregate(pipeline) if IS_PYMONGO_3: result = list(result) else: diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f6a522af..9d926803 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2766,25 +2766,15 @@ class QuerySetTest(unittest.TestCase): avg = float(sum(ages)) / (len(ages) + 1) # take into account the 0 self.assertAlmostEqual(int(self.Person.objects.average('age')), avg) - self.assertAlmostEqual( - int(self.Person.objects.aggregate_average('age')), avg - ) self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.average('age')), avg) - self.assertEqual( - int(self.Person.objects.aggregate_average('age')), avg - ) # dot notation self.Person( name='person meta', person_meta=self.PersonMeta(weight=0)).save() self.assertAlmostEqual( int(self.Person.objects.average('person_meta.weight')), 0) - self.assertAlmostEqual( - int(self.Person.objects.aggregate_average('person_meta.weight')), - 0 - ) for i, weight in enumerate(ages): self.Person( @@ -2793,19 +2783,11 @@ class QuerySetTest(unittest.TestCase): self.assertAlmostEqual( int(self.Person.objects.average('person_meta.weight')), avg ) - self.assertAlmostEqual( - int(self.Person.objects.aggregate_average('person_meta.weight')), - avg - ) self.Person(name='test meta none').save() self.assertEqual( int(self.Person.objects.average('person_meta.weight')), avg ) - self.assertEqual( - int(self.Person.objects.aggregate_average('person_meta.weight')), - avg - ) # test summing over a filtered queryset over_50 = [a for a in ages if a >= 50] @@ -2814,10 +2796,6 @@ class QuerySetTest(unittest.TestCase): self.Person.objects.filter(age__gte=50).average('age'), avg ) - self.assertEqual( - self.Person.objects.filter(age__gte=50).aggregate_average('age'), - avg - ) def test_sum(self): """Ensure that field can be summed over correctly. @@ -2827,15 +2805,9 @@ class QuerySetTest(unittest.TestCase): self.Person(name='test%s' % i, age=age).save() self.assertEqual(self.Person.objects.sum('age'), sum(ages)) - self.assertEqual( - self.Person.objects.aggregate_sum('age'), sum(ages) - ) self.Person(name='ageless person').save() self.assertEqual(self.Person.objects.sum('age'), sum(ages)) - self.assertEqual( - self.Person.objects.aggregate_sum('age'), sum(ages) - ) for i, age in enumerate(ages): self.Person(name='test meta%s' % @@ -2844,26 +2816,15 @@ class QuerySetTest(unittest.TestCase): self.assertEqual( self.Person.objects.sum('person_meta.weight'), sum(ages) ) - self.assertEqual( - self.Person.objects.aggregate_sum('person_meta.weight'), - sum(ages) - ) self.Person(name='weightless person').save() self.assertEqual(self.Person.objects.sum('age'), sum(ages)) - self.assertEqual( - self.Person.objects.aggregate_sum('age'), sum(ages) - ) # test summing over a filtered queryset self.assertEqual( self.Person.objects.filter(age__gte=50).sum('age'), sum([a for a in ages if a >= 50]) ) - self.assertEqual( - self.Person.objects.filter(age__gte=50).aggregate_sum('age'), - sum([a for a in ages if a >= 50]) - ) def test_embedded_average(self): class Pay(EmbeddedDocument): @@ -2876,21 +2837,12 @@ class QuerySetTest(unittest.TestCase): Doc.drop_collection() - Doc(name=u"Wilson Junior", - pay=Pay(value=150)).save() + Doc(name='Wilson Junior', pay=Pay(value=150)).save() + Doc(name='Isabella Luanna', pay=Pay(value=530)).save() + Doc(name='Tayza mariana', pay=Pay(value=165)).save() + Doc(name='Eliana Costa', pay=Pay(value=115)).save() - Doc(name=u"Isabella Luanna", - pay=Pay(value=530)).save() - - Doc(name=u"Tayza mariana", - pay=Pay(value=165)).save() - - Doc(name=u"Eliana Costa", - pay=Pay(value=115)).save() - - self.assertEqual( - Doc.objects.average('pay.value'), - 240) + self.assertEqual(Doc.objects.average('pay.value'), 240) def test_embedded_array_average(self): class Pay(EmbeddedDocument): @@ -2898,26 +2850,16 @@ class QuerySetTest(unittest.TestCase): class Doc(Document): name = StringField() - pay = EmbeddedDocumentField( - Pay) + pay = EmbeddedDocumentField(Pay) Doc.drop_collection() - Doc(name=u"Wilson Junior", - pay=Pay(values=[150, 100])).save() + Doc(name='Wilson Junior', pay=Pay(values=[150, 100])).save() + Doc(name='Isabella Luanna', pay=Pay(values=[530, 100])).save() + Doc(name='Tayza mariana', pay=Pay(values=[165, 100])).save() + Doc(name='Eliana Costa', pay=Pay(values=[115, 100])).save() - Doc(name=u"Isabella Luanna", - pay=Pay(values=[530, 100])).save() - - Doc(name=u"Tayza mariana", - pay=Pay(values=[165, 100])).save() - - Doc(name=u"Eliana Costa", - pay=Pay(values=[115, 100])).save() - - self.assertEqual( - Doc.objects.average('pay.values'), - 170) + self.assertEqual(Doc.objects.average('pay.values'), 170) def test_array_average(self): class Doc(Document): @@ -2930,9 +2872,7 @@ class QuerySetTest(unittest.TestCase): Doc(values=[165, 100]).save() Doc(values=[115, 100]).save() - self.assertEqual( - Doc.objects.average('values'), - 170) + self.assertEqual(Doc.objects.average('values'), 170) def test_embedded_sum(self): class Pay(EmbeddedDocument): @@ -2940,26 +2880,16 @@ class QuerySetTest(unittest.TestCase): class Doc(Document): name = StringField() - pay = EmbeddedDocumentField( - Pay) + pay = EmbeddedDocumentField(Pay) Doc.drop_collection() - Doc(name=u"Wilson Junior", - pay=Pay(value=150)).save() + Doc(name='Wilson Junior', pay=Pay(value=150)).save() + Doc(name='Isabella Luanna', pay=Pay(value=530)).save() + Doc(name='Tayza mariana', pay=Pay(value=165)).save() + Doc(name='Eliana Costa', pay=Pay(value=115)).save() - Doc(name=u"Isabella Luanna", - pay=Pay(value=530)).save() - - Doc(name=u"Tayza mariana", - pay=Pay(value=165)).save() - - Doc(name=u"Eliana Costa", - pay=Pay(value=115)).save() - - self.assertEqual( - Doc.objects.sum('pay.value'), - 960) + self.assertEqual(Doc.objects.sum('pay.value'), 960) def test_embedded_array_sum(self): class Pay(EmbeddedDocument): @@ -2967,26 +2897,16 @@ class QuerySetTest(unittest.TestCase): class Doc(Document): name = StringField() - pay = EmbeddedDocumentField( - Pay) + pay = EmbeddedDocumentField(Pay) Doc.drop_collection() - Doc(name=u"Wilson Junior", - pay=Pay(values=[150, 100])).save() + Doc(name='Wilson Junior', pay=Pay(values=[150, 100])).save() + Doc(name='Isabella Luanna', pay=Pay(values=[530, 100])).save() + Doc(name='Tayza mariana', pay=Pay(values=[165, 100])).save() + Doc(name='Eliana Costa', pay=Pay(values=[115, 100])).save() - Doc(name=u"Isabella Luanna", - pay=Pay(values=[530, 100])).save() - - Doc(name=u"Tayza mariana", - pay=Pay(values=[165, 100])).save() - - Doc(name=u"Eliana Costa", - pay=Pay(values=[115, 100])).save() - - self.assertEqual( - Doc.objects.sum('pay.values'), - 1360) + self.assertEqual(Doc.objects.sum('pay.values'), 1360) def test_array_sum(self): class Doc(Document): @@ -2999,9 +2919,7 @@ class QuerySetTest(unittest.TestCase): Doc(values=[165, 100]).save() Doc(values=[115, 100]).save() - self.assertEqual( - Doc.objects.sum('values'), - 1360) + self.assertEqual(Doc.objects.sum('values'), 1360) def test_distinct(self): """Ensure that the QuerySet.distinct method works. From 3fa9e703836b978076de80193235d97e2f169654 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 11 Jul 2016 10:42:27 -0700 Subject: [PATCH 93/99] prefer tuples over lists for immutable structures --- mongoengine/queryset/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 4b8461fe..7efb0fb6 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1255,7 +1255,7 @@ class BaseQuerySet(object): result = self._document._get_collection().aggregate(pipeline) if IS_PYMONGO_3: - result = list(result) + result = tuple(result) else: result = result.get('result') @@ -1285,7 +1285,7 @@ class BaseQuerySet(object): result = self._document._get_collection().aggregate(pipeline) if IS_PYMONGO_3: - result = list(result) + result = tuple(result) else: result = result.get('result') if result: From 6148a608fb1fe588549870b76bcf43a52a8879d3 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 11 Jul 2016 10:45:40 -0700 Subject: [PATCH 94/99] update the changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a5b0dd2f..c52f2776 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,7 @@ Changes in 0.10.7 - DEV - ListField now handles negative indicies correctly. #1270 - Fixed AttributeError when initializing EmbeddedDocument with positional args. #681 - Fixed no_cursor_timeout error with pymongo 3.0+ #1304 +- Replaced map-reduce based QuerySet.sum/average with aggregation-based implementations #1336 Changes in 0.10.6 ================= From f35034b989d83cae883c37524d86711a4014046f Mon Sep 17 00:00:00 2001 From: Amos Latteier Date: Mon, 18 Jul 2016 13:23:01 -0700 Subject: [PATCH 95/99] fix example for register_delete_rule. see issue #1339 --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c807809a..2db6383c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -890,7 +890,7 @@ class ReferenceField(BaseField): content = StringField() foo = ReferenceField('Foo') - Bar.register_delete_rule(Foo, 'foo', NULLIFY) + Foo.register_delete_rule(Bar, 'foo', NULLIFY) .. note :: `reverse_delete_rule` does not trigger pre / post delete signals to be From 709983eea673a55564fce9a5d497ac3cf050abd1 Mon Sep 17 00:00:00 2001 From: DionysusG Date: Thu, 4 Aug 2016 16:21:52 +0800 Subject: [PATCH 96/99] fix typo at docs/guide/defineing-documents.rst --- docs/guide/defining-documents.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 1a5fbfc9..6ac88f01 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -29,7 +29,7 @@ documents are serialized based on their field order. Dynamic document schemas ======================== -One of the benefits of MongoDb is dynamic schemas for a collection, whilst data +One of the benefits of MongoDB is dynamic schemas for a collection, whilst data should be planned and organised (after all explicit is better than implicit!) there are scenarios where having dynamic / expando style documents is desirable. From 8f55d385d674211f01a8eb78088f4324664adffe Mon Sep 17 00:00:00 2001 From: Gallaecio Date: Thu, 11 Aug 2016 08:52:53 +0200 Subject: [PATCH 97/99] Fix array-slicing documentation Fixes #1359. --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5f7a3de9..913de5d6 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -237,7 +237,7 @@ is preferred for achieving this:: # All except for the first 5 people users = User.objects[5:] - # 5 users, starting from the 10th user found + # 5 users, starting from the 11th user found users = User.objects[10:15] You may also index the query to retrieve a single result. If an item at that From 5ef59c06dff223acfcce23f41780abc1faf47aa2 Mon Sep 17 00:00:00 2001 From: Sergey Kovalev Date: Sat, 13 Aug 2016 09:41:26 +0300 Subject: [PATCH 98/99] Fix misleading comment about the descriptor --- mongoengine/queryset/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 47c2143d..199205e9 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -29,7 +29,7 @@ class QuerySetManager(object): Document.objects is accessed. """ if instance is not None: - # Document class being used rather than a document object + # Document object being used rather than a document class return self # owner is the document that contains the QuerySetManager From 327e164869e9fd1bda1a7065863bd1ad13981e32 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 4 Aug 2016 08:31:19 +0300 Subject: [PATCH 99/99] Fix for #1176 -- similar to https://github.com/MongoEngine/mongoengine/pull/982 but for `update`. --- AUTHORS | 1 + README.rst | 2 +- docs/changelog.rst | 1 + mongoengine/queryset/transform.py | 6 +++++- tests/queryset/transform.py | 4 ++++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8a983f86..9ed06c7a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -239,3 +239,4 @@ that much better: * Joshua Nedrud (https://github.com/Neurostack) * Shu Shen (https://github.com/shushen) * xiaost7 (https://github.com/xiaost7) + * Victor Varvaryuk diff --git a/README.rst b/README.rst index c94ec45c..547ecbd9 100644 --- a/README.rst +++ b/README.rst @@ -99,7 +99,7 @@ Some simple examples of what MongoEngine code looks like: Tests ===== To run the test suite, ensure you are running a local instance of MongoDB on -the standard port, and run: ``python setup.py nosetests``. +the standard port and have installed ``nose`` and ``rednose``, and run: ``python setup.py nosetests``. To run the test suite on every supported Python version and every supported PyMongo version, you can use ``tox``. diff --git a/docs/changelog.rst b/docs/changelog.rst index c52f2776..e71b1ff9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,7 @@ Changes in 0.10.7 - DEV - Fixed AttributeError when initializing EmbeddedDocument with positional args. #681 - Fixed no_cursor_timeout error with pymongo 3.0+ #1304 - Replaced map-reduce based QuerySet.sum/average with aggregation-based implementations #1336 +- Fixed support for `__` to escape field names that match operators names in `update` #1351 Changes in 0.10.6 ================= diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 13302afa..e5e7f83f 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -44,7 +44,7 @@ def query(_doc_cls=None, **kwargs): if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() - # Allw to escape operator-like field name by __ + # Allow to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == "": parts.pop() @@ -212,6 +212,10 @@ def update(_doc_cls=None, **update): if parts[-1] in COMPARISON_OPERATORS: match = parts.pop() + # Allow to escape operator-like field name by __ + if len(parts) > 1 and parts[-1] == "": + parts.pop() + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index a543317a..1cb8223d 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -224,6 +224,10 @@ class TransformTest(unittest.TestCase): self.assertEqual(1, Doc.objects(item__type__="axe").count()) self.assertEqual(1, Doc.objects(item__name__="Heroic axe").count()) + Doc.objects(id=doc.id).update(set__item__type__='sword') + self.assertEqual(1, Doc.objects(item__type__="sword").count()) + self.assertEqual(0, Doc.objects(item__type__="axe").count()) + def test_understandable_error_raised(self): class Event(Document): title = StringField()