From 5a038de1d52405e32602b238cb66b4c3f779fc7c Mon Sep 17 00:00:00 2001 From: Lars Butler Date: Mon, 5 Oct 2015 14:31:41 +0200 Subject: [PATCH 01/22] fields.ReferenceField: add integer values to `reverse_delete_rule` docs When I first tried to use the `reverse_delete_rule` feature of `ReferenceField`, I had to dig through the source code to find what the actual integer values were expected to be for DO_NOTHING, NULLIFY, CASCADE, DENY, and PULL (or at least, where these constants were defined so that I could import and use them). This patch adds the integer values for those constants (which are defined in mongoengine.queryset.base) to the docs so that users can easily choose the correct integer value. Note: A possible improvement on this change would be to include `mongoengine.queryset.base` module documentation in the generated docs, and then update the `ReferenceField` docs to link to the documentation of these constants (DO_NOTHING, NULLIFY, etc.). --- mongoengine/fields.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f5899311..ad3c468b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -863,12 +863,11 @@ class ReferenceField(BaseField): The options are: - * DO_NOTHING - don't do anything (default). - * NULLIFY - Updates the reference to null. - * CASCADE - Deletes the documents associated with the reference. - * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.fields.ListField` - of references + * DO_NOTHING (0) - don't do anything (default). + * NULLIFY (1) - Updates the reference to null. + * CASCADE (2) - Deletes the documents associated with the reference. + * DENY (3) - Prevent the deletion of the reference object. + * PULL (4) - Pull the reference from a :class:`~mongoengine.fields.ListField` of references Alternative syntax for registering delete rules (useful when implementing bi-directional delete rules) From e049cef00afb2fc1ae0e315674282a2d39464b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:54:58 -0400 Subject: [PATCH 02/22] Add arbitrary metadata capture to `BaseField`. Includes ability to detect and report conflicts. --- mongoengine/base/fields.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 304c084d..d92e9b3a 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -42,7 +42,8 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None, verbose_name=None, - help_text=None, null=False, sparse=False, custom_data=None): + help_text=None, null=False, sparse=False, custom_data=None + **kwargs): """ :param db_field: The database field to store this field in (defaults to the name of the field) @@ -70,6 +71,7 @@ class BaseField(object): :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values :param custom_data: (optional) Custom metadata for this field. + :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. """ self.db_field = (db_field or name) if not primary_key else '_id' @@ -89,6 +91,15 @@ class BaseField(object): self.sparse = sparse self._owner_document = None self.custom_data = custom_data + + conflicts = set(dir(self)).intersect(kwargs) + if conflicts: + raise TypeError("%s already has attribute(s): %s" % ( + self.__class__.__name__, + ', '.join(conflicts) + )) + + self.__dict__.update(kwargs) # Adjust the appropriate creation counter, and save our local copy. if self.db_field == '_id': From d133913c3dd561e3b0a0002489cad4be9973637f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:59:29 -0400 Subject: [PATCH 03/22] Remove now superfluous special cases. Removes `verbose_name`, `help_text`, and `custom_data`. All three are covered by the one metadata assignment and will continue working as expected. --- mongoengine/base/fields.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index d92e9b3a..3427a094 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -41,8 +41,7 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, - validation=None, choices=None, verbose_name=None, - help_text=None, null=False, sparse=False, custom_data=None + validation=None, choices=None, null=False, sparse=False, **kwargs): """ :param db_field: The database field to store this field in @@ -61,16 +60,10 @@ class BaseField(object): field. Generally this is deprecated in favour of the `FIELD.validate` method :param choices: (optional) The valid choices - :param verbose_name: (optional) The verbose name for the field. - Designed to be human readable and is often used when generating - model forms from the document model. - :param help_text: (optional) The help text for this field and is often - used when generating model forms from the document model. :param null: (optional) Is the field value can be null. If no and there is a default value then the default value is set :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values - :param custom_data: (optional) Custom metadata for this field. :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. """ self.db_field = (db_field or name) if not primary_key else '_id' @@ -85,12 +78,9 @@ class BaseField(object): self.primary_key = primary_key self.validation = validation self.choices = choices - self.verbose_name = verbose_name - self.help_text = help_text self.null = null self.sparse = sparse self._owner_document = None - self.custom_data = custom_data conflicts = set(dir(self)).intersect(kwargs) if conflicts: From 3f3747a2fe660d08bb8af4f079fb2f4b0c3540eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 21:59:46 -0400 Subject: [PATCH 04/22] Minor formatting tweaks and additional comments. --- mongoengine/base/fields.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 3427a094..9f054a2b 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -82,13 +82,14 @@ class BaseField(object): self.sparse = sparse self._owner_document = None + # Detect and report conflicts between metadata and base properties. conflicts = set(dir(self)).intersect(kwargs) if conflicts: raise TypeError("%s already has attribute(s): %s" % ( - self.__class__.__name__, - ', '.join(conflicts) - )) - + self.__class__.__name__, ', '.join(conflicts) )) + + # Assign metadata to the instance + # This efficient method is available because no __slots__ are defined. self.__dict__.update(kwargs) # Adjust the appropriate creation counter, and save our local copy. From a57f28ac832041ccb5c21cb7713c9723a3fd672f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 22:41:58 -0400 Subject: [PATCH 05/22] Correction for local monkeypatch. --- mongoengine/base/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 9f054a2b..d14ecce0 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -83,7 +83,7 @@ class BaseField(object): self._owner_document = None # Detect and report conflicts between metadata and base properties. - conflicts = set(dir(self)).intersect(kwargs) + conflicts = set(dir(self)) & set(kwargs) if conflicts: raise TypeError("%s already has attribute(s): %s" % ( self.__class__.__name__, ', '.join(conflicts) )) From 50b271c868c6b1fac95eb650d1b4b2592540e236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Bevan=E2=80=93McGregor?= Date: Tue, 13 Oct 2015 22:51:03 -0400 Subject: [PATCH 06/22] Arbitrary metadata documentation. --- docs/guide/defining-documents.rst | 10 +++++----- mongoengine/base/fields.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 8f7382ee..c3ad208e 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -172,11 +172,11 @@ arguments can be set on all fields: class Shirt(Document): size = StringField(max_length=3, choices=SIZE) -:attr:`help_text` (Default: None) - Optional help text to output with the field -- used by form libraries - -:attr:`verbose_name` (Default: None) - Optional human-readable name for the field -- used by form libraries +:attr:`**kwargs` (Optional) + You can supply additional metadata as arbitrary additional keyword + arguments. You can not override existing attributes, however. Common + choices include `help_text` and `verbose_name`, commonly used by form and + widget libraries. List fields diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index d14ecce0..b1024526 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -64,7 +64,11 @@ class BaseField(object): then the default value is set :param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False` means that uniqueness won't be enforced for `None` values - :param **kwargs: (optional) Arbitrary indirection-free metadata for this field. + :param **kwargs: (optional) Arbitrary indirection-free metadata for + this field can be supplied as additional keyword arguments and + accessed as attributes of the field. Must not conflict with any + existing attributes. Common metadata includes `verbose_name` and + `help_text`. """ self.db_field = (db_field or name) if not primary_key else '_id' From cceef33fef18886082f35f847677bab67ed56a59 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 17 Nov 2015 14:22:10 +0000 Subject: [PATCH 07/22] Fixed a couple of documentation typos --- mongoengine/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d2d9c5f..251f5bb3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -217,7 +217,7 @@ class Document(BaseDocument): Returns True if the document has been updated or False if the document in the database doesn't match the query. - .. note:: All unsaved changes that has been made to the document are + .. note:: All unsaved changes that have been made to the document are rejected if the method returns True. :param query: the update will be performed only if the document in the @@ -403,7 +403,7 @@ class Document(BaseDocument): def cascade_save(self, *args, **kwargs): """Recursively saves any references / - generic references on an objects""" + generic references on the document""" _refs = kwargs.get('_refs', []) or [] ReferenceField = _import_class('ReferenceField') From 5c464c3f5a6a927a976c671d51d73b56a5f0a62f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 18 Nov 2015 14:07:39 +0200 Subject: [PATCH 08/22] Bumped version to 0.10.1 --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51f0967f..cd349793 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.10.1 - DEV +Changes in 0.10.1 ======================= - Fix infinite recursion with CASCADE delete rules under specific conditions. #1046 - Fix CachedReferenceField bug when loading cached docs as DBRef but failing to save them. #1047 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 474c2154..eefaf08d 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ import errors __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 10, 0) +VERSION = (0, 10, 1) def get_version(): From deb5677a5729a8213285f0637fc2d700c45301a7 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:14:45 -0500 Subject: [PATCH 09/22] Allow shard key to be in an embedded document (#551) --- mongoengine/document.py | 15 +++++++++++--- tests/document/instance.py | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d2d9c5f..bd2e7c5b 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -341,8 +341,12 @@ class Document(BaseDocument): select_dict['_id'] = object_id shard_key = self.__class__._meta.get('shard_key', tuple()) for k in shard_key: - actual_key = self._db_field_map.get(k, k) - select_dict[actual_key] = doc[actual_key] + path = self._lookup_field(k.split('.')) + actual_key = [p.db_field for p in path] + val = doc + for ak in actual_key: + val = val[ak] + select_dict['.'.join(actual_key)] = val def is_new_object(last_error): if last_error is not None: @@ -444,7 +448,12 @@ class Document(BaseDocument): select_dict = {'pk': self.pk} shard_key = self.__class__._meta.get('shard_key', tuple()) for k in shard_key: - select_dict[k] = getattr(self, k) + path = self._lookup_field(k.split('.')) + actual_key = [p.db_field for p in path] + val = self + for ak in actual_key: + val = getattr(val, ak) + select_dict['__'.join(actual_key)] = val return select_dict def update(self, **kwargs): diff --git a/tests/document/instance.py b/tests/document/instance.py index 56e6765a..5494ac6b 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -484,6 +484,20 @@ class InstanceTest(unittest.TestCase): doc.reload() Animal.drop_collection() + def test_reload_sharded_nested(self): + class SuperPhylum(EmbeddedDocument): + name = StringField() + + class Animal(Document): + superphylum = EmbeddedDocumentField(SuperPhylum) + meta = {'shard_key': ('superphylum.name',)} + + Animal.drop_collection() + doc = Animal(superphylum=SuperPhylum(name='Deuterostomia')) + doc.save() + doc.reload() + Animal.drop_collection() + def test_reload_referencing(self): """Ensures reloading updates weakrefs correctly """ @@ -2715,6 +2729,32 @@ class InstanceTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_shard_key_in_embedded_document(self): + class Foo(EmbeddedDocument): + foo = StringField() + + class Bar(Document): + meta = { + 'shard_key': ('foo.foo',) + } + foo = EmbeddedDocumentField(Foo) + bar = StringField() + + foo_doc = Foo(foo='hello') + bar_doc = Bar(foo=foo_doc, bar='world') + bar_doc.save() + + self.assertTrue(bar_doc.id is not None) + + bar_doc.bar = 'baz' + bar_doc.save() + + def change_shard_key(): + bar_doc.foo.foo = 'something' + bar_doc.save() + + self.assertRaises(OperationError, change_shard_key) + def test_shard_key_primary(self): class LogEntry(Document): machine = StringField(primary_key=True) From f6d3bd8ccb692dd98b0ea45a5cd4f28a9068eff1 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:15:27 -0500 Subject: [PATCH 10/22] Update changelog for #551 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cd349793..6275d6d9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.10.2 +================= +- Allow shard key to point to a field in an embedded document. #551 + Changes in 0.10.1 ======================= - Fix infinite recursion with CASCADE delete rules under specific conditions. #1046 From 899e56e5b81e56f703b07df57a7e010f4c0eff32 Mon Sep 17 00:00:00 2001 From: George Macon Date: Thu, 19 Nov 2015 17:15:40 -0500 Subject: [PATCH 11/22] Add gmacon to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4d5e69a3..411e274d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -229,3 +229,4 @@ that much better: * Emile Caron (https://github.com/emilecaron) * Amit Lichtenberg (https://github.com/amitlicht) * Lars Butler (https://github.com/larsbutler) + * George Macon (https://github.com/gmacon) From 0a1ba7c434cc4d437b7992a44b9dc73627f293a9 Mon Sep 17 00:00:00 2001 From: Emmanuel Leblond Date: Sat, 21 Nov 2015 10:25:11 +0100 Subject: [PATCH 12/22] Add SaveConditionError to __all__ --- mongoengine/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 2c5c2946..15830b5c 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -6,7 +6,7 @@ from mongoengine.python_support import txt_type __all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError', 'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError', 'OperationError', 'NotUniqueError', 'FieldDoesNotExist', - 'ValidationError') + 'ValidationError', 'SaveConditionError') class NotRegistered(Exception): From 19cbb442ee0755abde120a91150310824873a7af Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 23 Nov 2015 13:57:15 +0200 Subject: [PATCH 13/22] Added #1129 to the changelog. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6275d6d9..5809314c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 +- Allow arbirary metadata in fields. #1129 Changes in 0.10.1 ======================= From 013227323d8725e6ffd2e13aecfd0f3a9165d276 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 14:29:25 +0000 Subject: [PATCH 14/22] ReferenceFields can now reference abstract Document types A class that inherits from an abstract Document type is stored in the database as a reference with a 'cls' field that is the class name of the document being stored. Fixes #837 --- mongoengine/fields.py | 25 ++++++++++++++++----- tests/fields/fields.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f5899311..13538c89 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -928,9 +928,14 @@ class ReferenceField(BaseField): self._auto_dereference = instance._fields[self.name]._auto_dereference # Dereference DBRefs if self._auto_dereference and isinstance(value, DBRef): - value = self.document_type._get_db().dereference(value) + if hasattr(value, 'cls'): + # Dereference using the class type specified in the reference + cls = get_document(value.cls) + else: + cls = self.document_type + value = cls._get_db().dereference(value) if value is not None: - instance._data[self.name] = self.document_type._from_son(value) + instance._data[self.name] = cls._from_son(value) return super(ReferenceField, self).__get__(instance, owner) @@ -940,22 +945,30 @@ class ReferenceField(BaseField): return document.id return document - id_field_name = self.document_type._meta['id_field'] - id_field = self.document_type._fields[id_field_name] - if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk if id_ is None: self.error('You can only reference documents once they have' ' been saved to the database') + + # Use the attributes from the document instance, so that they + # override the attributes of this field's document type + cls = document else: id_ = document + cls = self.document_type + + id_field_name = cls._meta['id_field'] + id_field = cls._fields[id_field_name] id_ = id_field.to_mongo(id_) if self.dbref: - collection = self.document_type._get_collection_name() + collection = cls._get_collection_name() return DBRef(collection, id_) + elif self.document_type._meta.get('abstract'): + collection = cls._get_collection_name() + return DBRef(collection, id_, cls=cls._class_name) return id_ diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7ef298fc..860a5749 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2281,6 +2281,56 @@ class FieldTest(unittest.TestCase): Member.drop_collection() BlogPost.drop_collection() + def test_reference_class_with_abstract_parent(self): + """Ensure that a class with an abstract parent can be referenced. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Sister(Sibling): + pass + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + Sister.drop_collection() + Brother.drop_collection() + + sister = Sister(name="Alice") + sister.save() + brother = Brother(name="Bob", sibling=sister) + brother.save() + + self.assertEquals(Brother.objects[0].sibling.name, sister.name) + + Sister.drop_collection() + Brother.drop_collection() + + def test_reference_abstract_class(self): + """Ensure that an abstract class instance cannot be used in the + reference of that abstract class. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Sister(Sibling): + pass + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + Sister.drop_collection() + Brother.drop_collection() + + sister = Sibling(name="Alice") + brother = Brother(name="Bob", sibling=sister) + self.assertRaises(ValidationError, brother.save) + + Sister.drop_collection() + Brother.drop_collection() + def test_generic_reference(self): """Ensure that a GenericReferenceField properly dereferences items. """ From f96e68cd11c72ab321bc1b7923bdfffd29f3bb11 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 15:02:19 +0000 Subject: [PATCH 15/22] Made type inheritance a validation check for abstract references --- mongoengine/fields.py | 8 ++++++++ tests/fields/fields.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 13538c89..4c8c8498 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -996,6 +996,14 @@ class ReferenceField(BaseField): self.error('You can only reference documents once they have been ' 'saved to the database') + if self.document_type._meta.get('abstract') and \ + not isinstance(value, self.document_type): + self.error('%s is not an instance of abstract reference' + ' type %s' % (value._class_name, + self.document_type._class_name) + ) + + def lookup_member(self, member_name): return self.document_type._fields.get(member_name) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 860a5749..15daecb9 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2331,6 +2331,31 @@ class FieldTest(unittest.TestCase): Sister.drop_collection() Brother.drop_collection() + def test_abstract_reference_base_type(self): + """Ensure that an an abstract reference fails validation when given a + Document that does not inherit from the abstract type. + """ + class Sibling(Document): + name = StringField() + meta = {"abstract": True} + + class Brother(Sibling): + sibling = ReferenceField(Sibling) + + class Mother(Document): + name = StringField() + + Brother.drop_collection() + Mother.drop_collection() + + mother = Mother(name="Carol") + mother.save() + brother = Brother(name="Bob", sibling=mother) + self.assertRaises(ValidationError, brother.save) + + Brother.drop_collection() + Mother.drop_collection() + def test_generic_reference(self): """Ensure that a GenericReferenceField properly dereferences items. """ From aa9d5969301964c3e25fe5205b4d25ce1a9dd296 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Tue, 10 Nov 2015 15:03:15 +0000 Subject: [PATCH 16/22] Updated documentation for abstract reference changes --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 411e274d..fbca84e4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -230,3 +230,4 @@ that much better: * Amit Lichtenberg (https://github.com/amitlicht) * Lars Butler (https://github.com/larsbutler) * George Macon (https://github.com/gmacon) + * Ashley Whetter (https://github.com/AWhetter) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5809314c..37720431 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 +- ReferenceFields now support abstract document types. #837 Changes in 0.10.1 ======================= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4c8c8498..6ef13640 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -896,6 +896,10 @@ class ReferenceField(BaseField): or as the :class:`~pymongo.objectid.ObjectId`.id . :param reverse_delete_rule: Determines what to do when the referring object is deleted + + .. note :: + A reference to an abstract document type is always stored as a + :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`. """ if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): From 04497aec36232c893755698911edff31c044f192 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Wed, 11 Nov 2015 09:24:31 +0000 Subject: [PATCH 17/22] Fixed setting dbref to True on abstract reference fields causing the reference to be stored incorrectly --- mongoengine/fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 6ef13640..f89d7d8b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -967,12 +967,12 @@ class ReferenceField(BaseField): id_field = cls._fields[id_field_name] id_ = id_field.to_mongo(id_) - if self.dbref: - collection = cls._get_collection_name() - return DBRef(collection, id_) - elif self.document_type._meta.get('abstract'): + if self.document_type._meta.get('abstract'): collection = cls._get_collection_name() return DBRef(collection, id_, cls=cls._class_name) + elif self.dbref: + collection = cls._get_collection_name() + return DBRef(collection, id_) return id_ From 3c0b00e42d63a3f51842b364a5f69f353d10b466 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 23 Nov 2015 15:40:16 +0200 Subject: [PATCH 18/22] Added python 3.5 to the build. --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 34702192..a0d75551 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - '3.2' - '3.3' - '3.4' +- '3.5' - pypy - pypy3 env: diff --git a/tox.ini b/tox.ini index e6aa7c81..2f84d668 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27,py32,py33,py34,pypy,pypy3}-{mg27,mg28} +envlist = {py26,py27,py32,py33,py34,py35,pypy,pypy3}-{mg27,mg28} #envlist = {py26,py27,py32,py33,py34,pypy,pypy3}-{mg27,mg28,mg30,mgdev} [testenv] From a7aead5138f8792e4030328e23531ee7d4024744 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 24 Jun 2015 22:34:38 -0700 Subject: [PATCH 19/22] re-create the cursor object whenever we apply read_preference --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 1 + tests/queryset/queryset.py | 48 +++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 37720431..1ea4d19f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changes in 0.10.2 - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 - ReferenceFields now support abstract document types. #837 +- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 Changes in 0.10.1 ======================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0867e0e2..8880c2e3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -930,6 +930,7 @@ class BaseQuerySet(object): validate_read_preference('read_preference', read_preference) queryset = self.clone() queryset._read_preference = read_preference + queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_preference return queryset def scalar(self, *fields): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3eff7cea..8726801e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4165,7 +4165,11 @@ class QuerySetTest(unittest.TestCase): def test_read_preference(self): class Bar(Document): - pass + txt = StringField() + + meta = { + 'indexes': [ 'txt' ] + } Bar.drop_collection() bars = list(Bar.objects(read_preference=ReadPreference.PRIMARY)) @@ -4177,9 +4181,51 @@ class QuerySetTest(unittest.TestCase): error_class = TypeError self.assertRaises(error_class, Bar.objects, read_preference='Primary') + # read_preference as a kwarg bars = Bar.objects(read_preference=ReadPreference.SECONDARY_PREFERRED) self.assertEqual( bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference as a query set method + bars = Bar.objects.read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after skip + bars = Bar.objects.skip(1) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after limit + bars = Bar.objects.limit(1) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after order_by + bars = Bar.objects.order_by('txt') \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) + + # read_preference after hint + bars = Bar.objects.hint([('txt', 1)]) \ + .read_preference(ReadPreference.SECONDARY_PREFERRED) + self.assertEqual( + bars._read_preference, ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._cursor._Cursor__read_preference, + ReadPreference.SECONDARY_PREFERRED) def test_json_simple(self): From 54975de0f387ad02a75134a4275401f684b855d7 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Sun, 5 Jul 2015 15:59:50 -0700 Subject: [PATCH 20/22] fix read_preference for PyMongo 3+ --- mongoengine/queryset/base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 8880c2e3..ac4764d3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1444,8 +1444,16 @@ class BaseQuerySet(object): def _cursor(self): if self._cursor_obj is None: - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) + # In PyMongo 3+, we define the read preference on a collection + # level, not a cursor level. Thus, we need to get a cloned + # collection object using `with_options` first. + if IS_PYMONGO_3 and self._read_preference is not None: + self._cursor_obj = self._collection\ + .with_options(read_preference=self._read_preference)\ + .find(self._query, **self._cursor_args) + else: + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) # Apply where clauses to cursor if self._where_clause: where_clause = self._sub_js_fields(self._where_clause) From f9284d20cae1b2596c164493f7bac0f240ade4d5 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Nov 2015 07:00:09 +0200 Subject: [PATCH 21/22] Moved #1042 to the next version in the changelog. --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1ea4d19f..a0c49900 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,12 +2,15 @@ Changelog ========= +Changes in 0.10.3 +================= +- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 + Changes in 0.10.2 ================= - Allow shard key to point to a field in an embedded document. #551 - Allow arbirary metadata in fields. #1129 - ReferenceFields now support abstract document types. #837 -- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042 Changes in 0.10.1 ======================= From a6e996d92142866a1d18570fe60d91eca806e14d Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 24 Nov 2015 07:06:54 +0200 Subject: [PATCH 22/22] Added #1165 to the changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0c49900..c9285416 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.10.4 - DEV +======================= +- SaveConditionError is now importable from the top level package. #1165 + Changes in 0.10.3 ================= - Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042