From 1eae97731f2a384cb7b27404aa9b59741155f7bc Mon Sep 17 00:00:00 2001 From: Erdenezul Date: Wed, 30 Aug 2017 12:04:04 +0800 Subject: [PATCH 01/11] Fix Document.modify fail on sharded collection #1569 --- mongoengine/document.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index f1622934..71929cf1 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -280,6 +280,9 @@ class Document(BaseDocument): elif query[id_field] != self.pk: raise InvalidQueryError('Invalid document modify query: it must modify only this document.') + # Need to add shard key to query, or you get an error + query.update(self._object_key) + updated = self._qs(**query).modify(new=True, **update) if updated is None: return False From e90f6a2fa3766872b44edad29a21527659984ae3 Mon Sep 17 00:00:00 2001 From: Andy Yankovsky Date: Thu, 14 Sep 2017 20:28:15 +0300 Subject: [PATCH 02/11] Fix update via pull__something__in=[] --- mongoengine/queryset/transform.py | 46 +++++++++++++++++++------------ tests/queryset/transform.py | 7 +++++ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index a9907ada..1f70a48b 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -101,21 +101,8 @@ def query(_doc_cls=None, **kwargs): value = value['_id'] elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): - # Raise an error if the in/nin/all/near param is not iterable. We need a - # special check for BaseDocument, because - although it's iterable - using - # it as such in the context of this method is most definitely a mistake. - BaseDocument = _import_class('BaseDocument') - if isinstance(value, BaseDocument): - raise TypeError("When using the `in`, `nin`, `all`, or " - "`near`-operators you can\'t use a " - "`Document`, you must wrap your object " - "in a list (object -> [object]).") - elif not hasattr(value, '__iter__'): - raise TypeError("The `in`, `nin`, `all`, or " - "`near`-operators must be applied to an " - "iterable (e.g. a list).") - else: - value = [field.prepare_query_value(op, v) for v in value] + # Raise an error if the in/nin/all/near param is not iterable. + value = _prepare_query_for_iterable(field, op, value) # If we're querying a GenericReferenceField, we need to alter the # key depending on the value: @@ -284,9 +271,15 @@ def update(_doc_cls=None, **update): if isinstance(field, GeoJsonBaseField): value = field.to_mongo(value) - if op == 'push' and isinstance(value, (list, tuple, set)): + if op == 'pull': + if field.required or value is not None: + if match == 'in' and not isinstance(value, dict): + value = _prepare_query_for_iterable(field, op, value) + else: + value = field.prepare_query_value(op, value) + elif op == 'push' and isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] - elif op in (None, 'set', 'push', 'pull'): + elif op in (None, 'set', 'push'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): @@ -439,3 +432,22 @@ def _infer_geometry(value): raise InvalidQueryError('Invalid $geometry data. Can be either a ' 'dictionary or (nested) lists of coordinate(s)') + + +def _prepare_query_for_iterable(field, op, value): + # We need a special check for BaseDocument, because - although it's iterable - using + # it as such in the context of this method is most definitely a mistake. + BaseDocument = _import_class('BaseDocument') + + if isinstance(value, BaseDocument): + raise TypeError("When using the `in`, `nin`, `all`, or " + "`near`-operators you can\'t use a " + "`Document`, you must wrap your object " + "in a list (object -> [object]).") + + if not hasattr(value, '__iter__'): + raise TypeError("The `in`, `nin`, `all`, or " + "`near`-operators must be applied to an " + "iterable (e.g. a list).") + + return [field.prepare_query_value(op, v) for v in value] diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index 20ab0b3f..a043a647 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -28,12 +28,16 @@ class TransformTest(unittest.TestCase): {'name': {'$exists': True}}) def test_transform_update(self): + class LisDoc(Document): + foo = ListField(StringField()) + class DicDoc(Document): dictField = DictField() class Doc(Document): pass + LisDoc.drop_collection() DicDoc.drop_collection() Doc.drop_collection() @@ -51,6 +55,9 @@ class TransformTest(unittest.TestCase): update = transform.update(DicDoc, pull__dictField__test=doc) self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict)) + update = transform.update(LisDoc, pull__foo__in=['a']) + self.assertEqual(update, {'$pull': {'foo': {'$in': ['a']}}}) + def test_query_field_name(self): """Ensure that the correct field name is used when querying. """ From 2f4e2bde6bb1d892ac22928870e01889fa983833 Mon Sep 17 00:00:00 2001 From: Andy Yankovsky Date: Thu, 14 Sep 2017 21:02:53 +0300 Subject: [PATCH 03/11] Update AUTHORS --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 4eac5eb2..2e7b56fc 100644 --- a/AUTHORS +++ b/AUTHORS @@ -244,4 +244,5 @@ that much better: * Stanislav Kaledin (https://github.com/sallyruthstruik) * Dmitry Yantsen (https://github.com/mrTable) * Renjianxin (https://github.com/Davidrjx) - * Erdenezul Batmunkh (https://github.com/erdenezul) \ No newline at end of file + * Erdenezul Batmunkh (https://github.com/erdenezul) + * Andy Yankovsky (https://github.com/werat) From 7d8916b6e9f60995f905885beac32be321417f1a Mon Sep 17 00:00:00 2001 From: Sangmin In Date: Sun, 11 Feb 2018 00:59:47 +0900 Subject: [PATCH 04/11] fix case inconsistent casing of "MongoDB" (#1744) --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index ea1a04c1..bcd0d17f 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -86,7 +86,7 @@ of them stand out as particularly intuitive solutions. Posts ^^^^^ -Happily mongoDB *isn't* a relational database, so we're not going to do it that +Happily MongoDB *isn't* a relational database, so we're not going to do it that way. As it turns out, we can use MongoDB's schemaless nature to provide us with a much nicer solution. We will store all of the posts in *one collection* and each post type will only store the fields it needs. If we later want to add From 0bd2103a8cb10c63371b2d4cefc8809a5d0adc55 Mon Sep 17 00:00:00 2001 From: Andy Yankovsky Date: Tue, 20 Feb 2018 00:02:12 +0300 Subject: [PATCH 05/11] Add test for document update --- tests/document/instance.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/document/instance.py b/tests/document/instance.py index 609bc900..22c44ffa 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1341,6 +1341,23 @@ class InstanceTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") + def test_update_list_field(self): + """Test update on `ListField` with $pull + $in. + """ + class Doc(Document): + foo = ListField(StringField()) + + Doc.drop_collection() + doc = Doc(foo=['a', 'b', 'c']) + doc.save() + + # Update + doc = Doc.objects.first() + doc.update(pull__foo__in=['a', 'c']) + + doc = Doc.objects.first() + self.assertEqual(doc.foo, ['b']) + def test_embedded_update_db_field(self): """Test update on `EmbeddedDocumentField` fields when db_field is other than default. From aa683226416dac3fa4d5fd757091e2c795cf2ff7 Mon Sep 17 00:00:00 2001 From: estein-de Date: Tue, 27 Feb 2018 08:43:09 -0600 Subject: [PATCH 06/11] MongoDB wants dates stored in UTC, but the functions used in this documentation to generate datetime objects would use server's local timezone - fix it! (#1662) --- docs/guide/defining-documents.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d41ae7e6..33b5292f 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -22,7 +22,7 @@ objects** as class attributes to the document class:: class Page(Document): title = StringField(max_length=200, required=True) - date_modified = DateTimeField(default=datetime.datetime.now) + date_modified = DateTimeField(default=datetime.datetime.utcnow) As BSON (the binary format for storing data in mongodb) is order dependent, documents are serialized based on their field order. @@ -224,7 +224,7 @@ store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate user = ReferenceField(User) answers = DictField() - survey_response = SurveyResponse(date=datetime.now(), user=request.user) + survey_response = SurveyResponse(date=datetime.utcnow(), user=request.user) response_form = ResponseForm(request.POST) survey_response.answers = response_form.cleaned_data() survey_response.save() @@ -618,7 +618,7 @@ collection after a given period. See the official documentation for more information. A common usecase might be session data:: class Session(Document): - created = DateTimeField(default=datetime.now) + created = DateTimeField(default=datetime.utcnow) meta = { 'indexes': [ {'fields': ['created'], 'expireAfterSeconds': 3600} From a0947d0c544a58516650a48b73b84d171e693ffc Mon Sep 17 00:00:00 2001 From: John Dupuy Date: Sat, 10 Mar 2018 23:24:04 -0600 Subject: [PATCH 07/11] Edit EmbeddedDocumentListField update() doc --- mongoengine/base/datastructures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 43f32810..fddd945a 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -351,7 +351,8 @@ class EmbeddedDocumentList(BaseList): def update(self, **update): """ - Updates the embedded documents with the given update values. + Updates the embedded documents with the given replacement values. This + function does not support mongoDB update operators such as ``inc__``. .. note:: The embedded document changes are not automatically saved From dabe8c1bb7e189d0dc8bdceb54eb1ca982a1f6b5 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Wed, 14 Mar 2018 14:26:06 -0400 Subject: [PATCH 08/11] highlight places where ValidationError is raised outside of validate() method --- mongoengine/fields.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7932f73a..f169f0f1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -614,6 +614,7 @@ class EmbeddedDocumentField(BaseField): """ def __init__(self, document_type, **kwargs): + # XXX ValidationError raised outside of the "validate" method. if not ( isinstance(document_type, six.string_types) or issubclass(document_type, EmbeddedDocument) @@ -919,8 +920,11 @@ class DictField(ComplexBaseField): self.field = field self._auto_dereference = False self.basecls = basecls or BaseField + + # XXX ValidationError raised outside of the "validate" method. if not issubclass(self.basecls, BaseField): self.error('DictField only accepts dict values') + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) @@ -969,6 +973,7 @@ class MapField(DictField): """ def __init__(self, field=None, *args, **kwargs): + # XXX ValidationError raised outside of the "validate" method. if not isinstance(field, BaseField): self.error('Argument to MapField constructor must be a valid ' 'field') @@ -1028,6 +1033,7 @@ class ReferenceField(BaseField): A reference to an abstract document type is always stored as a :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`. """ + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) @@ -1082,6 +1088,8 @@ class ReferenceField(BaseField): if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk + + # XXX ValidationError raised outside of the "validate" method. if id_ is None: self.error('You can only reference documents once they have' ' been saved to the database') @@ -1121,7 +1129,6 @@ class ReferenceField(BaseField): return self.to_mongo(value) def validate(self, value): - if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)): self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents') @@ -1129,11 +1136,14 @@ class ReferenceField(BaseField): self.error('You can only reference documents once they have been ' 'saved to the database') - if self.document_type._meta.get('abstract') and \ - not isinstance(value, self.document_type): + if ( + self.document_type._meta.get('abstract') and + not isinstance(value, self.document_type) + ): self.error( '%s is not an instance of abstract reference type %s' % ( - self.document_type._class_name) + self.document_type._class_name + ) ) def lookup_member(self, member_name): @@ -1156,6 +1166,7 @@ class CachedReferenceField(BaseField): if fields is None: fields = [] + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) @@ -1230,6 +1241,7 @@ class CachedReferenceField(BaseField): id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] + # XXX ValidationError raised outside of the "validate" method. if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk @@ -1238,7 +1250,6 @@ class CachedReferenceField(BaseField): ' been saved to the database') else: self.error('Only accept a document object') - # TODO: should raise here or will fail next statement value = SON(( ('_id', id_field.to_mongo(id_)), @@ -1256,6 +1267,7 @@ class CachedReferenceField(BaseField): if value is None: return None + # XXX ValidationError raised outside of the "validate" method. if isinstance(value, Document): if value.pk is None: self.error('You can only reference documents once they have' @@ -1269,7 +1281,6 @@ class CachedReferenceField(BaseField): raise NotImplementedError def validate(self, value): - if not isinstance(value, self.document_type): self.error('A CachedReferenceField only accepts documents') @@ -1330,6 +1341,8 @@ class GenericReferenceField(BaseField): elif isinstance(choice, type) and issubclass(choice, Document): self.choices.append(choice._class_name) else: + # XXX ValidationError raised outside of the "validate" + # method. self.error('Invalid choices provided: must be a list of' 'Document subclasses and/or six.string_typess') @@ -1393,6 +1406,7 @@ class GenericReferenceField(BaseField): # We need the id from the saved object to create the DBRef id_ = document.id if id_ is None: + # XXX ValidationError raised outside of the "validate" method. self.error('You can only reference documents once they have' ' been saved to the database') else: @@ -2209,6 +2223,7 @@ class LazyReferenceField(BaseField): automatically call `fetch()` and try to retrive the field on the fetched document. Note this only work getting field (not setting or deleting). """ + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) From e46779f87b8c6cc154d90939f10d35ffd06469ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malthe=20J=C3=B8rgensen?= Date: Tue, 27 Mar 2018 14:14:08 +0200 Subject: [PATCH 09/11] Docs, queryset.update: `full_result`-arg not clearly described The documentation for the `full_result`-argument to `queryset.update()` can be read as returning the update documents/objects, whereas it's really returning just the full "PyMongo result dictionary". This commit adds some wording and an example dictionary, to make it clear what the behavior is. --- mongoengine/queryset/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 6f9c372c..bf8a5b55 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -486,8 +486,9 @@ class BaseQuerySet(object): ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. - :param full_result: Return the full result rather than just the number - updated. + :param full_result: Return the full result dictionary rather than just the number + updated, e.g. return + `{u'n': 2, u'nModified': 2, u'ok': 1.0, 'updatedExisting': True}`. :param update: Django-style update keyword arguments .. versionadded:: 0.2 From 727778b7305cd01c81e3b198fd20079fcb66b0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malthe=20J=C3=B8rgensen?= Date: Fri, 30 Mar 2018 20:41:39 +0200 Subject: [PATCH 10/11] Docs, queryset.update(): Fix backtick mistake Code should be marked with double backticks --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index bf8a5b55..e5611226 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -488,7 +488,7 @@ class BaseQuerySet(object): will force an fsync on the primary server. :param full_result: Return the full result dictionary rather than just the number updated, e.g. return - `{u'n': 2, u'nModified': 2, u'ok': 1.0, 'updatedExisting': True}`. + ``{'n': 2, 'nModified': 2, 'ok': 1.0, 'updatedExisting': True}``. :param update: Django-style update keyword arguments .. versionadded:: 0.2 From 49bff5d544c30ba0a8433b07cda8f7d4e50c9d73 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 10:47:52 -0400 Subject: [PATCH 11/11] Add documentation for LazyReference and GenericLazyReference fields. --- docs/apireference.rst | 2 ++ docs/guide/defining-documents.rst | 2 ++ mongoengine/fields.py | 17 +++++++++++------ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 625d4a8b..05ba3f73 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -87,7 +87,9 @@ Fields .. autoclass:: mongoengine.fields.DictField .. autoclass:: mongoengine.fields.MapField .. autoclass:: mongoengine.fields.ReferenceField +.. autoclass:: mongoengine.fields.LazyReferenceField .. autoclass:: mongoengine.fields.GenericReferenceField +.. autoclass:: mongoengine.fields.GenericLazyReferenceField .. autoclass:: mongoengine.fields.CachedReferenceField .. autoclass:: mongoengine.fields.BinaryField .. autoclass:: mongoengine.fields.FileField diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 33b5292f..3ced284e 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -80,6 +80,7 @@ are as follows: * :class:`~mongoengine.fields.FloatField` * :class:`~mongoengine.fields.GenericEmbeddedDocumentField` * :class:`~mongoengine.fields.GenericReferenceField` +* :class:`~mongoengine.fields.GenericLazyReferenceField` * :class:`~mongoengine.fields.GeoPointField` * :class:`~mongoengine.fields.ImageField` * :class:`~mongoengine.fields.IntField` @@ -87,6 +88,7 @@ are as follows: * :class:`~mongoengine.fields.MapField` * :class:`~mongoengine.fields.ObjectIdField` * :class:`~mongoengine.fields.ReferenceField` +* :class:`~mongoengine.fields.LazyReferenceField` * :class:`~mongoengine.fields.SequenceField` * :class:`~mongoengine.fields.SortedListField` * :class:`~mongoengine.fields.StringField` diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f169f0f1..a661874a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -2204,8 +2204,11 @@ class MultiPolygonField(GeoJsonBaseField): class LazyReferenceField(BaseField): """A really lazy reference to a document. - Unlike the :class:`~mongoengine.fields.ReferenceField` it must be manually - dereferenced using it ``fetch()`` method. + Unlike the :class:`~mongoengine.fields.ReferenceField` it will + **not** be automatically (lazily) dereferenced on access. + Instead, access will return a :class:`~mongoengine.base.LazyReference` class + instance, allowing access to `pk` or manual dereference by using + ``fetch()`` method. .. versionadded:: 0.15 """ @@ -2331,10 +2334,12 @@ class LazyReferenceField(BaseField): class GenericLazyReferenceField(GenericReferenceField): - """A reference to *any* :class:`~mongoengine.document.Document` subclass - that will be automatically dereferenced on access (lazily). - Unlike the :class:`~mongoengine.fields.GenericReferenceField` it must be - manually dereferenced using it ``fetch()`` method. + """A reference to *any* :class:`~mongoengine.document.Document` subclass. + Unlike the :class:`~mongoengine.fields.GenericReferenceField` it will + **not** be automatically (lazily) dereferenced on access. + Instead, access will return a :class:`~mongoengine.base.LazyReference` class + instance, allowing access to `pk` or manual dereference by using + ``fetch()`` method. .. note :: * Any documents used as a generic reference must be registered in the