From c9ed9306068db2970f42a6ececbfd26dd783e848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 25 Jun 2019 22:20:47 +0200 Subject: [PATCH 1/4] Add a documentation page for validation --- docs/guide/index.rst | 1 + docs/guide/querying.rst | 3 +- docs/guide/validation.rst | 116 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 docs/guide/validation.rst diff --git a/docs/guide/index.rst b/docs/guide/index.rst index 46eb7af2..c94a4eab 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -10,6 +10,7 @@ User Guide defining-documents document-instances querying + validation gridfs signals text-indexes diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 151855a6..6937cf68 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -566,7 +566,8 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: ['database', 'mongodb'] From MongoDB version 2.6, push operator supports $position value which allows -to push values with index. +to push values with index:: + >>> post = BlogPost(title="Test", tags=["mongo"]) >>> post.save() >>> post.update(push__tags__0=["database", "code"]) diff --git a/docs/guide/validation.rst b/docs/guide/validation.rst new file mode 100644 index 00000000..793a797a --- /dev/null +++ b/docs/guide/validation.rst @@ -0,0 +1,116 @@ +==================== +Document Validation +==================== + +By design, mongoengine strictly validates the documents right before they are inserted in MongoDB +and make sure they are consistent with the fields defined in your models. + +Mongoengine will not validate a document when an object is loaded from the DB into an instance +of your model but this operation will fail under some circumstances (e.g: if there is a field in +the document fetched from the database that is not defined in your model) + + +Builtin validation +================= + +Mongoengine provides different fields that encapsulate the corresponding validation +out of the box. Validation runs when calling `.validate()` or `.save()` + +.. code-block:: python + + from mongoengine import Document, EmailField + + class User(Document): + email = EmailField() + age = IntField(min_value=0, max_value=99) + + user = User(email='invalid@', age=24) + user.validate() # raises ValidationError (Invalid email address: ['email']) + user.save() # raises ValidationError (Invalid email address: ['email']) + + user2 = User(email='john.doe@garbage.com', age=1000) + user2.save() # raises ValidationError (Integer value is too large: ['age']) + +Custom validation +================= + +The following feature can be used to customize the validation: + +* Field `validation` parameter + +.. code-block:: python + + def not_john_doe(name): + if name == 'John Doe': + raise ValidationError("John Doe is not a valid name") + + class Person(Document): + full_name = StringField(validation=not_john_doe) + + Person(full_name='Billy Doe').save() + Person(full_name='John Doe').save() # raises ValidationError (John Doe is not a valid name) + + +* Document `clean` method + +Although not its primary use case, `clean` may be use to do validation that involves multiple fields. +Note that `clean` runs before the validation when you save a Document. + +.. code-block:: python + + class Person(Document): + first_name = StringField() + last_name = StringField() + + def clean(self): + if self.first_name == 'John' and self.last_name == 'Doe': + raise ValidationError('John Doe is not a valid name') + + Person(first_name='Billy', last_name='Doe').save() + Person(first_name='John', last_name='Doe').save() # raises ValidationError (John Doe is not a valid name) + + + +* Adding custom Field classes + +We recommend as much as possible to use the standard field but it is also possible +to subclass a Field and encapsulate some validation by overriding the `validate` method + +.. code-block:: python + + class AgeField(IntField): + + def validate(self, value): + super(AgeField, self).validate(value) # let IntField.validate run first + if value == 60: + self.error('60 is not allowed') + + class Person(Document): + age = AgeField(min_value=0, max_value=99) + + Person(age=20).save() # passes + Person(age=1000).save() # raises ValidationError (Integer value is too large: ['age']) + Person(age=60).save() # raises ValidationError (Person:None) (60 is not allowed: ['age']) + + +.. note:: + + When overriding `validate`, use `self.error("your-custom-error")` instead of raising ValidationError explicitly, + it will provide a better context with the error message + +Disabling validation +==================== + +We do not recommend to do this but if for some reason you need to disable the validation of a document +when you call `.save()`, you can use `.save(validate=False)`. + +.. code-block:: python + + class Person(Document): + age = IntField() + + Person(age='garbage').save() # raises ValidationError (garbage could not be converted to int: ['age']) + + Person(age='garbage').save(validate=False) + person = Person.objects.first() + assert person.age == 'garbage' From d37a30e0838e10e95457c63f8842a22362c04bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 30 Jun 2019 20:46:40 +0200 Subject: [PATCH 2/4] improve doc (based on review) --- docs/guide/validation.rst | 10 +++++----- mongoengine/base/datastructures.py | 2 +- mongoengine/fields.py | 2 +- mongoengine/mongodb_support.py | 2 +- tests/utils.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/guide/validation.rst b/docs/guide/validation.rst index 793a797a..5f0346bd 100644 --- a/docs/guide/validation.rst +++ b/docs/guide/validation.rst @@ -2,15 +2,15 @@ Document Validation ==================== -By design, mongoengine strictly validates the documents right before they are inserted in MongoDB +By design, MongoEngine strictly validates the documents right before they are inserted in MongoDB and make sure they are consistent with the fields defined in your models. Mongoengine will not validate a document when an object is loaded from the DB into an instance -of your model but this operation will fail under some circumstances (e.g: if there is a field in -the document fetched from the database that is not defined in your model) +of your model but this operation will fail under some circumstances (e.g. if there is a field in +the document fetched from the database that is not defined in your model). -Builtin validation +Built-in validation ================= Mongoengine provides different fields that encapsulate the corresponding validation @@ -73,7 +73,7 @@ Note that `clean` runs before the validation when you save a Document. * Adding custom Field classes -We recommend as much as possible to use the standard field but it is also possible +We recommend as much as possible to use fields provided by MongoEngine. However, it is also possible to subclass a Field and encapsulate some validation by overriding the `validate` method .. code-block:: python diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index fafc08b7..b693b914 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -217,7 +217,7 @@ class EmbeddedDocumentList(BaseList): Filters the list by only including embedded documents with the given keyword arguments. - This method only supports simple comparison (e.g: .filter(name='John Doe')) + This method only supports simple comparison (e.g. .filter(name='John Doe')) and does not support operators like __gte, __lte, __icontains like queryset.filter does :param kwargs: The keyword arguments corresponding to the fields to diff --git a/mongoengine/fields.py b/mongoengine/fields.py index aa5aa805..591cf01a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -385,7 +385,7 @@ class DecimalField(BaseField): :param max_value: Validation rule for the maximum acceptable value. :param force_string: Store the value as a string (instead of a float). Be aware that this affects query sorting and operation like lte, gte (as string comparison is applied) - and some query operator won't work (e.g: inc, dec) + and some query operator won't work (e.g. inc, dec) :param precision: Number of decimal places to store. :param rounding: The rounding rule from the python decimal library: diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index b20ebc1e..423cd92b 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -15,5 +15,5 @@ def get_mongodb_version(): :return: tuple(int, int) """ - version_list = get_connection().server_info()['versionArray'][:2] # e.g: (3, 2) + version_list = get_connection().server_info()['versionArray'][:2] # e.g. (3, 2) return tuple(version_list) diff --git a/tests/utils.py b/tests/utils.py index 27d5ada7..be7a8095 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,7 @@ import unittest from nose.plugins.skip import SkipTest from mongoengine import connect -from mongoengine.connection import get_db, disconnect_all +from mongoengine.connection import disconnect_all, get_db from mongoengine.mongodb_support import get_mongodb_version @@ -51,7 +51,7 @@ def _decorated_with_ver_requirement(func, mongo_version_req, oper): ran against MongoDB < v3.6. :param mongo_version_req: The mongodb version requirement (tuple(int, int)) - :param oper: The operator to apply (e.g: operator.ge) + :param oper: The operator to apply (e.g. operator.ge) """ def _inner(*args, **kwargs): mongodb_v = get_mongodb_version() From 3fca3739de906772cb2090ec0c465576fe550953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 18 Oct 2020 21:11:16 +0200 Subject: [PATCH 3/4] rework validation documentation based on review --- docs/guide/defining-documents.rst | 13 -------- docs/guide/document-instances.rst | 29 ------------------ docs/guide/validation.rst | 49 ++++++++++++++++++------------- mongoengine/base/document.py | 3 +- 4 files changed, 30 insertions(+), 64 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 6dc35c30..f5c70728 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -426,19 +426,6 @@ either a single field name, or a list or tuple of field names:: first_name = StringField() last_name = StringField(unique_with='first_name') -Skipping Document validation on save ------------------------------------- -You can also skip the whole document validation process by setting -``validate=False`` when calling the :meth:`~mongoengine.document.Document.save` -method:: - - class Recipient(Document): - name = StringField() - email = EmailField() - - recipient = Recipient(name='admin', email='root@localhost') - recipient.save() # will raise a ValidationError while - recipient.save(validate=False) # won't Document collections ==================== diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 64f17c08..5dacc0dd 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -41,35 +41,6 @@ already exist, then any changes will be updated atomically. For example:: .. seealso:: :ref:`guide-atomic-updates` -Pre save data validation and cleaning -------------------------------------- -MongoEngine allows you to create custom cleaning rules for your documents when -calling :meth:`~mongoengine.Document.save`. By providing a custom -:meth:`~mongoengine.Document.clean` method you can do any pre validation / data -cleaning. - -This might be useful if you want to ensure a default value based on other -document values for example:: - - class Essay(Document): - status = StringField(choices=('Published', 'Draft'), required=True) - pub_date = DateTimeField() - - def clean(self): - """Ensures that only published essays have a `pub_date` and - automatically sets `pub_date` if essay is published and `pub_date` - is not set""" - if self.status == 'Draft' and self.pub_date is not None: - msg = 'Draft entries should not have a publication date.' - raise ValidationError(msg) - # Set the pub_date for published items if not set. - if self.status == 'Published' and self.pub_date is None: - self.pub_date = datetime.now() - -.. note:: - Cleaning is only called if validation is turned on and when calling - :meth:`~mongoengine.Document.save`. - Cascading Saves --------------- If your document contains :class:`~mongoengine.fields.ReferenceField` or diff --git a/docs/guide/validation.rst b/docs/guide/validation.rst index 5f0346bd..e5a70d34 100644 --- a/docs/guide/validation.rst +++ b/docs/guide/validation.rst @@ -3,10 +3,11 @@ Document Validation ==================== By design, MongoEngine strictly validates the documents right before they are inserted in MongoDB -and make sure they are consistent with the fields defined in your models. +and makes sure they are consistent with the fields defined in your models. -Mongoengine will not validate a document when an object is loaded from the DB into an instance -of your model but this operation will fail under some circumstances (e.g. if there is a field in +MongoEngine makes the assumption that the documents that exists in the DB are compliant with the schema. +This means that Mongoengine will not validate a document when an object is loaded from the DB into an instance +of your model but this operation may fail under some circumstances (e.g. if there is a field in the document fetched from the database that is not defined in your model). @@ -53,23 +54,28 @@ The following feature can be used to customize the validation: * Document `clean` method -Although not its primary use case, `clean` may be use to do validation that involves multiple fields. -Note that `clean` runs before the validation when you save a Document. +This method is called as part of :meth:`~mongoengine.document.Document.save` and should be used to provide +custom model validation and/or to modify some of the field values prior to validation. +For instance, you could use it to automatically provide a value for a field, or to do validation +that requires access to more than a single field. .. code-block:: python - class Person(Document): - first_name = StringField() - last_name = StringField() + class Essay(Document): + status = StringField(choices=('Published', 'Draft'), required=True) + pub_date = DateTimeField() def clean(self): - if self.first_name == 'John' and self.last_name == 'Doe': - raise ValidationError('John Doe is not a valid name') - - Person(first_name='Billy', last_name='Doe').save() - Person(first_name='John', last_name='Doe').save() # raises ValidationError (John Doe is not a valid name) - + # Validate that only published essays have a `pub_date` + if self.status == 'Draft' and self.pub_date is not None: + raise ValidationError('Draft entries should not have a publication date.') + # Set the pub_date for published items if not set. + if self.status == 'Published' and self.pub_date is None: + self.pub_date = datetime.now() +.. note:: + Cleaning is only called if validation is turned on and when calling + :meth:`~mongoengine.Document.save`. * Adding custom Field classes @@ -98,19 +104,20 @@ to subclass a Field and encapsulate some validation by overriding the `validate` When overriding `validate`, use `self.error("your-custom-error")` instead of raising ValidationError explicitly, it will provide a better context with the error message -Disabling validation +Skipping validation ==================== -We do not recommend to do this but if for some reason you need to disable the validation of a document -when you call `.save()`, you can use `.save(validate=False)`. +Although discouraged as it allows to violate fields constraints, if for some reason you need to disable +the validation and cleaning of a document when you call `.save()`, you can use `.save(validate=False)`. .. code-block:: python class Person(Document): - age = IntField() + age = IntField(max_value=100) - Person(age='garbage').save() # raises ValidationError (garbage could not be converted to int: ['age']) + Person(age=1000).save() # raises ValidationError (Integer value is too large) - Person(age='garbage').save(validate=False) + Person(age=1000).save(validate=False) person = Person.objects.first() - assert person.age == 'garbage' + assert person.age == 1000 + diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index e697fe40..0bab2886 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -314,7 +314,8 @@ class BaseDocument: def clean(self): """ - Hook for doing document level data cleaning before validation is run. + Hook for doing document level data cleaning (usually validation or assignment) + before validation is run. Any ValidationError raised by this method will not be associated with a particular field; it will have a special-case association with the From 015a36c85fa4d7457a97972428474278e68dd956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 19 Oct 2020 23:45:13 +0200 Subject: [PATCH 4/4] minor styling fix in .rst --- docs/guide/validation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/validation.rst b/docs/guide/validation.rst index e5a70d34..971f6eb5 100644 --- a/docs/guide/validation.rst +++ b/docs/guide/validation.rst @@ -12,7 +12,7 @@ the document fetched from the database that is not defined in your model). Built-in validation -================= +=================== Mongoengine provides different fields that encapsulate the corresponding validation out of the box. Validation runs when calling `.validate()` or `.save()` @@ -108,7 +108,7 @@ Skipping validation ==================== Although discouraged as it allows to violate fields constraints, if for some reason you need to disable -the validation and cleaning of a document when you call `.save()`, you can use `.save(validate=False)`. +the validation and cleaning of a document when you call :meth:`~mongoengine.document.Document.save`, you can use `.save(validate=False)`. .. code-block:: python