Undefined data should not override instance methods (#49)

docs/changelog.rst

@@ -40,6 +40,7 @@ Changes in 0.8.X
 - Added support for compound primary keys (#149) (#121)
 - Fixed overriding objects with custom manager (#58)
 - Added no_dereference method for querysets (#82) (#61)
+- Undefined data should not override instance methods (#49)
 
 Changes in 0.7.9
 ================

docs/guide/defining-documents.rst

@@ -600,8 +600,7 @@ Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and
 easily get working with existing data.  Just define the document to match
-the expected schema in your database.  If you have wildly varying schemas then
+the expected schema in your database ::
-a :class:`~mongoengine.DynamicDocument` might be more appropriate. ::
 
     # Will work with data in an existing collection named 'cmsPage'
     class Page(Document):
@@ -609,3 +608,10 @@ a :class:`~mongoengine.DynamicDocument` might be more appropriate. ::
         meta = {
             'collection': 'cmsPage'
         }
+
+If you have wildly varying schemas then using a
+:class:`~mongoengine.DynamicDocument` might be more appropriate, instead of
+defining all possible field types.
+
+If you use :class:`~mongoengine.Document` and the database contains data that
+isn't defined then that data will be stored in the `document._data` dictionary.

mongoengine/base/document.py

@@ -60,13 +60,17 @@ class BaseDocument(object):
         else:
             FileField = _import_class('FileField')
             for key, value in values.iteritems():
+                if key == '__auto_convert':
+                    continue
                 key = self._reverse_db_field_map.get(key, key)
-                if (value is not None and __auto_convert and
+                if key in self._fields or key in ('id', 'pk', '_cls'):
-                    key in self._fields):
+                    if __auto_convert and value is not None:
-                    field = self._fields.get(key)
+                        field = self._fields.get(key)
-                    if not isinstance(field, FileField):
+                        if field and not isinstance(field, FileField):
-                        value = field.to_python(value)
+                            value = field.to_python(value)
-                setattr(self, key, value)
+                    setattr(self, key, value)
+                else:
+                    self._data[key] = value
 
         # Set any get_fieldname_display methods
         self.__set_field_display()

tests/document/__init__.py

@@ -9,6 +9,7 @@ from indexes import *
 from inheritance import *
 from instance import *
 from json_serialisation import *
+from validation import *
 
 if __name__ == '__main__':
     unittest.main()

tests/document/instance.py

@@ -23,7 +23,7 @@ from mongoengine.context_managers import switch_db
 TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__),
                                '../fields/mongoengine.png')
 
-__all__ = ("InstanceTest", "ValidatorErrorTest")
+__all__ = ("InstanceTest",)
 
 
 class InstanceTest(unittest.TestCase):
@@ -1977,192 +1977,29 @@ class InstanceTest(unittest.TestCase):
         group = Group.objects.first()
         self.assertEqual("hello - default", group.name)
 
-
+    def test_no_overwritting_no_data_loss(self):
-class ValidatorErrorTest(unittest.TestCase):
-
-    def test_to_dict(self):
-        """Ensure a ValidationError handles error to_dict correctly.
-        """
-        error = ValidationError('root')
-        self.assertEqual(error.to_dict(), {})
-
-        # 1st level error schema
-        error.errors = {'1st': ValidationError('bad 1st'), }
-        self.assertTrue('1st' in error.to_dict())
-        self.assertEqual(error.to_dict()['1st'], 'bad 1st')
-
-        # 2nd level error schema
-        error.errors = {'1st': ValidationError('bad 1st', errors={
-            '2nd': ValidationError('bad 2nd'),
-        })}
-        self.assertTrue('1st' in error.to_dict())
-        self.assertTrue(isinstance(error.to_dict()['1st'], dict))
-        self.assertTrue('2nd' in error.to_dict()['1st'])
-        self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd')
-
-        # moar levels
-        error.errors = {'1st': ValidationError('bad 1st', errors={
-            '2nd': ValidationError('bad 2nd', errors={
-                '3rd': ValidationError('bad 3rd', errors={
-                    '4th': ValidationError('Inception'),
-                }),
-            }),
-        })}
-        self.assertTrue('1st' in error.to_dict())
-        self.assertTrue('2nd' in error.to_dict()['1st'])
-        self.assertTrue('3rd' in error.to_dict()['1st']['2nd'])
-        self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd'])
-        self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'],
-                         'Inception')
-
-        self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])")
-
-    def test_model_validation(self):
 
         class User(Document):
             username = StringField(primary_key=True)
-            name = StringField(required=True)
-
-        try:
-            User().validate()
-        except ValidationError, e:
-            self.assertEqual(e.to_dict(), {
-                'username': 'Field is required',
-                'name': 'Field is required'})
-
-    def test_spaces_in_keys(self):
-
-        class Embedded(DynamicEmbeddedDocument):
-            pass
-
-        class Doc(DynamicDocument):
-            pass
-
-        Doc.drop_collection()
-        doc = Doc()
-        setattr(doc, 'hello world', 1)
-        doc.save()
-
-        one = Doc.objects.filter(**{'hello world': 1}).count()
-        self.assertEqual(1, one)
-
-    def test_fields_rewrite(self):
-        class BasePerson(Document):
-            name = StringField()
-            age = IntField()
-            meta = {'abstract': True}
-
-        class Person(BasePerson):
-            name = StringField(required=True)
-
-        p = Person(age=15)
-        self.assertRaises(ValidationError, p.validate)
-
-    def test_cascaded_save_wrong_reference(self):
-
-        class ADocument(Document):
-            val = IntField()
-
-        class BDocument(Document):
-            a = ReferenceField(ADocument)
-
-        ADocument.drop_collection()
-        BDocument.drop_collection()
-
-        a = ADocument()
-        a.val = 15
-        a.save()
-
-        b = BDocument()
-        b.a = a
-        b.save()
-
-        a.delete()
-
-        b = BDocument.objects.first()
-        b.save(cascade=True)
-
-    def test_shard_key(self):
-        class LogEntry(Document):
-            machine = StringField()
-            log = StringField()
-
-            meta = {
-                'shard_key': ('machine',)
-            }
-
-        LogEntry.drop_collection()
-
-        log = LogEntry()
-        log.machine = "Localhost"
-        log.save()
-
-        log.log = "Saving"
-        log.save()
-
-        def change_shard_key():
-            log.machine = "127.0.0.1"
-
-        self.assertRaises(OperationError, change_shard_key)
-
-    def test_shard_key_primary(self):
-        class LogEntry(Document):
-            machine = StringField(primary_key=True)
-            log = StringField()
-
-            meta = {
-                'shard_key': ('machine',)
-            }
-
-        LogEntry.drop_collection()
-
-        log = LogEntry()
-        log.machine = "Localhost"
-        log.save()
-
-        log.log = "Saving"
-        log.save()
-
-        def change_shard_key():
-            log.machine = "127.0.0.1"
-
-        self.assertRaises(OperationError, change_shard_key)
-
-    def test_kwargs_simple(self):
-
-        class Embedded(EmbeddedDocument):
             name = StringField()
 
-        class Doc(Document):
+            @property
-            doc_name = StringField()
+            def foo(self):
-            doc = EmbeddedDocumentField(Embedded)
+                return True
 
-        classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc"))
+        User.drop_collection()
-        dict_doc = Doc(**{"doc_name": "my doc",
-                          "doc": {"name": "embedded doc"}})
 
-        self.assertEqual(classic_doc, dict_doc)
+        user = User(username="Ross", foo="bar")
-        self.assertEqual(classic_doc._data, dict_doc._data)
+        self.assertTrue(user.foo)
 
-    def test_kwargs_complex(self):
+        User._get_collection().save({"_id": "Ross", "foo": "Bar",
-
+                                     "data": [1, 2, 3]})
-        class Embedded(EmbeddedDocument):
-            name = StringField()
-
-        class Doc(Document):
-            doc_name = StringField()
-            docs = ListField(EmbeddedDocumentField(Embedded))
-
-        classic_doc = Doc(doc_name="my doc", docs=[
-                            Embedded(name="embedded doc1"),
-                            Embedded(name="embedded doc2")])
-        dict_doc = Doc(**{"doc_name": "my doc",
-                          "docs": [{"name": "embedded doc1"},
-                                   {"name": "embedded doc2"}]})
-
-        self.assertEqual(classic_doc, dict_doc)
-        self.assertEqual(classic_doc._data, dict_doc._data)
 
+        user = User.objects.first()
+        self.assertEqual("Ross", user.username)
+        self.assertEqual(True, user.foo)
+        self.assertEqual("Bar", user._data["foo"])
+        self.assertEqual([1, 2, 3], user._data["data"])
 
 if __name__ == '__main__':
     unittest.main()

tests/document/validation.py

@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+import sys
+sys.path[0:0] = [""]
+
+import unittest
+
+from mongoengine import *
+
+__all__ = ("ValidatorErrorTest",)
+
+
+class ValidatorErrorTest(unittest.TestCase):
+
+    def test_to_dict(self):
+        """Ensure a ValidationError handles error to_dict correctly.
+        """
+        error = ValidationError('root')
+        self.assertEqual(error.to_dict(), {})
+
+        # 1st level error schema
+        error.errors = {'1st': ValidationError('bad 1st'), }
+        self.assertTrue('1st' in error.to_dict())
+        self.assertEqual(error.to_dict()['1st'], 'bad 1st')
+
+        # 2nd level error schema
+        error.errors = {'1st': ValidationError('bad 1st', errors={
+            '2nd': ValidationError('bad 2nd'),
+        })}
+        self.assertTrue('1st' in error.to_dict())
+        self.assertTrue(isinstance(error.to_dict()['1st'], dict))
+        self.assertTrue('2nd' in error.to_dict()['1st'])
+        self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd')
+
+        # moar levels
+        error.errors = {'1st': ValidationError('bad 1st', errors={
+            '2nd': ValidationError('bad 2nd', errors={
+                '3rd': ValidationError('bad 3rd', errors={
+                    '4th': ValidationError('Inception'),
+                }),
+            }),
+        })}
+        self.assertTrue('1st' in error.to_dict())
+        self.assertTrue('2nd' in error.to_dict()['1st'])
+        self.assertTrue('3rd' in error.to_dict()['1st']['2nd'])
+        self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd'])
+        self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'],
+                         'Inception')
+
+        self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])")
+
+    def test_model_validation(self):
+
+        class User(Document):
+            username = StringField(primary_key=True)
+            name = StringField(required=True)
+
+        try:
+            User().validate()
+        except ValidationError, e:
+            self.assertEqual(e.to_dict(), {
+                'username': 'Field is required',
+                'name': 'Field is required'})
+
+    def test_spaces_in_keys(self):
+
+        class Embedded(DynamicEmbeddedDocument):
+            pass
+
+        class Doc(DynamicDocument):
+            pass
+
+        Doc.drop_collection()
+        doc = Doc()
+        setattr(doc, 'hello world', 1)
+        doc.save()
+
+        one = Doc.objects.filter(**{'hello world': 1}).count()
+        self.assertEqual(1, one)
+
+    def test_fields_rewrite(self):
+        class BasePerson(Document):
+            name = StringField()
+            age = IntField()
+            meta = {'abstract': True}
+
+        class Person(BasePerson):
+            name = StringField(required=True)
+
+        p = Person(age=15)
+        self.assertRaises(ValidationError, p.validate)
+
+    def test_cascaded_save_wrong_reference(self):
+
+        class ADocument(Document):
+            val = IntField()
+
+        class BDocument(Document):
+            a = ReferenceField(ADocument)
+
+        ADocument.drop_collection()
+        BDocument.drop_collection()
+
+        a = ADocument()
+        a.val = 15
+        a.save()
+
+        b = BDocument()
+        b.a = a
+        b.save()
+
+        a.delete()
+
+        b = BDocument.objects.first()
+        b.save(cascade=True)
+
+    def test_shard_key(self):
+        class LogEntry(Document):
+            machine = StringField()
+            log = StringField()
+
+            meta = {
+                'shard_key': ('machine',)
+            }
+
+        LogEntry.drop_collection()
+
+        log = LogEntry()
+        log.machine = "Localhost"
+        log.save()
+
+        log.log = "Saving"
+        log.save()
+
+        def change_shard_key():
+            log.machine = "127.0.0.1"
+
+        self.assertRaises(OperationError, change_shard_key)
+
+    def test_shard_key_primary(self):
+        class LogEntry(Document):
+            machine = StringField(primary_key=True)
+            log = StringField()
+
+            meta = {
+                'shard_key': ('machine',)
+            }
+
+        LogEntry.drop_collection()
+
+        log = LogEntry()
+        log.machine = "Localhost"
+        log.save()
+
+        log.log = "Saving"
+        log.save()
+
+        def change_shard_key():
+            log.machine = "127.0.0.1"
+
+        self.assertRaises(OperationError, change_shard_key)
+
+    def test_kwargs_simple(self):
+
+        class Embedded(EmbeddedDocument):
+            name = StringField()
+
+        class Doc(Document):
+            doc_name = StringField()
+            doc = EmbeddedDocumentField(Embedded)
+
+        classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc"))
+        dict_doc = Doc(**{"doc_name": "my doc",
+                          "doc": {"name": "embedded doc"}})
+
+        self.assertEqual(classic_doc, dict_doc)
+        self.assertEqual(classic_doc._data, dict_doc._data)
+
+    def test_kwargs_complex(self):
+
+        class Embedded(EmbeddedDocument):
+            name = StringField()
+
+        class Doc(Document):
+            doc_name = StringField()
+            docs = ListField(EmbeddedDocumentField(Embedded))
+
+        classic_doc = Doc(doc_name="my doc", docs=[
+                            Embedded(name="embedded doc1"),
+                            Embedded(name="embedded doc2")])
+        dict_doc = Doc(**{"doc_name": "my doc",
+                          "docs": [{"name": "embedded doc1"},
+                                   {"name": "embedded doc2"}]})
+
+        self.assertEqual(classic_doc, dict_doc)
+        self.assertEqual(classic_doc._data, dict_doc._data)