From b8d53a6f0d69c81e0566f5cadbe9237e1f530131 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 12:04:14 +0000 Subject: [PATCH] Added json serialisation support - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) --- docs/changelog.rst | 4 +- mongoengine/base/document.py | 10 ++++ mongoengine/queryset/queryset.py | 10 ++++ tests/document/__init__.py | 5 +- tests/document/instance.py | 2 +- tests/document/json_serialisation.py | 81 ++++++++++++++++++++++++++++ tests/test_fields.py | 39 ++++++++------ tests/test_queryset.py | 70 +++++++++++++++++++++++- 8 files changed, 201 insertions(+), 20 deletions(-) create mode 100644 tests/document/json_serialisation.py diff --git a/docs/changelog.rst b/docs/changelog.rst index ca450f11..26108b5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,9 @@ Changelog Changes in 0.8 ============== -- Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) +- Added to_json and from_json to Document (MongoEngine/mongoengine#1) +- Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) +- Updated index creation now tied to Document class (MongoEngine/mongoengine#102) - Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 46f53205..939c9fbc 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -2,6 +2,7 @@ import operator from functools import partial import pymongo +from bson import json_util from bson.dbref import DBRef from mongoengine import signals @@ -253,6 +254,15 @@ class BaseDocument(object): if errors: raise ValidationError('ValidationError', errors=errors) + def to_json(self): + """Converts a document to JSON""" + return json_util.dumps(self.to_mongo()) + + @classmethod + def from_json(cls, json_data): + """Converts json data to an unsaved document instance""" + return cls._from_son(json_util.loads(json_data)) + def __expand_dynamic_values(self, name, value): """expand any dynamic values to their correct types / values""" if not isinstance(value, (dict, list, tuple)): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 058bdd86..3c44f012 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -6,6 +6,7 @@ import re import warnings from bson.code import Code +from bson import json_util import pymongo from pymongo.common import validate_read_preference @@ -1216,6 +1217,15 @@ class QuerySet(object): max_depth += 1 return self._dereference(self, max_depth=max_depth) + def to_json(self): + """Converts a queryset to JSON""" + return json_util.dumps(self._collection_obj.find(self._query)) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + @property def _dereference(self): if not self.__dereference: diff --git a/tests/document/__init__.py b/tests/document/__init__.py index 1ef25201..7774ee19 100644 --- a/tests/document/__init__.py +++ b/tests/document/__init__.py @@ -1,4 +1,6 @@ -# TODO EXPLICT IMPORTS +import sys +sys.path[0:0] = [""] +import unittest from class_methods import * from delta import * @@ -6,6 +8,7 @@ from dynamic import * from indexes import * from inheritance import * from instance import * +from json_serialisation import * if __name__ == '__main__': unittest.main() diff --git a/tests/document/instance.py b/tests/document/instance.py index 2e07eb26..2118575e 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -346,7 +346,7 @@ class InstanceTest(unittest.TestCase): meta = {'shard_key': ('superphylum',)} Animal.drop_collection() - doc = Animal(superphylum = 'Deuterostomia') + doc = Animal(superphylum='Deuterostomia') doc.save() doc.reload() Animal.drop_collection() diff --git a/tests/document/json_serialisation.py b/tests/document/json_serialisation.py new file mode 100644 index 00000000..dbc09d83 --- /dev/null +++ b/tests/document/json_serialisation.py @@ -0,0 +1,81 @@ +import sys +sys.path[0:0] = [""] + +import unittest +import uuid + +from nose.plugins.skip import SkipTest +from datetime import datetime +from bson import ObjectId + +import pymongo + +from mongoengine import * + +__all__ = ("TestJson",) + + +class TestJson(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + doc = Doc(string="Hi", embedded_field=Embedded(string="Hi")) + + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + def test_json_complex(self): + + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + doc = Doc() + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_fields.py b/tests/test_fields.py index f1a36ed7..69cce871 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -606,7 +606,8 @@ class FieldTest(unittest.TestCase): name = StringField() class CategoryList(Document): - categories = SortedListField(EmbeddedDocumentField(Category), ordering='count', reverse=True) + categories = SortedListField(EmbeddedDocumentField(Category), + ordering='count', reverse=True) name = StringField() catlist = CategoryList(name="Top categories") @@ -1616,8 +1617,9 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) Shirt.drop_collection() @@ -1633,12 +1635,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) - style = StringField(max_length=3, choices=(('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + style = StringField(max_length=3, choices=( + ('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') Shirt.drop_collection() @@ -1665,7 +1670,8 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) Shirt.drop_collection() @@ -1681,11 +1687,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_simple_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) - style = StringField(max_length=3, choices=('Small', 'Baggy', 'wide'), default='Small') + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) + style = StringField(max_length=3, + choices=('Small', 'Baggy', 'wide'), + default='Small') Shirt.drop_collection() @@ -1736,7 +1746,7 @@ class FieldTest(unittest.TestCase): self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS + result.the_file.delete() # Remove file from GridFS PutFile.objects.delete() # Ensure file-like objects are stored @@ -1801,7 +1811,6 @@ class FieldTest(unittest.TestCase): the_file = FileField() DemoFile.objects.create() - def test_file_field_no_default(self): class GridDocument(Document): @@ -1817,7 +1826,6 @@ class FieldTest(unittest.TestCase): doc_a = GridDocument() doc_a.save() - doc_b = GridDocument.objects.with_id(doc_a.id) doc_b.the_file.replace(f, filename='doc_b') doc_b.save() @@ -1859,7 +1867,7 @@ class FieldTest(unittest.TestCase): # Second instance test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None + data = test_file_dupe.the_file.read() # Should be None self.assertTrue(test_file.name != test_file_dupe.name) self.assertTrue(test_file.the_file.read() != data) @@ -2328,7 +2336,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(error_dict['comments'][1]['content'], u'Field is required') - post.comments[1].content = 'here we go' post.validate() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 09b6b3ff..9dfe9a27 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,7 +1,10 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] + import unittest +import uuid +from nose.plugins.skip import SkipTest from datetime import datetime, timedelta @@ -74,7 +77,6 @@ class QuerySetTest(unittest.TestCase): def test_generic_reference(): list(BlogPost.objects(author2__name="test")) - def test_find(self): """Ensure that a query returns a valid set of results. """ @@ -3672,6 +3674,72 @@ class QueryFieldListTest(unittest.TestCase): self.assertRaises(ConfigurationError, Bar.objects, read_preference='Primary') + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + Doc(string="Hi", embedded_field=Embedded(string="Hi")).save() + Doc(string="Bye", embedded_field=Embedded(string="Bye")).save() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) + + def test_json_complex(self): + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + Simple.drop_collection() + Doc.drop_collection() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) if __name__ == '__main__':