From 0512dd4c25e0f8395c317725fa149a64025faf3e Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 03:53:39 +0800 Subject: [PATCH 1/9] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22aa..4fff317a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..7d9c47f4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e264..e22e6ddd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 6bfd6c322b166d55eb151de33e9eaee7ca214b8a Mon Sep 17 00:00:00 2001 From: martin Date: Fri, 18 Jun 2010 10:41:23 +0800 Subject: [PATCH 2/9] Fixed bug with GeoLocationField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f4..5893049d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -388,7 +388,7 @@ class GeoLocationField(DictField): return {'x': value[0], 'y': value[1]} def to_python(self, value): - return value.keys() + return (value['x'], value['y']) class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on From 47bfeec115a297c1a859543e842797751ab9c14a Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 15:27:21 +0800 Subject: [PATCH 3/9] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..7b8dcd5b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..b6d5a63b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5893049d..ebefbb75 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6ddd..8dddcb3e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 9596a25bb92212806e98fd77b6ced1b79c2ed1bf Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 00:56:16 +0200 Subject: [PATCH 4/9] Fixed documentation bug. --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..1fd2ed57 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -174,7 +174,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset(published=True).filter(published=True) + return queryset.filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From f9057e1a288dea4b99ac1936a75696071476870d Mon Sep 17 00:00:00 2001 From: martin Date: Thu, 24 Jun 2010 00:56:51 +0800 Subject: [PATCH 5/9] Fixed bug in FileField, proxy was not getting the grid_id set --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ebefbb75..d2b41ab9 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -542,6 +542,7 @@ class GridFSProxy(object): return self def get(self, id=None): + if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: return None # File has been deleted @@ -611,7 +612,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() + return self.gridfs.get(id=value) def validate(self, value): assert isinstance(value, GridFSProxy) From ec519f20fa8e1c769ffb6ae766dd00bb36f9eabc Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 01:32:28 +0200 Subject: [PATCH 6/9] Makes the tests compatible to pymongo 1.7+. Not backwards compatible! --- tests/document.py | 14 ++++++++------ tests/fields.py | 2 +- tests/queryset.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/document.py b/tests/document.py index 8bc907c5..1160b353 100644 --- a/tests/document.py +++ b/tests/document.py @@ -264,11 +264,12 @@ class DocumentTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info.values()) + self.assertTrue([('tags', 1)] in info) class ExtendedBlogPost(BlogPost): title = StringField() @@ -278,10 +279,11 @@ class DocumentTest(unittest.TestCase): list(ExtendedBlogPost.objects) info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) - self.assertTrue([('_types', 1), ('title', 1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) + self.assertTrue([('_types', 1), ('title', 1)] in info) BlogPost.drop_collection() diff --git a/tests/fields.py b/tests/fields.py index d95f4d3f..136437b8 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -689,7 +689,7 @@ class FieldTest(unittest.TestCase): info = Event.objects._collection.index_information() self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) Event.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index 4187d550..0424d323 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1087,8 +1087,9 @@ class QuerySetTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() - self.assertTrue([('_types', 1)] in info.values()) - self.assertTrue([('_types', 1), ('date', -1)] in info.values()) + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1)] in info) + self.assertTrue([('_types', 1), ('date', -1)] in info) BlogPost.drop_collection() From 6093e88eebf932d04741364f53de77878198c1f1 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Fri, 16 Jul 2010 00:20:29 +0800 Subject: [PATCH 7/9] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 24c5b569..8e8a97c8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,6 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 03c0fd9ada5eb1c90a9213fe4579e64d9a6a111c Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 19:01:53 +0200 Subject: [PATCH 8/9] Make default value of DictField an empty dict instead of None. --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8e8a97c8..759697ac 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault("default", []) + kwargs.setdefault('default', []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,6 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) + kwargs.setdefault('default', {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From aa00feb6a55eba7f720969828242436103817e64 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 20 Jul 2010 22:46:00 +0200 Subject: [PATCH 9/9] FileField's values are now optional. When no value is applied, no File object is created and referenced. --- mongoengine/base.py | 4 ++-- mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 806d83bb..086c7874 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,8 +339,8 @@ class BaseDocument(object): try: field._validate(value) except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field of type "' + - field.__class__.__name__ + '"') + raise ValidationError('Invalid value for field of type "%s": %s' + % (field.__class__.__name__, value)) elif field.required: raise ValidationError('Field "%s" is required' % field.name) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 759697ac..f84f751b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -591,15 +591,20 @@ class FileField(BaseField): def to_mongo(self, value): # Store the GridFS file id in MongoDB - return self.gridfs.grid_id + if self.gridfs.grid_id is not None: + return self.gridfs.grid_id + return None def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get(id=value) + if self.gridfs.grid_id is not None: + return self.gridfs.get(id=value) + return None def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + if value.grid_id is not None: + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) class GeoPointField(BaseField): """A list storing a latitude and longitude.