diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dd5b00d..29f49cf1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in v0.4 =============== +- Added ``GridFSStorage`` Django storage backend +- Added ``FileField`` for GridFS support - Added ``SortedListField`` - Added ``EmailField`` - Added ``GeoPointField`` diff --git a/docs/django.rst b/docs/django.rst index 92a8a52b..2cce3f02 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -19,7 +19,7 @@ MongoDB but still use many of the Django authentication infrastucture (such as the :func:`login_required` decorator and the :func:`authenticate` function). To enable the MongoEngine auth backend, add the following to you **settings.py** file:: - + AUTHENTICATION_BACKENDS = ( 'mongoengine.django.auth.MongoEngineBackend', ) @@ -44,3 +44,42 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' .. versionadded:: 0.2.1 + +Storage +======= +With MongoEngine's support for GridFS via the FileField, it is useful to have a +Django file storage backend that wraps this. The new storage module is called +GridFSStorage. Using it is very similar to using the default FileSystemStorage.:: + + fs = mongoengine.django.GridFSStorage() + + filename = fs.save('hello.txt', 'Hello, World!') + +All of the `Django Storage API methods +`_ have been +implemented except ``path()``. If the filename provided already exists, an +underscore and a number (before # the file extension, if one exists) will be +appended to the filename until the generated filename doesn't exist. The +``save()`` method will return the new filename.:: + + > fs.exists('hello.txt') + True + > fs.open('hello.txt').read() + 'Hello, World!' + > fs.size('hello.txt') + 13 + > fs.url('hello.txt') + 'http://your_media_url/hello.txt' + > fs.open('hello.txt').name + 'hello.txt' + > fs.listdir() + ([], [u'hello.txt']) + +All files will be saved and retrieved in GridFS via the ``FileDocument`` document, +allowing easy access to the files without the GridFSStorage backend.:: + + > from mongoengine.django.storage import FileDocument + > FileDocument.objects() + [] + +.. versionadded:: 0.4 diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817da..91a12ab3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -23,7 +23,7 @@ class BaseField(object): # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False - + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): @@ -89,7 +89,7 @@ class BaseField(object): if value not in self.choices: raise ValidationError("Value must be one of %s." % unicode(self.choices)) - + # check validation argument if self.validation is not None: if callable(self.validation): @@ -98,13 +98,13 @@ class BaseField(object): 'validation method.') else: raise ValueError('validation argument must be a callable.') - + self.validate(value) class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ - + def to_python(self, value): return value # return unicode(value) @@ -150,7 +150,7 @@ class DocumentMetaclass(type): # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) - + if hasattr(base, '_meta'): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on @@ -191,20 +191,20 @@ class DocumentMetaclass(type): field.owner_document = new_class module = attrs.get('__module__') - + base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) exc = subclass_exception('DoesNotExist', base_excs, module) new_class.add_to_class('DoesNotExist', exc) - + base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, 'MultipleObjectsReturned')) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception('MultipleObjectsReturned', base_excs, module) new_class.add_to_class('MultipleObjectsReturned', exc) - + return new_class - + def add_to_class(self, name, value): setattr(self, name, value) @@ -227,7 +227,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): return super_new(cls, name, bases, attrs) collection = name.lower() - + id_field = None base_indexes = [] base_meta = {} @@ -265,7 +265,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - + # Provide a default queryset unless one has been manually provided if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() @@ -273,7 +273,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes new_class._meta['indexes'] = user_indexes - + unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints @@ -431,7 +431,7 @@ class BaseDocument(object): if data.has_key('_id') and not data['_id']: del data['_id'] return data - + @classmethod def _from_son(cls, son): """Create an instance of a Document (subclass) from a PyMongo SON. @@ -468,7 +468,7 @@ class BaseDocument(object): obj = cls(**data) obj._present_fields = present_fields return obj - + def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): if self.id == other.id: diff --git a/mongoengine/django/storage.py b/mongoengine/django/storage.py new file mode 100644 index 00000000..341455cd --- /dev/null +++ b/mongoengine/django/storage.py @@ -0,0 +1,112 @@ +import os +import itertools +import urlparse + +from mongoengine import * +from django.conf import settings +from django.core.files.storage import Storage +from django.core.exceptions import ImproperlyConfigured + + +class FileDocument(Document): + """A document used to store a single file in GridFS. + """ + file = FileField() + + +class GridFSStorage(Storage): + """A custom storage backend to store files in GridFS + """ + + def __init__(self, base_url=None): + + if base_url is None: + base_url = settings.MEDIA_URL + self.base_url = base_url + self.document = FileDocument + self.field = 'file' + + def delete(self, name): + """Deletes the specified file from the storage system. + """ + if self.exists(name): + doc = self.document.objects.first() + field = getattr(doc, self.field) + self._get_doc_with_name(name).delete() # Delete the FileField + field.delete() # Delete the FileDocument + + def exists(self, name): + """Returns True if a file referened by the given name already exists in the + storage system, or False if the name is available for a new file. + """ + doc = self._get_doc_with_name(name) + if doc: + field = getattr(doc, self.field) + return bool(field.name) + else: + return False + + def listdir(self, path=None): + """Lists the contents of the specified path, returning a 2-tuple of lists; + the first item being directories, the second item being files. + """ + def name(doc): + return getattr(doc, self.field).name + docs = self.document.objects + return [], [name(d) for d in docs if name(d)] + + def size(self, name): + """Returns the total size, in bytes, of the file specified by name. + """ + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field).length + else: + raise ValueError("No such file or directory: '%s'" % name) + + def url(self, name): + """Returns an absolute URL where the file's contents can be accessed + directly by a web browser. + """ + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + return urlparse.urljoin(self.base_url, name).replace('\\', '/') + + def _get_doc_with_name(self, name): + """Find the documents in the store with the given name + """ + docs = self.document.objects + doc = [d for d in docs if getattr(d, self.field).name == name] + if doc: + return doc[0] + else: + return None + + def _open(self, name, mode='rb'): + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field) + else: + raise ValueError("No file found with the name '%s'." % name) + + def get_available_name(self, name): + """Returns a filename that's free on the target storage system, and + available for new content to be written to. + """ + file_root, file_ext = os.path.splitext(name) + # If the filename already exists, add an underscore and a number (before + # the file extension, if one exists) to the filename until the generated + # filename doesn't exist. + count = itertools.count(1) + while self.exists(name): + # file_ext includes the dot. + name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext)) + + return name + + def _save(self, name, content): + doc = self.document() + getattr(doc, self.field).put(content, filename=name) + doc.save() + + return name diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec145..368b5805 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -15,7 +15,7 @@ class EmbeddedDocument(BaseDocument): fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. """ - + __metaclass__ = DocumentMetaclass @@ -119,23 +119,23 @@ class Document(BaseDocument): class MapReduceDocument(object): """A document returned from a map/reduce query. - + :param collection: An instance of :class:`~pymongo.Collection` :param key: Document/result key, often an instance of :class:`~pymongo.objectid.ObjectId`. If supplied as an ``ObjectId`` found in the given ``collection``, the object can be accessed via the ``object`` property. :param value: The result(s) for this key. - + .. versionadded:: 0.3 """ - + def __init__(self, document, collection, key, value): self._document = document self._collection = collection self.key = key self.value = value - + @property def object(self): """Lazy-load the object referenced by ``self.key``. ``self.key`` @@ -143,7 +143,7 @@ class MapReduceDocument(object): """ id_field = self._document()._meta['id_field'] id_field_type = type(id_field) - + if not isinstance(self.key, id_field_type): try: self.key = id_field_type(self.key) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 76bc4fbe..1b689f28 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,11 +16,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', -<<<<<<< HEAD - 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] -======= 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField'] ->>>>>>> 32e66b29f44f3015be099851201241caee92054f RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -42,7 +38,7 @@ class StringField(BaseField): if self.max_length is not None and len(value) > self.max_length: raise ValidationError('String value is too long') - + if self.min_length is not None and len(value) < self.min_length: raise ValidationError('String value is too short') @@ -350,7 +346,8 @@ class SortedListField(ListField): def to_mongo(self, value): if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value], + key=itemgetter(self._ordering)) return sorted([self.field.to_mongo(item) for item in value]) class DictField(BaseField): @@ -514,25 +511,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS + + .. versionadded:: 0.4 """ -<<<<<<< HEAD - def __init__(self): - self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file -======= def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __getattr__(self, name): obj = self.get() @@ -543,17 +532,13 @@ class GridFSProxy(object): return self def get(self, id=None): -<<<<<<< HEAD - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted -======= if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: - return None # File has been deleted ->>>>>>> 32e66b29f44f3015be099851201241caee92054f + # File has been deleted + return None def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -575,20 +560,19 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): -<<<<<<< HEAD - try: return self.get().read() - except: return None -======= try: return self.get().read() except: return None ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def delete(self): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) - self.grid_id = None + + #self.grid_id = None + # Doesn't make a difference because will be put back in when + # reinstantiated We should delete all the metadata stored with the + # file too def replace(self, file, **kwargs): self.delete() @@ -601,41 +585,30 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class FileField(BaseField): """A GridFS storage field. + + .. versionadded:: 0.4 """ def __init__(self, **kwargs): -<<<<<<< HEAD - self.gridfs = GridFSProxy() -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self -<<<<<<< HEAD - return self.gridfs -======= # Check if a file already exists for this model grid_file = instance._data.get(self.name) - if grid_file: - return grid_file + self.grid_file = grid_file + if self.grid_file: + return self.grid_file return GridFSProxy() ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation -<<<<<<< HEAD - self.gridfs.put(value) -======= grid_file = instance._data.get(self.name) # If a file already exists, delete it if grid_file: @@ -649,24 +622,11 @@ class FileField(BaseField): # Create a new proxy object as we don't already have one instance._data[self.name] = GridFSProxy() instance._data[self.name].put(value) ->>>>>>> 32e66b29f44f3015be099851201241caee92054f else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB -<<<<<<< HEAD - return self.gridfs.grid_id - - def to_python(self, value): - # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() - - def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) - -======= if isinstance(value, GridFSProxy) and value.grid_id is not None: return value.grid_id return None @@ -680,6 +640,7 @@ class FileField(BaseField): assert isinstance(value, GridFSProxy) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + class GeoPointField(BaseField): """A list storing a latitude and longitude. """ @@ -692,10 +653,9 @@ class GeoPointField(BaseField): if not isinstance(value, (list, tuple)): raise ValidationError('GeoPointField can only accept tuples or ' 'lists of (x, y)') - + if not len(value) == 2: raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') ->>>>>>> 32e66b29f44f3015be099851201241caee92054f diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c3..2fb8a9d8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -163,7 +163,7 @@ class QuerySet(object): self._where_clause = None self._loaded_fields = [] self._ordering = [] - + # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance'): @@ -240,7 +240,7 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) - + def all(self): """Returns all documents.""" return self.__call__() @@ -256,7 +256,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_options', {}) - + # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: @@ -267,12 +267,12 @@ class QuerySet(object): for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) - + # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: self._collection.ensure_index('_types', background=background, **index_opts) - + # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: @@ -471,7 +471,7 @@ class QuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - + :param object_ids: a list or tuple of ``ObjectId``\ s :rtype: dict of ObjectIds as keys and collection-specific Document subclasses as values. @@ -483,7 +483,7 @@ class QuerySet(object): docs = self._collection.find({'_id': {'$in': object_ids}}) for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) - + return doc_map def next(self): @@ -637,7 +637,7 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) - + def distinct(self, field): """Return a list of distinct values for a given field. @@ -649,9 +649,9 @@ class QuerySet(object): def only(self, *fields): """Load only a subset of this document's fields. :: - + post = BlogPost.objects(...).only("title") - + :param fields: fields to include .. versionadded:: 0.3 diff --git a/tests/connnection.py b/tests/connnection.py new file mode 100644 index 00000000..1903a5f4 --- /dev/null +++ b/tests/connnection.py @@ -0,0 +1,44 @@ +import unittest +import datetime +import pymongo + +import mongoengine.connection +from mongoengine import * +from mongoengine.connection import _get_db, _get_connection + + +class ConnectionTest(unittest.TestCase): + + def tearDown(self): + mongoengine.connection._connection_settings = {} + mongoengine.connection._connections = {} + mongoengine.connection._dbs = {} + + def test_connect(self): + """Ensure that the connect() method works properly. + """ + connect('mongoenginetest') + + conn = _get_connection() + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + + def test_register_connection(self): + """Ensure that connections with different aliases may be registered. + """ + register_connection('testdb', 'mongoenginetest2') + + self.assertRaises(ConnectionError, _get_connection) + conn = _get_connection('testdb') + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db('testdb') + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest2') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/fields.py b/tests/fields.py index 536a9f1a..f5f38fc2 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,7 +693,7 @@ class FieldTest(unittest.TestCase): testfile.name = "Hello, World!" testfile.file.put('Hello, World!') testfile.save() - + # Second instance testfiledupe = TestFile() data = testfiledupe.file.read() # Should be None