Added a Django storage backend.

- New GridFSStorage storage backend
- New FileDocument document for storing files in GridFS
- Whitespace cleaned up in various files

docs/changelog.rst

@@ -4,6 +4,8 @@ Changelog
 
 Changes in v0.4
 ===============
+- Added ``GridFSStorage`` Django storage backend
+- Added ``FileField`` for GridFS support
 - Added ``SortedListField``
 - Added ``EmailField``
 - Added ``GeoPointField``

docs/django.rst

@@ -19,7 +19,7 @@ MongoDB but still use many of the Django authentication infrastucture (such as
 the :func:`login_required` decorator and the :func:`authenticate` function). To
 enable the MongoEngine auth backend, add the following to you **settings.py**
 file::
-    
+
     AUTHENTICATION_BACKENDS = (
         'mongoengine.django.auth.MongoEngineBackend',
     )
@@ -44,3 +44,42 @@ into you settings module::
     SESSION_ENGINE = 'mongoengine.django.sessions'
 
 .. versionadded:: 0.2.1
+
+Storage
+=======
+With MongoEngine's support for GridFS via the FileField, it is useful to have a
+Django file storage backend that wraps this. The new storage module is called
+GridFSStorage. Using it is very similar to using the default FileSystemStorage.::
+
+    fs = mongoengine.django.GridFSStorage()
+
+    filename = fs.save('hello.txt', 'Hello, World!')
+
+All of the `Django Storage API methods
+<http://docs.djangoproject.com/en/dev/ref/files/storage/>`_ have been
+implemented except ``path()``. If the filename provided already exists, an
+underscore and a number (before # the file extension, if one exists) will be
+appended to the filename until the generated filename doesn't exist. The
+``save()`` method will return the new filename.::
+
+    > fs.exists('hello.txt')
+    True
+    > fs.open('hello.txt').read()
+    'Hello, World!'
+    > fs.size('hello.txt')
+    13
+    > fs.url('hello.txt')
+    'http://your_media_url/hello.txt'
+    > fs.open('hello.txt').name
+    'hello.txt'
+    > fs.listdir()
+    ([], [u'hello.txt'])
+
+All files will be saved and retrieved in GridFS via the ``FileDocument`` document,
+allowing easy access to the files without the GridFSStorage backend.::
+
+    > from mongoengine.django.storage import FileDocument
+    > FileDocument.objects()
+    [<FileDocument: FileDocument object>]
+
+.. versionadded:: 0.4

mongoengine/base.py

@@ -23,7 +23,7 @@ class BaseField(object):
     # Fields may have _types inserted into indexes by default 
     _index_with_types = True
     _geo_index = False
-    
+
     def __init__(self, db_field=None, name=None, required=False, default=None, 
                  unique=False, unique_with=None, primary_key=False,
                  validation=None, choices=None):
@@ -89,7 +89,7 @@ class BaseField(object):
             if value not in self.choices:
                 raise ValidationError("Value must be one of %s."
                     % unicode(self.choices))
-        
+
         # check validation argument
         if self.validation is not None:
             if callable(self.validation):
@@ -98,13 +98,13 @@ class BaseField(object):
                                           'validation method.')
             else:
                 raise ValueError('validation argument must be a callable.')
-    
+
         self.validate(value)
 
 class ObjectIdField(BaseField):
     """An field wrapper around MongoDB's ObjectIds.
     """
-    
+
     def to_python(self, value):
         return value
         # return unicode(value)
@@ -150,7 +150,7 @@ class DocumentMetaclass(type):
                 # Get superclasses from superclass
                 superclasses[base._class_name] = base
                 superclasses.update(base._superclasses)
-                
+
             if hasattr(base, '_meta'):
                 # Ensure that the Document class may be subclassed - 
                 # inheritance may be disabled to remove dependency on 
@@ -191,20 +191,20 @@ class DocumentMetaclass(type):
             field.owner_document = new_class
 
         module = attrs.get('__module__')
-        
+
         base_excs = tuple(base.DoesNotExist for base in bases 
                           if hasattr(base, 'DoesNotExist')) or (DoesNotExist,)
         exc = subclass_exception('DoesNotExist', base_excs, module)
         new_class.add_to_class('DoesNotExist', exc)
-        
+
         base_excs = tuple(base.MultipleObjectsReturned for base in bases 
                           if hasattr(base, 'MultipleObjectsReturned'))
         base_excs = base_excs or (MultipleObjectsReturned,)
         exc = subclass_exception('MultipleObjectsReturned', base_excs, module)
         new_class.add_to_class('MultipleObjectsReturned', exc)
-    
+
         return new_class
-        
+
     def add_to_class(self, name, value):
         setattr(self, name, value)
 
@@ -227,7 +227,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
             return super_new(cls, name, bases, attrs)
 
         collection = name.lower()
-        
+
         id_field = None
         base_indexes = []
         base_meta = {}
@@ -265,7 +265,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         # Set up collection manager, needs the class to have fields so use
         # DocumentMetaclass before instantiating CollectionManager object
         new_class = super_new(cls, name, bases, attrs)
-        
+
         # Provide a default queryset unless one has been manually provided
         if not hasattr(new_class, 'objects'):
             new_class.objects = QuerySetManager()
@@ -273,7 +273,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
         user_indexes = [QuerySet._build_index_spec(new_class, spec)
                         for spec in meta['indexes']] + base_indexes
         new_class._meta['indexes'] = user_indexes
-        
+
         unique_indexes = []
         for field_name, field in new_class._fields.items():
             # Generate a list of indexes needed by uniqueness constraints
@@ -431,7 +431,7 @@ class BaseDocument(object):
         if data.has_key('_id') and not data['_id']:
             del data['_id']
         return data
-    
+
     @classmethod
     def _from_son(cls, son):
         """Create an instance of a Document (subclass) from a PyMongo SON.
@@ -468,7 +468,7 @@ class BaseDocument(object):
         obj = cls(**data)
         obj._present_fields = present_fields
         return obj
-    
+
     def __eq__(self, other):
         if isinstance(other, self.__class__) and hasattr(other, 'id'):
             if self.id == other.id:

mongoengine/django/storage.py

@@ -0,0 +1,112 @@
+import os
+import itertools
+import urlparse
+
+from mongoengine import *
+from django.conf import settings
+from django.core.files.storage import Storage
+from django.core.exceptions import ImproperlyConfigured
+
+
+class FileDocument(Document):
+    """A document used to store a single file in GridFS.
+    """
+    file = FileField()
+
+
+class GridFSStorage(Storage):
+    """A custom storage backend to store files in GridFS
+    """
+
+    def __init__(self, base_url=None):
+
+        if base_url is None:
+            base_url = settings.MEDIA_URL
+        self.base_url = base_url
+        self.document = FileDocument
+        self.field = 'file'
+
+    def delete(self, name):
+        """Deletes the specified file from the storage system.
+        """
+        if self.exists(name):
+            doc = self.document.objects.first()
+            field = getattr(doc, self.field)
+            self._get_doc_with_name(name).delete()  # Delete the FileField
+            field.delete()                          # Delete the FileDocument
+
+    def exists(self, name):
+        """Returns True if a file referened by the given name already exists in the
+        storage system, or False if the name is available for a new file.
+        """
+        doc = self._get_doc_with_name(name)
+        if doc:
+            field = getattr(doc, self.field)
+            return bool(field.name)
+        else:
+            return False
+
+    def listdir(self, path=None):
+        """Lists the contents of the specified path, returning a 2-tuple of lists;
+        the first item being directories, the second item being files.
+        """
+        def name(doc):
+            return getattr(doc, self.field).name
+        docs = self.document.objects
+        return [], [name(d) for d in docs if name(d)]
+
+    def size(self, name):
+        """Returns the total size, in bytes, of the file specified by name.
+        """
+        doc = self._get_doc_with_name(name)
+        if doc:
+            return getattr(doc, self.field).length
+        else:
+            raise ValueError("No such file or directory: '%s'" % name)
+
+    def url(self, name):
+        """Returns an absolute URL where the file's contents can be accessed
+        directly by a web browser.
+        """
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
+
+    def _get_doc_with_name(self, name):
+        """Find the documents in the store with the given name
+        """
+        docs = self.document.objects
+        doc = [d for d in docs if getattr(d, self.field).name == name]
+        if doc:
+            return doc[0]
+        else:
+            return None
+
+    def _open(self, name, mode='rb'):
+        doc = self._get_doc_with_name(name)
+        if doc:
+            return getattr(doc, self.field)
+        else:
+            raise ValueError("No file found with the name '%s'." % name)
+
+    def get_available_name(self, name):
+        """Returns a filename that's free on the target storage system, and
+        available for new content to be written to.
+        """
+        file_root, file_ext = os.path.splitext(name)
+        # If the filename already exists, add an underscore and a number (before
+        # the file extension, if one exists) to the filename until the generated
+        # filename doesn't exist.
+        count = itertools.count(1)
+        while self.exists(name):
+            # file_ext includes the dot.
+            name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext))
+
+        return name
+
+    def _save(self, name, content):
+        doc = self.document()
+        getattr(doc, self.field).put(content, filename=name)
+        doc.save()
+
+        return name

mongoengine/document.py

@@ -15,7 +15,7 @@ class EmbeddedDocument(BaseDocument):
     fields on :class:`~mongoengine.Document`\ s through the
     :class:`~mongoengine.EmbeddedDocumentField` field type.
     """
-    
+
     __metaclass__ = DocumentMetaclass
 
 
@@ -119,23 +119,23 @@ class Document(BaseDocument):
 
 class MapReduceDocument(object):
     """A document returned from a map/reduce query.
-    
+
     :param collection: An instance of :class:`~pymongo.Collection`
     :param key: Document/result key, often an instance of 
                 :class:`~pymongo.objectid.ObjectId`. If supplied as 
                 an ``ObjectId`` found in the given ``collection``, 
                 the object can be accessed via the ``object`` property.
     :param value: The result(s) for this key.
-    
+
     .. versionadded:: 0.3
     """
-    
+
     def __init__(self, document, collection, key, value):
         self._document = document
         self._collection = collection
         self.key = key
         self.value = value
-    
+
     @property
     def object(self):
         """Lazy-load the object referenced by ``self.key``. ``self.key`` 
@@ -143,7 +143,7 @@ class MapReduceDocument(object):
         """
         id_field = self._document()._meta['id_field']
         id_field_type = type(id_field)
-        
+
         if not isinstance(self.key, id_field_type):
             try:
                 self.key = id_field_type(self.key)

mongoengine/fields.py

@@ -16,11 +16,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
            'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField',
            'ObjectIdField', 'ReferenceField', 'ValidationError',
            'DecimalField', 'URLField', 'GenericReferenceField', 'FileField',
-<<<<<<< HEAD
-           'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField']
-=======
            'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField']
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 
 RECURSIVE_REFERENCE_CONSTANT = 'self'
 
@@ -42,7 +38,7 @@ class StringField(BaseField):
 
         if self.max_length is not None and len(value) > self.max_length:
             raise ValidationError('String value is too long')
-        
+
         if self.min_length is not None and len(value) < self.min_length:
             raise ValidationError('String value is too short')
 
@@ -350,7 +346,8 @@ class SortedListField(ListField):
 
     def to_mongo(self, value):
         if self._ordering is not None:
-            return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering))
+            return sorted([self.field.to_mongo(item) for item in value],
+                          key=itemgetter(self._ordering))
         return sorted([self.field.to_mongo(item) for item in value])
 
 class DictField(BaseField):
@@ -514,25 +511,17 @@ class BinaryField(BaseField):
         if self.max_bytes is not None and len(value) > self.max_bytes:
             raise ValidationError('Binary value is too long')
 
-<<<<<<< HEAD
-=======
 
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 class GridFSProxy(object):
     """Proxy object to handle writing and reading of files to and from GridFS
+
+    .. versionadded:: 0.4
     """
 
-<<<<<<< HEAD
-    def __init__(self):
-        self.fs = gridfs.GridFS(_get_db())  # Filesystem instance
-        self.newfile = None                 # Used for partial writes
-        self.grid_id = None                 # Store GridFS id for file
-=======
     def __init__(self, grid_id=None):
         self.fs = gridfs.GridFS(_get_db())  # Filesystem instance
         self.newfile = None                 # Used for partial writes
         self.grid_id = grid_id              # Store GridFS id for file
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 
     def __getattr__(self, name):
         obj = self.get()
@@ -543,17 +532,13 @@ class GridFSProxy(object):
         return self
 
     def get(self, id=None):
-<<<<<<< HEAD
-        try: return self.fs.get(id or self.grid_id)
-        except: return None # File has been deleted
-=======
         if id:
             self.grid_id = id
         try:
             return self.fs.get(id or self.grid_id)
         except:
-            return None # File has been deleted
+            # File has been deleted
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
+            return None
 
     def new_file(self, **kwargs):
         self.newfile = self.fs.new_file(**kwargs)
@@ -575,20 +560,19 @@ class GridFSProxy(object):
         self.newfile.writelines(lines) 
 
     def read(self):
-<<<<<<< HEAD
-        try: return self.get().read()
-        except: return None
-=======
         try:
             return self.get().read()
         except:
             return None
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 
     def delete(self):
         # Delete file from GridFS, FileField still remains
         self.fs.delete(self.grid_id)
-        self.grid_id = None
+
+        #self.grid_id = None
+        # Doesn't make a difference because will be put back in when
+        # reinstantiated We should delete all the metadata stored with the
+        # file too
 
     def replace(self, file, **kwargs):
         self.delete()
@@ -601,41 +585,30 @@ class GridFSProxy(object):
             msg = "The close() method is only necessary after calling write()"
             warnings.warn(msg)
 
-<<<<<<< HEAD
-=======
 
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 class FileField(BaseField):
     """A GridFS storage field.
+
+    .. versionadded:: 0.4
     """
 
     def __init__(self, **kwargs):
-<<<<<<< HEAD
-        self.gridfs = GridFSProxy()
-=======
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
         super(FileField, self).__init__(**kwargs)
 
     def __get__(self, instance, owner):
         if instance is None:
             return self
 
-<<<<<<< HEAD
-        return self.gridfs
-=======
         # Check if a file already exists for this model
         grid_file = instance._data.get(self.name)
-        if grid_file:
+        self.grid_file = grid_file
-            return grid_file
+        if self.grid_file:
+            return self.grid_file
         return GridFSProxy()
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
 
     def __set__(self, instance, value):
         if isinstance(value, file) or isinstance(value, str):
             # using "FileField() = file/string" notation
-<<<<<<< HEAD
-            self.gridfs.put(value)
-=======
             grid_file = instance._data.get(self.name)
             # If a file already exists, delete it
             if grid_file:
@@ -649,24 +622,11 @@ class FileField(BaseField):
                 # Create a new proxy object as we don't already have one
                 instance._data[self.name] = GridFSProxy()
                 instance._data[self.name].put(value)
->>>>>>> 32e66b29f44f3015be099851201241caee92054f
         else:
             instance._data[self.name] = value
 
     def to_mongo(self, value):
         # Store the GridFS file id in MongoDB
-<<<<<<< HEAD
-        return self.gridfs.grid_id
-
-    def to_python(self, value):
-        # Use stored value (id) to lookup file in GridFS
-        return self.gridfs.get()
-
-    def validate(self, value):
-        assert isinstance(value, GridFSProxy)
-        assert isinstance(value.grid_id, pymongo.objectid.ObjectId)
-
-=======
         if isinstance(value, GridFSProxy) and value.grid_id is not None:
             return value.grid_id
         return None
@@ -680,6 +640,7 @@ class FileField(BaseField):
             assert isinstance(value, GridFSProxy)
             assert isinstance(value.grid_id, pymongo.objectid.ObjectId)
 
+
 class GeoPointField(BaseField):
     """A list storing a latitude and longitude.
     """
@@ -692,10 +653,9 @@ class GeoPointField(BaseField):
         if not isinstance(value, (list, tuple)):
             raise ValidationError('GeoPointField can only accept tuples or '
                                   'lists of (x, y)')
-        
+
         if not len(value) == 2:
             raise ValidationError('Value must be a two-dimensional point.')
         if (not isinstance(value[0], (float, int)) and
             not isinstance(value[1], (float, int))):
             raise ValidationError('Both values in point must be float or int.')
->>>>>>> 32e66b29f44f3015be099851201241caee92054f

mongoengine/queryset.py

@@ -163,7 +163,7 @@ class QuerySet(object):
         self._where_clause = None
         self._loaded_fields = []
         self._ordering = []
-        
+
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
         if document._meta.get('allow_inheritance'):
@@ -240,7 +240,7 @@ class QuerySet(object):
         """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__`
         """
         return self.__call__(*q_objs, **query)
-    
+
     def all(self):
         """Returns all documents."""
         return self.__call__()
@@ -256,7 +256,7 @@ class QuerySet(object):
             background = self._document._meta.get('index_background', False)
             drop_dups = self._document._meta.get('index_drop_dups', False)
             index_opts = self._document._meta.get('index_options', {})
-            
+
             # Ensure document-defined indexes are created
             if self._document._meta['indexes']:
                 for key_or_list in self._document._meta['indexes']:
@@ -267,12 +267,12 @@ class QuerySet(object):
             for index in self._document._meta['unique_indexes']:
                 self._collection.ensure_index(index, unique=True,
                     background=background, drop_dups=drop_dups, **index_opts)
-                
+
             # If _types is being used (for polymorphism), it needs an index
             if '_types' in self._query:
                 self._collection.ensure_index('_types',
                     background=background, **index_opts)
-            
+
             # Ensure all needed field indexes are created
             for field in self._document._fields.values():
                 if field.__class__._geo_index:
@@ -471,7 +471,7 @@ class QuerySet(object):
 
     def in_bulk(self, object_ids):
         """Retrieve a set of documents by their ids.
-        
+
         :param object_ids: a list or tuple of ``ObjectId``\ s
         :rtype: dict of ObjectIds as keys and collection-specific
                 Document subclasses as values.
@@ -483,7 +483,7 @@ class QuerySet(object):
         docs = self._collection.find({'_id': {'$in': object_ids}})
         for doc in docs:
             doc_map[doc['_id']] = self._document._from_son(doc)
- 
+
         return doc_map
 
     def next(self):
@@ -637,7 +637,7 @@ class QuerySet(object):
         # Integer index provided
         elif isinstance(key, int):
             return self._document._from_son(self._cursor[key])
-    
+
     def distinct(self, field):
         """Return a list of distinct values for a given field.
 
@@ -649,9 +649,9 @@ class QuerySet(object):
 
     def only(self, *fields):
         """Load only a subset of this document's fields. ::
-        
+
             post = BlogPost.objects(...).only("title")
-        
+
         :param fields: fields to include
 
         .. versionadded:: 0.3

tests/connnection.py

@@ -0,0 +1,44 @@
+import unittest
+import datetime
+import pymongo
+
+import mongoengine.connection
+from mongoengine import *
+from mongoengine.connection import _get_db, _get_connection
+
+
+class ConnectionTest(unittest.TestCase):
+
+    def tearDown(self):
+        mongoengine.connection._connection_settings = {}
+        mongoengine.connection._connections = {}
+        mongoengine.connection._dbs = {}
+    
+    def test_connect(self):
+        """Ensure that the connect() method works properly.
+        """
+        connect('mongoenginetest')
+
+        conn = _get_connection()
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = _get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+    def test_register_connection(self):
+        """Ensure that connections with different aliases may be registered.
+        """
+        register_connection('testdb', 'mongoenginetest2')
+
+        self.assertRaises(ConnectionError, _get_connection)
+        conn = _get_connection('testdb')
+        self.assertTrue(isinstance(conn, pymongo.connection.Connection))
+
+        db = _get_db('testdb')
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest2')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/fields.py

@@ -693,7 +693,7 @@ class FieldTest(unittest.TestCase):
         testfile.name = "Hello, World!"
         testfile.file.put('Hello, World!')
         testfile.save()
-        
+
         # Second instance
         testfiledupe = TestFile()
         data = testfiledupe.file.read() # Should be None