From 6363b6290b14b5f6a36fcb9c3b35ea15541928f4 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Jan 2010 03:33:42 +0000 Subject: [PATCH] Added capped collections support --- docs/userguide.rst | 15 +++++++++++++++ mongoengine/base.py | 2 ++ mongoengine/document.py | 8 ++++++++ mongoengine/queryset.py | 33 +++++++++++++++++++++++++++++++-- tests/document.py | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/userguide.rst b/docs/userguide.rst index 12059f89..152e3402 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -153,6 +153,21 @@ document class to use:: title = StringField(max_length=200, required=True) meta = {'collection': 'cmsPage'} +Capped collections +^^^^^^^^^^^^^^^^^^ +A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying +:attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary. +:attr:`max_documents` is the maximum number of documents that is allowed to be +stored in the collection, and :attr:`max_size` is the maximum size of the +collection in bytes. If :attr:`max_size` is not specified and +:attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB). +The following example shows a :class:`Log` document that will be limited to +1000 entries and 2MB of disk space:: + + class Log(Document): + ip_address = StringField() + meta = {'max_documents': 1000, 'max_size': 2000000} + Document inheritance -------------------- To create a specialised type of a :class:`~mongoengine.Document` you have diff --git a/mongoengine/base.py b/mongoengine/base.py index 40b03dfa..cbea67df 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -144,6 +144,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta = { 'collection': collection, 'allow_inheritance': True, + 'max_documents': None, + 'max_size': None, } meta.update(attrs.get('meta', {})) # Only simple classes - direct subclasses of Document - may set diff --git a/mongoengine/document.py b/mongoengine/document.py index c031c860..61687dd7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -36,6 +36,14 @@ class Document(BaseDocument): though). To disable this behaviour and remove the dependence on the presence of `_cls` and `_types`, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary. + + A :class:`~mongoengine.Document` may use a **Capped Collection** by + specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` + dictionary. :attr:`max_documents` is the maximum number of documents that + is allowed to be stored in the collection, and :attr:`max_size` is the + maximum size of the collection in bytes. If :attr:`max_size` is not + specified and :attr:`max_documents` is, :attr:`max_size` defaults to + 10000000 bytes (10MB). """ __metaclass__ = TopLevelDocumentMetaclass diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f5c1cdd1..ff2d8a3e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -3,7 +3,7 @@ from connection import _get_db import pymongo -__all__ = ['queryset_manager'] +__all__ = ['queryset_manager', 'InvalidQueryError', 'InvalidCollectionError'] class InvalidQueryError(Exception): @@ -280,6 +280,10 @@ class QuerySet(object): return self.exec_js(freq_func, list_field, normalize=normalize) +class InvalidCollectionError(Exception): + pass + + class QuerySetManager(object): def __init__(self, manager_func=None): @@ -296,7 +300,32 @@ class QuerySetManager(object): if self._collection is None: db = _get_db() - self._collection = db[owner._meta['collection']] + collection = owner._meta['collection'] + + # Create collection as a capped collection if specified + if owner._meta['max_size'] or owner._meta['max_documents']: + # Get max document limit and max byte size from meta + max_size = owner._meta['max_size'] or 10000000 # 10MB default + max_documents = owner._meta['max_documents'] + + if collection in db.collection_names(): + self._collection = db[collection] + # The collection already exists, check if its capped + # options match the specified capped options + options = self._collection.options() + if options.get('max') != max_documents or \ + options.get('size') != max_size: + msg = ('Cannot create collection "%s" as a capped ' + 'collection as it already exists') % collection + raise InvalidCollectionError(msg) + else: + # Create the collection as a capped collection + opts = {'capped': True, 'size': max_size} + if max_documents: + opts['max'] = max_documents + self._collection = db.create_collection(collection, opts) + else: + self._collection = db[collection] # owner is the document that contains the QuerySetManager queryset = QuerySet(owner, self._collection) diff --git a/tests/document.py b/tests/document.py index ac4cbc38..e977eaa0 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1,4 +1,5 @@ import unittest +import datetime import pymongo from mongoengine import * @@ -180,6 +181,46 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() self.assertFalse(collection in self.db.collection_names()) + def test_capped_collection(self): + """Ensure that capped collections work properly. + """ + class Log(Document): + date = DateTimeField(default=datetime.datetime.now) + meta = { + 'max_documents': 10, + 'max_size': 90000, + } + + Log.drop_collection() + + # Ensure that the collection handles up to its maximum + for i in range(10): + Log().save() + + self.assertEqual(len(Log.objects), 10) + + # Check that extra documents don't increase the size + Log().save() + self.assertEqual(len(Log.objects), 10) + + options = Log.objects._collection.options() + self.assertEqual(options['capped'], True) + self.assertEqual(options['max'], 10) + self.assertEqual(options['size'], 90000) + + # Check that the document cannot be redefined with different options + def recreate_log_document(): + class Log(Document): + date = DateTimeField(default=datetime.datetime.now) + meta = { + 'max_documents': 11, + } + # Create the collection by accessing Document.objects + Log.objects + self.assertRaises(InvalidCollectionError, recreate_log_document) + + Log.drop_collection() + def test_creation(self): """Ensure that document may be created using keyword arguments. """