Merge pull request #1031 from MRigal/fix/1011-capped-collection-size-multiple-of-256

CappedCollection max_size normalized to multiple of 256

docs/changelog.rst

@@ -24,6 +24,7 @@ Changes in 0.9.X - DEV
 - Fixes some internal _id handling issue. #961
 - Updated URL and Email Field regex validators, added schemes argument to URLField validation. #652
 - Removed get_or_create() deprecated since 0.8.0. #300
+- Capped collection multiple of 256. #1011
 
 Changes in 0.9.0
 ================

docs/guide/defining-documents.rst

@@ -447,8 +447,10 @@ A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying
 :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary.
 :attr:`max_documents` is the maximum number of documents that is allowed to be
 stored in the collection, and :attr:`max_size` is the maximum size of the
-collection in bytes. If :attr:`max_size` is not specified and
-:attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB).
+collection in bytes. :attr:`max_size` is rounded up to the next multiple of 256
+by MongoDB internally and mongoengine before. Use also a multiple of 256 to
+avoid confusions. If :attr:`max_size` is not specified and
+:attr:`max_documents` is, :attr:`max_size` defaults to 10485760 bytes (10MB).
 The following example shows a :class:`Log` document that will be limited to
 1000 entries and 2MB of disk space::
 

mongoengine/document.py

@@ -114,9 +114,11 @@ class Document(BaseDocument):
     specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta`
     dictionary. :attr:`max_documents` is the maximum number of documents that
     is allowed to be stored in the collection, and :attr:`max_size` is the
-    maximum size of the collection in bytes. If :attr:`max_size` is not
+    maximum size of the collection in bytes. :attr:`max_size` is rounded up
+    to the next multiple of 256 by MongoDB internally and mongoengine before.
+    Use also a multiple of 256 to avoid confusions.  If :attr:`max_size` is not
     specified and :attr:`max_documents` is, :attr:`max_size` defaults to
-    10000000 bytes (10MB).
+    10485760 bytes (10MB).
 
     Indexes may be created by specifying :attr:`indexes` in the :attr:`meta`
     dictionary. The value should be a list of field names or tuples of field
@@ -137,7 +139,7 @@ class Document(BaseDocument):
     By default, any extra attribute existing in stored data but not declared
     in your model will raise a :class:`~mongoengine.FieldDoesNotExist` error.
     This can be disabled by setting :attr:`strict` to ``False``
-    in the :attr:`meta` dictionnary.
+    in the :attr:`meta` dictionary.
     """
 
     # The __metaclass__ attribute is removed by 2to3 when running with Python3
@@ -176,8 +178,11 @@ class Document(BaseDocument):
             # Create collection as a capped collection if specified
             if cls._meta.get('max_size') or cls._meta.get('max_documents'):
                 # Get max document limit and max byte size from meta
-                max_size = cls._meta.get('max_size') or 10000000  # 10MB default
+                max_size = cls._meta.get('max_size') or 10 * 2 ** 20  # 10MB default
                 max_documents = cls._meta.get('max_documents')
+                # Round up to next 256 bytes as MongoDB would do it to avoid exception
+                if max_size % 256:
+                    max_size = (max_size // 256 + 1) * 256
 
                 if collection_name in db.collection_names():
                     cls._collection = db[collection_name]

tests/document/instance.py

@@ -88,7 +88,7 @@ class InstanceTest(unittest.TestCase):
         options = Log.objects._collection.options()
         self.assertEqual(options['capped'], True)
         self.assertEqual(options['max'], 10)
-        self.assertTrue(options['size'] >= 4096)
+        self.assertEqual(options['size'], 4096)
 
         # Check that the document cannot be redefined with different options
         def recreate_log_document():
@@ -103,6 +103,69 @@ class InstanceTest(unittest.TestCase):
 
         Log.drop_collection()
 
+    def test_capped_collection_default(self):
+        """Ensure that capped collections defaults work properly.
+        """
+        class Log(Document):
+            date = DateTimeField(default=datetime.now)
+            meta = {
+                'max_documents': 10,
+            }
+
+        Log.drop_collection()
+
+        # Create a doc to create the collection
+        Log().save()
+
+        options = Log.objects._collection.options()
+        self.assertEqual(options['capped'], True)
+        self.assertEqual(options['max'], 10)
+        self.assertEqual(options['size'], 10 * 2**20)
+
+        # Check that the document with default value can be recreated
+        def recreate_log_document():
+            class Log(Document):
+                date = DateTimeField(default=datetime.now)
+                meta = {
+                    'max_documents': 10,
+                }
+            # Create the collection by accessing Document.objects
+            Log.objects
+        recreate_log_document()
+        Log.drop_collection()
+
+    def test_capped_collection_no_max_size_problems(self):
+        """Ensure that capped collections with odd max_size work properly.
+        MongoDB rounds up max_size to next multiple of 256, recreating a doc
+        with the same spec failed in mongoengine <0.10
+        """
+        class Log(Document):
+            date = DateTimeField(default=datetime.now)
+            meta = {
+                'max_size': 10000,
+            }
+
+        Log.drop_collection()
+
+        # Create a doc to create the collection
+        Log().save()
+
+        options = Log.objects._collection.options()
+        self.assertEqual(options['capped'], True)
+        self.assertTrue(options['size'] >= 10000)
+
+        # Check that the document with odd max_size value can be recreated
+        def recreate_log_document():
+            class Log(Document):
+                date = DateTimeField(default=datetime.now)
+                meta = {
+                    'max_size': 10000,
+                }
+            # Create the collection by accessing Document.objects
+            Log.objects
+        recreate_log_document()
+        Log.drop_collection()
+
     def test_repr(self):
         """Ensure that unicode representation works
         """