From 2753e02cda4f3a1aae7ba1eba13c7ff31c496781 Mon Sep 17 00:00:00 2001 From: Charanpal Date: Sat, 23 May 2015 14:46:56 +0100 Subject: [PATCH 1/5] Fix for case where Document is deleted and it's files (FieldFields) in GridFS remain. --- mongoengine/document.py | 6 ++++ tests/fields/file_tests.py | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index 838feb81..813fcb2e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -463,6 +463,12 @@ class Document(BaseDocument): """ signals.pre_delete.send(self.__class__, document=self) + #Delete FileFields seperately + FileField = _import_class('FileField') + for name, field in self._fields.iteritems(): + if isinstance(field, FileField): + getattr(self, name).delete() + try: self._qs.filter( **self._object_key).delete(write_concern=write_concern, _from_doc_delete=True) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 811f17ed..7db90059 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -297,6 +297,71 @@ class FileTest(unittest.TestCase): test_file = TestFile() self.assertFalse(test_file.the_file in [{"test": 1}]) + def test_file_disk_space(self): + """ Test disk space usage when we delete/replace a file """ + class TestFile(Document): + the_file = FileField() + + text = b('Hello, World!') + content_type = 'text/plain' + + testfile = TestFile() + testfile.the_file.put(text, content_type=content_type, filename="hello") + testfile.save() + + #Now check fs.files and fs.chunks + db = TestFile._get_db() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 1) + self.assertEquals(len(list(chunks)), 1) + + #Deleting the docoument should delete the files + testfile.delete() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 0) + self.assertEquals(len(list(chunks)), 0) + + #Test case where we don't store a file in the first place + testfile = TestFile() + testfile.save() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 0) + self.assertEquals(len(list(chunks)), 0) + + testfile.delete() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 0) + self.assertEquals(len(list(chunks)), 0) + + #Test case where we overwrite the file + testfile = TestFile() + testfile.the_file.put(text, content_type=content_type, filename="hello") + testfile.save() + + text = b('Bonjour, World!') + testfile.the_file.replace(text, content_type=content_type, filename="hello") + testfile.save() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 1) + self.assertEquals(len(list(chunks)), 1) + + testfile.delete() + + files = db.fs.files.find() + chunks = db.fs.chunks.find() + self.assertEquals(len(list(files)), 0) + self.assertEquals(len(list(chunks)), 0) + def test_image_field(self): if not HAS_PIL: raise SkipTest('PIL not installed') From fc140d04ef666930709ac185fc23a319372b443a Mon Sep 17 00:00:00 2001 From: Charanpal Dhanjal Date: Tue, 2 Jun 2015 10:15:27 +0100 Subject: [PATCH 2/5] Fix comment in delete --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 813fcb2e..f83f9fa1 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -463,7 +463,7 @@ class Document(BaseDocument): """ signals.pre_delete.send(self.__class__, document=self) - #Delete FileFields seperately + # Delete FileFields separately FileField = _import_class('FileField') for name, field in self._fields.iteritems(): if isinstance(field, FileField): From ad81470d350d427b7c10e8c584e51af7c81e7c52 Mon Sep 17 00:00:00 2001 From: Charanpal Dhanjal Date: Tue, 2 Jun 2015 10:17:17 +0100 Subject: [PATCH 3/5] Put space after hash --- tests/fields/file_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 7db90059..4bd1b164 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -309,7 +309,7 @@ class FileTest(unittest.TestCase): testfile.the_file.put(text, content_type=content_type, filename="hello") testfile.save() - #Now check fs.files and fs.chunks + # Now check fs.files and fs.chunks db = TestFile._get_db() files = db.fs.files.find() @@ -317,7 +317,7 @@ class FileTest(unittest.TestCase): self.assertEquals(len(list(files)), 1) self.assertEquals(len(list(chunks)), 1) - #Deleting the docoument should delete the files + # Deleting the docoument should delete the files testfile.delete() files = db.fs.files.find() @@ -325,7 +325,7 @@ class FileTest(unittest.TestCase): self.assertEquals(len(list(files)), 0) self.assertEquals(len(list(chunks)), 0) - #Test case where we don't store a file in the first place + # Test case where we don't store a file in the first place testfile = TestFile() testfile.save() @@ -341,7 +341,7 @@ class FileTest(unittest.TestCase): self.assertEquals(len(list(files)), 0) self.assertEquals(len(list(chunks)), 0) - #Test case where we overwrite the file + # Test case where we overwrite the file testfile = TestFile() testfile.the_file.put(text, content_type=content_type, filename="hello") testfile.save() From 73893f2a335c4389b544020f71b95d05922779c8 Mon Sep 17 00:00:00 2001 From: Charanpal Dhanjal Date: Tue, 2 Jun 2015 10:20:37 +0100 Subject: [PATCH 4/5] Added charanpald --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index f424dbc2..37170ffa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -222,3 +222,4 @@ that much better: * Catstyle Lee (https://github.com/Catstyle) * Kiryl Yermakou (https://github.com/rma4ok) * Matthieu Rigal (https://github.com/MRigal) + * Charanpal Dhanjal (https://github.com/charanpald) From 380d869195f42e63c3e5f9b23eb3c22b4ad4466f Mon Sep 17 00:00:00 2001 From: Charanpal Dhanjal Date: Tue, 2 Jun 2015 10:23:37 +0100 Subject: [PATCH 5/5] Add fix to FileField deletion --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19d30698..ffbc8c54 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,7 @@ Changes in 0.9.X - DEV - Fix for updating sorting in SortedListField. #978 - Added __ support to escape field name in fields lookup keywords that match operators names #949 - Support for PyMongo 3+ #946 +- Fix for issue where FileField deletion did not free space in GridFS. Changes in 0.9.0 ================