UUIDField now stores as a binary by default (#292)

docs/changelog.rst

@@ -4,6 +4,7 @@ Changelog
 
 Changes in 0.8.X
 ================
+- UUIDField now stores as a binary by default (#292)
 - Added Custom User Model for Django 1.5 (#285)
 - Cascading saves now default to off (#291)
 - ReferenceField now store ObjectId's by default rather than DBRef (#290)

docs/upgrade.rst

@@ -120,6 +120,30 @@ eg::
         p._mark_as_dirty('friends')
         p.save()
 
+UUIDField
+---------
+
+UUIDFields now default to storing binary values::
+
+    # Old code
+    class Animal(Document):
+        uuid = UUIDField()
+
+    # New code
+    class Animal(Document):
+        uuid = UUIDField(binary=False)
+
+To migrate all the uuid's you need to touch each object and mark it as dirty
+eg::
+
+    # Doc definition
+    class Animal(Document):
+        uuid = UUIDField()
+
+    # Mark all ReferenceFields as dirty and save
+    for a in Animal.objects:
+        a._mark_as_dirty('uuid')
+        a.save()
 
 Cascading Saves
 ---------------

mongoengine/fields.py

@@ -1474,19 +1474,15 @@ class UUIDField(BaseField):
     """
     _binary = None
 
-    def __init__(self, binary=None, **kwargs):
+    def __init__(self, binary=True, **kwargs):
         """
         Store UUID data in the database
 
-        :param binary: (optional) boolean store as binary.
+        :param binary: if False store as a string.
 
+        .. versionchanged:: 0.8.0
         .. versionchanged:: 0.6.19
         """
-        if binary is None:
-            binary = False
-            msg = ("UUIDFields will soon default to store as binary, please "
-                  "configure binary=False if you wish to store as a string")
-            warnings.warn(msg, FutureWarning)
         self._binary = binary
         super(UUIDField, self).__init__(**kwargs)
 
@@ -1504,6 +1500,8 @@ class UUIDField(BaseField):
     def to_mongo(self, value):
         if not self._binary:
             return unicode(value)
+        elif isinstance(value, basestring):
+            return uuid.UUID(value)
         return value
 
     def prepare_query_value(self, op, value):

tests/__init__.py

@@ -1,3 +1,5 @@
 from all_warnings import AllWarnings
 from document import *
-from queryset import *
+from queryset import *
+from fields import *
+from migration import *

tests/document/delta.py

@@ -129,14 +129,14 @@ class DeltaTest(unittest.TestCase):
         }
         self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {}))
         self.assertEqual(doc._delta(),
-            ({'embedded_field': embedded_delta}, {}))
+                         ({'embedded_field': embedded_delta}, {}))
 
         doc.save()
         doc = doc.reload(10)
 
         doc.embedded_field.dict_field = {}
         self.assertEqual(doc._get_changed_fields(),
-                        ['embedded_field.dict_field'])
+                         ['embedded_field.dict_field'])
         self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1}))
         self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1}))
         doc.save()
@@ -145,7 +145,7 @@ class DeltaTest(unittest.TestCase):
 
         doc.embedded_field.list_field = []
         self.assertEqual(doc._get_changed_fields(),
-            ['embedded_field.list_field'])
+                         ['embedded_field.list_field'])
         self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1}))
         self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1}))
         doc.save()
@@ -160,7 +160,7 @@ class DeltaTest(unittest.TestCase):
 
         doc.embedded_field.list_field = ['1', 2, embedded_2]
         self.assertEqual(doc._get_changed_fields(),
-            ['embedded_field.list_field'])
+                         ['embedded_field.list_field'])
 
         self.assertEqual(doc.embedded_field._delta(), ({
             'list_field': ['1', 2, {
@@ -192,11 +192,11 @@ class DeltaTest(unittest.TestCase):
 
         doc.embedded_field.list_field[2].string_field = 'world'
         self.assertEqual(doc._get_changed_fields(),
-            ['embedded_field.list_field.2.string_field'])
+                         ['embedded_field.list_field.2.string_field'])
         self.assertEqual(doc.embedded_field._delta(),
-            ({'list_field.2.string_field': 'world'}, {}))
+                         ({'list_field.2.string_field': 'world'}, {}))
         self.assertEqual(doc._delta(),
-            ({'embedded_field.list_field.2.string_field': 'world'}, {}))
+                         ({'embedded_field.list_field.2.string_field': 'world'}, {}))
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].string_field,
@@ -206,7 +206,7 @@ class DeltaTest(unittest.TestCase):
         doc.embedded_field.list_field[2].string_field = 'hello world'
         doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2]
         self.assertEqual(doc._get_changed_fields(),
-            ['embedded_field.list_field'])
+                         ['embedded_field.list_field'])
         self.assertEqual(doc.embedded_field._delta(), ({
             'list_field': ['1', 2, {
             '_cls': 'Embedded',
@@ -225,40 +225,40 @@ class DeltaTest(unittest.TestCase):
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].string_field,
-                        'hello world')
+                         'hello world')
 
         # Test list native methods
         doc.embedded_field.list_field[2].list_field.pop(0)
         self.assertEqual(doc._delta(),
-            ({'embedded_field.list_field.2.list_field':
+                         ({'embedded_field.list_field.2.list_field':
-                [2, {'hello': 'world'}]}, {}))
+                          [2, {'hello': 'world'}]}, {}))
         doc.save()
         doc = doc.reload(10)
 
         doc.embedded_field.list_field[2].list_field.append(1)
         self.assertEqual(doc._delta(),
-            ({'embedded_field.list_field.2.list_field':
+                         ({'embedded_field.list_field.2.list_field':
-                [2, {'hello': 'world'}, 1]}, {}))
+                          [2, {'hello': 'world'}, 1]}, {}))
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].list_field,
-            [2, {'hello': 'world'}, 1])
+                         [2, {'hello': 'world'}, 1])
 
         doc.embedded_field.list_field[2].list_field.sort(key=str)
         doc.save()
         doc = doc.reload(10)
         self.assertEqual(doc.embedded_field.list_field[2].list_field,
-            [1, 2, {'hello': 'world'}])
+                         [1, 2, {'hello': 'world'}])
 
         del(doc.embedded_field.list_field[2].list_field[2]['hello'])
         self.assertEqual(doc._delta(),
-            ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {}))
+                         ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {}))
         doc.save()
         doc = doc.reload(10)
 
         del(doc.embedded_field.list_field[2].list_field)
         self.assertEqual(doc._delta(),
-            ({}, {'embedded_field.list_field.2.list_field': 1}))
+                         ({}, {'embedded_field.list_field.2.list_field': 1}))
 
         doc.save()
         doc = doc.reload(10)
@@ -269,9 +269,9 @@ class DeltaTest(unittest.TestCase):
 
         doc.dict_field['Embedded'].string_field = 'Hello World'
         self.assertEqual(doc._get_changed_fields(),
-            ['dict_field.Embedded.string_field'])
+                         ['dict_field.Embedded.string_field'])
         self.assertEqual(doc._delta(),
-            ({'dict_field.Embedded.string_field': 'Hello World'}, {}))
+                         ({'dict_field.Embedded.string_field': 'Hello World'}, {}))
 
     def test_circular_reference_deltas(self):
         self.circular_reference_deltas(Document, Document)
@@ -289,10 +289,11 @@ class DeltaTest(unittest.TestCase):
             name = StringField()
             owner = ReferenceField('Person')
 
-        person = Person(name="owner")
+        Person.drop_collection()
-        person.save()
+        Organization.drop_collection()
-        organization = Organization(name="company")
+
-        organization.save()
+        person = Person(name="owner").save()
+        organization = Organization(name="company").save()
 
         person.owns.append(organization)
         organization.owner = person

tests/fields/fields.py

@@ -354,7 +354,6 @@ class FieldTest(unittest.TestCase):
             person.api_key = api_key
             self.assertRaises(ValidationError, person.validate)
 
-
     def test_datetime_validation(self):
         """Ensure that invalid values cannot be assigned to datetime fields.
         """
@@ -1805,304 +1804,6 @@ class FieldTest(unittest.TestCase):
 
         Shirt.drop_collection()
 
-    def test_file_fields(self):
-        """Ensure that file fields can be written to and their data retrieved
-        """
-        class PutFile(Document):
-            the_file = FileField()
-
-        class StreamFile(Document):
-            the_file = FileField()
-
-        class SetFile(Document):
-            the_file = FileField()
-
-        text = b('Hello, World!')
-        more_text = b('Foo Bar')
-        content_type = 'text/plain'
-
-        PutFile.drop_collection()
-        StreamFile.drop_collection()
-        SetFile.drop_collection()
-
-        putfile = PutFile()
-        putfile.the_file.put(text, content_type=content_type)
-        putfile.save()
-        putfile.validate()
-        result = PutFile.objects.first()
-        self.assertTrue(putfile == result)
-        self.assertEqual(result.the_file.read(), text)
-        self.assertEqual(result.the_file.content_type, content_type)
-        result.the_file.delete() # Remove file from GridFS
-        PutFile.objects.delete()
-
-        # Ensure file-like objects are stored
-        putfile = PutFile()
-        putstring = StringIO()
-        putstring.write(text)
-        putstring.seek(0)
-        putfile.the_file.put(putstring, content_type=content_type)
-        putfile.save()
-        putfile.validate()
-        result = PutFile.objects.first()
-        self.assertTrue(putfile == result)
-        self.assertEqual(result.the_file.read(), text)
-        self.assertEqual(result.the_file.content_type, content_type)
-        result.the_file.delete()
-
-        streamfile = StreamFile()
-        streamfile.the_file.new_file(content_type=content_type)
-        streamfile.the_file.write(text)
-        streamfile.the_file.write(more_text)
-        streamfile.the_file.close()
-        streamfile.save()
-        streamfile.validate()
-        result = StreamFile.objects.first()
-        self.assertTrue(streamfile == result)
-        self.assertEqual(result.the_file.read(), text + more_text)
-        self.assertEqual(result.the_file.content_type, content_type)
-        result.the_file.seek(0)
-        self.assertEqual(result.the_file.tell(), 0)
-        self.assertEqual(result.the_file.read(len(text)), text)
-        self.assertEqual(result.the_file.tell(), len(text))
-        self.assertEqual(result.the_file.read(len(more_text)), more_text)
-        self.assertEqual(result.the_file.tell(), len(text + more_text))
-        result.the_file.delete()
-
-        # Ensure deleted file returns None
-        self.assertTrue(result.the_file.read() == None)
-
-        setfile = SetFile()
-        setfile.the_file = text
-        setfile.save()
-        setfile.validate()
-        result = SetFile.objects.first()
-        self.assertTrue(setfile == result)
-        self.assertEqual(result.the_file.read(), text)
-
-        # Try replacing file with new one
-        result.the_file.replace(more_text)
-        result.save()
-        result.validate()
-        result = SetFile.objects.first()
-        self.assertTrue(setfile == result)
-        self.assertEqual(result.the_file.read(), more_text)
-        result.the_file.delete()
-
-        PutFile.drop_collection()
-        StreamFile.drop_collection()
-        SetFile.drop_collection()
-
-        # Make sure FileField is optional and not required
-        class DemoFile(Document):
-            the_file = FileField()
-        DemoFile.objects.create()
-
-
-    def test_file_field_no_default(self):
-
-        class GridDocument(Document):
-            the_file = FileField()
-
-        GridDocument.drop_collection()
-
-        with tempfile.TemporaryFile() as f:
-            f.write(b("Hello World!"))
-            f.flush()
-
-            # Test without default
-            doc_a = GridDocument()
-            doc_a.save()
-
-
-            doc_b = GridDocument.objects.with_id(doc_a.id)
-            doc_b.the_file.replace(f, filename='doc_b')
-            doc_b.save()
-            self.assertNotEqual(doc_b.the_file.grid_id, None)
-
-            # Test it matches
-            doc_c = GridDocument.objects.with_id(doc_b.id)
-            self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id)
-
-            # Test with default
-            doc_d = GridDocument(the_file=b(''))
-            doc_d.save()
-
-            doc_e = GridDocument.objects.with_id(doc_d.id)
-            self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id)
-
-            doc_e.the_file.replace(f, filename='doc_e')
-            doc_e.save()
-
-            doc_f = GridDocument.objects.with_id(doc_e.id)
-            self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id)
-
-        db = GridDocument._get_db()
-        grid_fs = gridfs.GridFS(db)
-        self.assertEqual(['doc_b', 'doc_e'], grid_fs.list())
-
-    def test_file_uniqueness(self):
-        """Ensure that each instance of a FileField is unique
-        """
-        class TestFile(Document):
-            name = StringField()
-            the_file = FileField()
-
-        # First instance
-        test_file = TestFile()
-        test_file.name = "Hello, World!"
-        test_file.the_file.put(b('Hello, World!'))
-        test_file.save()
-
-        # Second instance
-        test_file_dupe = TestFile()
-        data = test_file_dupe.the_file.read() # Should be None
-
-        self.assertTrue(test_file.name != test_file_dupe.name)
-        self.assertTrue(test_file.the_file.read() != data)
-
-        TestFile.drop_collection()
-
-    def test_file_boolean(self):
-        """Ensure that a boolean test of a FileField indicates its presence
-        """
-        class TestFile(Document):
-            the_file = FileField()
-
-        test_file = TestFile()
-        self.assertFalse(bool(test_file.the_file))
-        test_file.the_file = b('Hello, World!')
-        test_file.the_file.content_type = 'text/plain'
-        test_file.save()
-        self.assertTrue(bool(test_file.the_file))
-
-        TestFile.drop_collection()
-
-    def test_file_cmp(self):
-        """Test comparing against other types"""
-        class TestFile(Document):
-            the_file = FileField()
-
-        test_file = TestFile()
-        self.assertFalse(test_file.the_file in [{"test": 1}])
-
-    def test_image_field(self):
-        if PY3:
-            raise SkipTest('PIL does not have Python 3 support')
-
-        class TestImage(Document):
-            image = ImageField()
-
-        TestImage.drop_collection()
-
-        t = TestImage()
-        t.image.put(open(TEST_IMAGE_PATH, 'rb'))
-        t.save()
-
-        t = TestImage.objects.first()
-
-        self.assertEqual(t.image.format, 'PNG')
-
-        w, h = t.image.size
-        self.assertEqual(w, 371)
-        self.assertEqual(h, 76)
-
-        t.image.delete()
-
-    def test_image_field_resize(self):
-        if PY3:
-            raise SkipTest('PIL does not have Python 3 support')
-
-        class TestImage(Document):
-            image = ImageField(size=(185, 37))
-
-        TestImage.drop_collection()
-
-        t = TestImage()
-        t.image.put(open(TEST_IMAGE_PATH, 'rb'))
-        t.save()
-
-        t = TestImage.objects.first()
-
-        self.assertEqual(t.image.format, 'PNG')
-        w, h = t.image.size
-
-        self.assertEqual(w, 185)
-        self.assertEqual(h, 37)
-
-        t.image.delete()
-
-    def test_image_field_resize_force(self):
-        if PY3:
-            raise SkipTest('PIL does not have Python 3 support')
-
-        class TestImage(Document):
-            image = ImageField(size=(185, 37, True))
-
-        TestImage.drop_collection()
-
-        t = TestImage()
-        t.image.put(open(TEST_IMAGE_PATH, 'rb'))
-        t.save()
-
-        t = TestImage.objects.first()
-
-        self.assertEqual(t.image.format, 'PNG')
-        w, h = t.image.size
-
-        self.assertEqual(w, 185)
-        self.assertEqual(h, 37)
-
-        t.image.delete()
-
-    def test_image_field_thumbnail(self):
-        if PY3:
-            raise SkipTest('PIL does not have Python 3 support')
-
-        class TestImage(Document):
-            image = ImageField(thumbnail_size=(92, 18))
-
-        TestImage.drop_collection()
-
-        t = TestImage()
-        t.image.put(open(TEST_IMAGE_PATH, 'rb'))
-        t.save()
-
-        t = TestImage.objects.first()
-
-        self.assertEqual(t.image.thumbnail.format, 'PNG')
-        self.assertEqual(t.image.thumbnail.width, 92)
-        self.assertEqual(t.image.thumbnail.height, 18)
-
-        t.image.delete()
-
-    def test_file_multidb(self):
-        register_connection('test_files', 'test_files')
-        class TestFile(Document):
-            name = StringField()
-            the_file = FileField(db_alias="test_files",
-                                 collection_name="macumba")
-
-        TestFile.drop_collection()
-
-        # delete old filesystem
-        get_db("test_files").macumba.files.drop()
-        get_db("test_files").macumba.chunks.drop()
-
-        # First instance
-        test_file = TestFile()
-        test_file.name = "Hello, World!"
-        test_file.the_file.put(b('Hello, World!'),
-                          name="hello.txt")
-        test_file.save()
-
-        data = get_db("test_files").macumba.files.find_one()
-        self.assertEqual(data.get('name'), 'hello.txt')
-
-        test_file = TestFile.objects.first()
-        self.assertEqual(test_file.the_file.read(),
-                          b('Hello, World!'))
-
     def test_geo_indexes(self):
         """Ensure that indexes are created automatically for GeoPointFields.
         """
@@ -2170,7 +1871,7 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(c['next'], 10)
 
         ids = [i.id for i in Person.objects]
-        self.assertEqual(ids, xrange(1, 11))
+        self.assertEqual(ids, range(1, 11))
 
         c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'})
         self.assertEqual(c['next'], 10)
@@ -2219,10 +1920,10 @@ class FieldTest(unittest.TestCase):
         self.assertEqual(c['next'], 10)
 
         ids = [i.id for i in Person.objects]
-        self.assertEqual(ids, xrange(1, 11))
+        self.assertEqual(ids, range(1, 11))
 
         counters = [i.counter for i in Person.objects]
-        self.assertEqual(counters, xrange(1, 11))
+        self.assertEqual(counters, range(1, 11))
 
         c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'})
         self.assertEqual(c['next'], 10)

tests/migration/__init__.py

@@ -1,6 +1,7 @@
 from convert_to_new_inheritance_model import *
 from refrencefield_dbref_to_object_id import *
 from turn_off_inheritance import *
+from uuidfield_to_binary import *
 
 if __name__ == '__main__':
     unittest.main()

tests/migration/uuidfield_to_binary.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import unittest
+import uuid
+
+from mongoengine import Document, connect
+from mongoengine.connection import get_db
+from mongoengine.fields import StringField, UUIDField, ListField
+
+__all__ = ('ConvertToBinaryUUID', )
+
+
+class ConvertToBinaryUUID(unittest.TestCase):
+
+    def setUp(self):
+        connect(db='mongoenginetest')
+        self.db = get_db()
+
+    def test_how_to_convert_to_binary_uuid_fields(self):
+        """Demonstrates migrating from 0.7 to 0.8
+        """
+
+        # 1. Old definition - using dbrefs
+        class Person(Document):
+            name = StringField()
+            uuid = UUIDField(binary=False)
+            uuids = ListField(UUIDField(binary=False))
+
+        Person.drop_collection()
+        Person(name="Wilson Jr", uuid=uuid.uuid4(),
+               uuids=[uuid.uuid4(), uuid.uuid4()]).save()
+
+        # 2. Start the migration by changing the schema
+        # Change UUIDFIeld as now binary defaults to True
+        class Person(Document):
+            name = StringField()
+            uuid = UUIDField()
+            uuids = ListField(UUIDField())
+
+        # 3. Loop all the objects and mark parent as changed
+        for p in Person.objects:
+            p._mark_as_changed('uuid')
+            p._mark_as_changed('uuids')
+            p.save()
+
+        # 4. Confirmation of the fix!
+        wilson = Person.objects(name="Wilson Jr").as_pymongo()[0]
+        self.assertTrue(isinstance(wilson['uuid'], uuid.UUID))
+        self.assertTrue(all([isinstance(u, uuid.UUID) for u in wilson['uuids']]))