Inital tests for dereferencing improvements

mongoengine/base.py

@@ -126,6 +126,7 @@ class BaseField(object):
 
         self.validate(value)
 
+
 class ObjectIdField(BaseField):
     """An field wrapper around MongoDB's ObjectIds.
     """

mongoengine/fields.py

@@ -337,33 +337,54 @@ class ListField(BaseField):
             # Document class being used rather than a document object
             return self
 
-        if isinstance(self.field, ReferenceField):
+        # Get value from document instance if available
-            referenced_type = self.field.document_type
+        value_list = instance._data.get(self.name)
-            # Get value from document instance if available
+        if isinstance(self.field, ReferenceField) and value_list:
-            value_list = instance._data.get(self.name)
+            db = _get_db()
-            if value_list:
+            value_list = [(k,v) for k,v in enumerate(value_list)]
-                deref_list = []
+            deref_list = []
-                for value in value_list:
+            collections = {}
-                    # Dereference DBRefs
-                    if isinstance(value, (pymongo.dbref.DBRef)):
-                        value = _get_db().dereference(value)
-                        deref_list.append(referenced_type._from_son(value))
-                    else:
-                        deref_list.append(value)
-                instance._data[self.name] = deref_list
 
-        if isinstance(self.field, GenericReferenceField):
+            for k, v in value_list:
-            value_list = instance._data.get(self.name)
+                deref_list.append(v)
-            if value_list:
+                # Save any DBRefs
-                deref_list = []
+                if isinstance(v, (pymongo.dbref.DBRef)):
-                for value in value_list:
+                    collections.setdefault(v.collection, []).append((k, v))
-                    # Dereference DBRefs
-                    if isinstance(value, (dict, pymongo.son.SON)):
-                        deref_list.append(self.field.dereference(value))
-                    else:
-                        deref_list.append(value)
-                instance._data[self.name] = deref_list
 
+            # For each collection get the references
+            for collection, dbrefs in collections.items():
+                id_map = dict([(v.id, k) for k, v in dbrefs])
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    deref_list[key] = get_document(ref['_cls'])._from_son(ref)
+            instance._data[self.name] = deref_list
+
+        # Get value from document instance if available
+        if isinstance(self.field, GenericReferenceField) and value_list:
+
+            db = _get_db()
+            value_list = [(k,v) for k,v in enumerate(value_list)]
+            deref_list = []
+            classes = {}
+
+            for k, v in value_list:
+                deref_list.append(v)
+                # Save any DBRefs
+                if isinstance(v, (dict, pymongo.son.SON)):
+                    classes.setdefault(v['_cls'], []).append((k, v))
+
+            # For each collection get the references
+            for doc_cls, dbrefs in classes.items():
+                id_map = dict([(v['_ref'].id, k) for k, v in dbrefs])
+                doc_cls = get_document(doc_cls)
+                collection = doc_cls._meta['collection']
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
+
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    deref_list[key] = doc_cls._from_son(ref)
+            instance._data[self.name] = deref_list
         return super(ListField, self).__get__(instance, owner)
 
     def to_python(self, value):
@@ -501,32 +522,53 @@ class MapField(BaseField):
             # Document class being used rather than a document object
             return self
 
-        if isinstance(self.field, ReferenceField):
+        # Get value from document instance if available
-            referenced_type = self.field.document_type
+        value_list = instance._data.get(self.name)
-            # Get value from document instance if available
+        if isinstance(self.field, ReferenceField) and value_list:
-            value_dict = instance._data.get(self.name)
+            db = _get_db()
-            if value_dict:
+            deref_dict = {}
-                deref_dict = []
+            collections = {}
-                for key,value in value_dict.iteritems():
-                    # Dereference DBRefs
-                    if isinstance(value, (pymongo.dbref.DBRef)):
-                        value = _get_db().dereference(value)
-                        deref_dict[key] = referenced_type._from_son(value)
-                    else:
-                        deref_dict[key] = value
-                instance._data[self.name] = deref_dict
 
-        if isinstance(self.field, GenericReferenceField):
+            for k, v in value_list.items():
-            value_dict = instance._data.get(self.name)
+                deref_dict[k] = v
-            if value_dict:
+                # Save any DBRefs
-                deref_dict = []
+                if isinstance(v, (pymongo.dbref.DBRef)):
-                for key,value in value_dict.iteritems():
+                    collections.setdefault(v.collection, []).append((k, v))
-                    # Dereference DBRefs
+
-                    if isinstance(value, (dict, pymongo.son.SON)):
+            # For each collection get the references
-                        deref_dict[key] = self.field.dereference(value)
+            for collection, dbrefs in collections.items():
-                    else:
+                id_map = dict([(v.id, k) for k, v in dbrefs])
-                        deref_dict[key] = value
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
-                instance._data[self.name] = deref_dict
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    deref_dict[key] = get_document(ref['_cls'])._from_son(ref)
+            instance._data[self.name] = deref_dict
+
+        # Get value from document instance if available
+        if isinstance(self.field, GenericReferenceField) and value_list:
+
+            db = _get_db()
+            value_list = [(k,v) for k,v in value_list.items()]
+            deref_dict = {}
+            classes = {}
+
+            for k, v in value_list:
+                deref_dict[k] = v
+                # Save any DBRefs
+                if isinstance(v, (dict, pymongo.son.SON)):
+                    classes.setdefault(v['_cls'], []).append((k, v))
+
+            # For each collection get the references
+            for doc_cls, dbrefs in classes.items():
+                id_map = dict([(v['_ref'].id, k) for k, v in dbrefs])
+                doc_cls = get_document(doc_cls)
+                collection = doc_cls._meta['collection']
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
+
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    deref_dict[key] = doc_cls._from_son(ref)
+            instance._data[self.name] = deref_dict
 
         return super(MapField, self).__get__(instance, owner)
 
@@ -869,3 +911,76 @@ class GeoPointField(BaseField):
         if (not isinstance(value[0], (float, int)) and
             not isinstance(value[1], (float, int))):
             raise ValidationError('Both values in point must be float or int.')
+
+
+
+class DereferenceMixin(object):
+    """ WORK IN PROGRESS"""
+
+    def __get__(self, instance, owner):
+        """Descriptor to automatically dereference references.
+        """
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        # Get value from document instance if available
+        value_list = instance._data.get(self.name)
+        if not value_list:
+            return super(MapField, self).__get__(instance, owner)
+
+        is_dict = True
+        if not hasattr(value_list, 'items'):
+            is_dict = False
+            value_list = dict([(k,v) for k,v in enumerate(value_list)])
+
+        if isinstance(self.field, ReferenceField) and value_list:
+            db = _get_db()
+            dbref = {}
+            if not is_dict:
+                dbref = []
+            collections = {}
+
+            for k, v in value_list.items():
+                dbref[k] = v
+                # Save any DBRefs
+                if isinstance(v, (pymongo.dbref.DBRef)):
+                    collections.setdefault(v.collection, []).append((k, v))
+
+            # For each collection get the references
+            for collection, dbrefs in collections.items():
+                id_map = dict([(v.id, k) for k, v in dbrefs])
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    dbref[key] = get_document(ref['_cls'])._from_son(ref)
+
+            instance._data[self.name] = dbref
+
+        # Get value from document instance if available
+        if isinstance(self.field, GenericReferenceField) and value_list:
+
+            db = _get_db()
+            value_list = [(k,v) for k,v in value_list.items()]
+            dbref = {}
+            classes = {}
+
+            for k, v in value_list:
+                dbref[k] = v
+                # Save any DBRefs
+                if isinstance(v, (dict, pymongo.son.SON)):
+                    classes.setdefault(v['_cls'], []).append((k, v))
+
+            # For each collection get the references
+            for doc_cls, dbrefs in classes.items():
+                id_map = dict([(v['_ref'].id, k) for k, v in dbrefs])
+                doc_cls = get_document(doc_cls)
+                collection = doc_cls._meta['collection']
+                references = db[collection].find({'_id': {'$in': id_map.keys()}})
+
+                for ref in references:
+                    key = id_map[ref['_id']]
+                    dbref[key] = doc_cls._from_son(ref)
+            instance._data[self.name] = dbref
+
+        return super(DereferenceField, self).__get__(instance, owner)

mongoengine/tests.py

@@ -0,0 +1,58 @@
+from mongoengine.connection import _get_db
+
+class query_counter(object):
+    """ Query_counter contextmanager to get the number of queries. """
+
+    def __init__(self):
+        """ Construct the query_counter. """
+        self.counter = 0
+        self.db = _get_db()
+
+    def __enter__(self):
+        """ On every with block we need to drop the profile collection. """
+        self.db.set_profiling_level(0)
+        self.db.system.profile.drop()
+        self.db.set_profiling_level(2)
+        return self
+
+    def __exit__(self, t, value, traceback):
+        """ Reset the profiling level. """
+        self.db.set_profiling_level(0)
+
+    def __eq__(self, value):
+        """ == Compare querycounter. """
+        return value == self._get_count()
+
+    def __ne__(self, value):
+        """ != Compare querycounter. """
+        return not self.__eq__(value)
+
+    def __lt__(self, value):
+        """ < Compare querycounter. """
+        return self._get_count() < value
+
+    def __le__(self, value):
+        """ <= Compare querycounter. """
+        return self._get_count() <= value
+
+    def __gt__(self, value):
+        """ > Compare querycounter. """
+        return self._get_count() > value
+
+    def __ge__(self, value):
+        """ >= Compare querycounter. """
+        return self._get_count() >= value
+
+    def __int__(self):
+        """ int representation. """
+        return self._get_count()
+
+    def __repr__(self):
+        """ repr query_counter as the number of queries. """
+        return u"%s" % self._get_count()
+
+    def _get_count(self):
+        """ Get the number of queries. """
+        count = self.db.system.profile.find().count() - self.counter
+        self.counter += 1
+        return count

tests/dereference.py

@@ -0,0 +1,288 @@
+import unittest
+
+from mongoengine import *
+from mongoengine.connection import _get_db
+from mongoengine.tests import query_counter
+
+
+class FieldTest(unittest.TestCase):
+
+    def setUp(self):
+        connect(db='mongoenginetest')
+        self.db = _get_db()
+
+    def ztest_list_item_dereference(self):
+        """Ensure that DBRef items in ListFields are dereferenced.
+        """
+        class User(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = ListField(ReferenceField(User))
+
+        User.drop_collection()
+        Group.drop_collection()
+
+        for i in xrange(1, 51):
+            user = User(name='user %s' % i)
+            user.save()
+
+        group = Group(members=User.objects)
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 2)
+
+        User.drop_collection()
+        Group.drop_collection()
+
+    def ztest_recursive_reference(self):
+        """Ensure that ReferenceFields can reference their own documents.
+        """
+        class Employee(Document):
+            name = StringField()
+            boss = ReferenceField('self')
+            friends = ListField(ReferenceField('self'))
+
+        bill = Employee(name='Bill Lumbergh')
+        bill.save()
+
+        michael = Employee(name='Michael Bolton')
+        michael.save()
+
+        samir = Employee(name='Samir Nagheenanajar')
+        samir.save()
+
+        friends = [michael, samir]
+        peter = Employee(name='Peter Gibbons', boss=bill, friends=friends)
+        peter.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            peter = Employee.objects.with_id(peter.id)
+            self.assertEqual(q, 1)
+
+            peter.boss
+            self.assertEqual(q, 2)
+
+            peter.friends
+            self.assertEqual(q, 3)
+
+    def ztest_generic_reference(self):
+
+        class UserA(Document):
+            name = StringField()
+
+        class UserB(Document):
+            name = StringField()
+
+        class UserC(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = ListField(GenericReferenceField())
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()
+
+        members = []
+        for i in xrange(1, 51):
+            a = UserA(name='User A %s' % i)
+            a.save()
+
+            b = UserB(name='User B %s' % i)
+            b.save()
+
+            c = UserC(name='User C %s' % i)
+            c.save()
+
+            members += [a, b, c]
+
+        group = Group(members=members)
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()
+
+    def test_map_field_reference(self):
+
+        class User(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = MapField(ReferenceField(User))
+
+        User.drop_collection()
+        Group.drop_collection()
+
+        members = []
+        for i in xrange(1, 51):
+            user = User(name='user %s' % i)
+            user.save()
+            members.append(user)
+
+        group = Group(members=dict([(str(u.id), u) for u in members]))
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 2)
+
+        User.drop_collection()
+        Group.drop_collection()
+
+    def ztest_generic_reference_dict_field(self):
+
+        class UserA(Document):
+            name = StringField()
+
+        class UserB(Document):
+            name = StringField()
+
+        class UserC(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = DictField()
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()
+
+        members = []
+        for i in xrange(1, 51):
+            a = UserA(name='User A %s' % i)
+            a.save()
+
+            b = UserB(name='User B %s' % i)
+            b.save()
+
+            c = UserC(name='User C %s' % i)
+            c.save()
+
+            members += [a, b, c]
+
+        group = Group(members=dict([(str(u.id), u) for u in members]))
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+        group.members = {}
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 1)
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()
+
+    def test_generic_reference_map_field(self):
+
+        class UserA(Document):
+            name = StringField()
+
+        class UserB(Document):
+            name = StringField()
+
+        class UserC(Document):
+            name = StringField()
+
+        class Group(Document):
+            members = MapField(GenericReferenceField())
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()
+
+        members = []
+        for i in xrange(1, 51):
+            a = UserA(name='User A %s' % i)
+            a.save()
+
+            b = UserB(name='User B %s' % i)
+            b.save()
+
+            c = UserC(name='User C %s' % i)
+            c.save()
+
+            members += [a, b, c]
+
+        group = Group(members=dict([(str(u.id), u) for u in members]))
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 4)
+
+        group.members = {}
+        group.save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+
+            group_obj = Group.objects.first()
+            self.assertEqual(q, 1)
+
+            [m for m in group_obj.members]
+            self.assertEqual(q, 1)
+
+        UserA.drop_collection()
+        UserB.drop_collection()
+        UserC.drop_collection()
+        Group.drop_collection()