Merge branch 'fix-reloading-strict' of https://github.com/paularmand/mongoengine into fix-reloading-strict and bumped version.

# Conflicts:
#	AUTHORS

AUTHORS

@@ -230,3 +230,6 @@ that much better:
  * Amit Lichtenberg (https://github.com/amitlicht)
  * Lars Butler (https://github.com/larsbutler)
  * George Macon (https://github.com/gmacon)
+ * Ashley Whetter (https://github.com/AWhetter)
+ * Paul-Armand Verhaegen (https://github.com/paularmand)
+ * Steven Rossiter (https://github.com/BeardedSteve)

docs/changelog.rst

@@ -2,13 +2,27 @@
 Changelog
 =========
 
+Changes in 0.10.5
+=================
+- Fix for reloading of strict with special fields. #1156
+
+Changes in 0.10.4
+=================
+- SaveConditionError is now importable from the top level package. #1165
+- upsert_one method added. #1157
+
+Changes in 0.10.3
+=================
+- Fix `read_preference` (it had chaining issues with PyMongo 2.x and it didn't work at all with PyMongo 3.x) #1042
+
 Changes in 0.10.2
 =================
 - Allow shard key to point to a field in an embedded document. #551
 - Allow arbirary metadata in fields. #1129
+- ReferenceFields now support abstract document types. #837
 
 Changes in 0.10.1
-=======================
+=================
 - Fix infinite recursion with CASCADE delete rules under specific conditions. #1046
 - Fix CachedReferenceField bug when loading cached docs as DBRef but failing to save them. #1047
 - Fix ignored chained options #842

mongoengine/__init__.py

@@ -14,7 +14,7 @@ import errors
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
 
-VERSION = (0, 10, 1)
+VERSION = (0, 10, 5)
 
 
 def get_version():

mongoengine/document.py

@@ -217,7 +217,7 @@ class Document(BaseDocument):
         Returns True if the document has been updated or False if the document
         in the database doesn't match the query.
 
-        .. note:: All unsaved changes that has been made to the document are
+        .. note:: All unsaved changes that have been made to the document are
             rejected if the method returns True.
 
         :param query: the update will be performed only if the document in the
@@ -407,7 +407,7 @@ class Document(BaseDocument):
 
     def cascade_save(self, *args, **kwargs):
         """Recursively saves any references /
-           generic references on an objects"""
+           generic references on the document"""
         _refs = kwargs.get('_refs', []) or []
 
         ReferenceField = _import_class('ReferenceField')
@@ -604,11 +604,16 @@ class Document(BaseDocument):
             if not fields or field in fields:
                 try:
                     setattr(self, field, self._reload(field, obj[field]))
-                except KeyError:
-                    # If field is removed from the database while the object
-                    # is in memory, a reload would cause a KeyError
-                    # i.e. obj.update(unset__field=1) followed by obj.reload()
-                    delattr(self, field)
+                except (KeyError, AttributeError):
+                    try:
+                        # If field is a special field, e.g. items is stored as _reserved_items,
+                        # an KeyError is thrown. So try to retrieve the field from _data
+                        setattr(self, field, self._reload(field, obj._data.get(field)))
+                    except KeyError:
+                        # If field is removed from the database while the object
+                        # is in memory, a reload would cause a KeyError
+                        # i.e. obj.update(unset__field=1) followed by obj.reload()
+                        delattr(self, field)
 
         self._changed_fields = obj._changed_fields
         self._created = False

mongoengine/errors.py

@@ -6,7 +6,7 @@ from mongoengine.python_support import txt_type
 __all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError',
            'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError',
            'OperationError', 'NotUniqueError', 'FieldDoesNotExist',
-           'ValidationError')
+           'ValidationError', 'SaveConditionError')
 
 
 class NotRegistered(Exception):

mongoengine/fields.py

@@ -895,6 +895,10 @@ class ReferenceField(BaseField):
           or as the :class:`~pymongo.objectid.ObjectId`.id .
         :param reverse_delete_rule: Determines what to do when the referring
           object is deleted
+
+        .. note ::
+            A reference to an abstract document type is always stored as a
+            :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`.
         """
         if not isinstance(document_type, basestring):
             if not issubclass(document_type, (Document, basestring)):
@@ -927,9 +931,14 @@ class ReferenceField(BaseField):
         self._auto_dereference = instance._fields[self.name]._auto_dereference
         # Dereference DBRefs
         if self._auto_dereference and isinstance(value, DBRef):
-            value = self.document_type._get_db().dereference(value)
+            if hasattr(value, 'cls'):
+                # Dereference using the class type specified in the reference
+                cls = get_document(value.cls)
+            else:
+                cls = self.document_type
+            value = cls._get_db().dereference(value)
             if value is not None:
-                instance._data[self.name] = self.document_type._from_son(value)
+                instance._data[self.name] = cls._from_son(value)
 
         return super(ReferenceField, self).__get__(instance, owner)
 
@@ -939,21 +948,29 @@ class ReferenceField(BaseField):
                 return document.id
             return document
 
-        id_field_name = self.document_type._meta['id_field']
-        id_field = self.document_type._fields[id_field_name]
-
         if isinstance(document, Document):
             # We need the id from the saved object to create the DBRef
             id_ = document.pk
             if id_ is None:
                 self.error('You can only reference documents once they have'
                            ' been saved to the database')
+
+            # Use the attributes from the document instance, so that they
+            # override the attributes of this field's document type
+            cls = document
         else:
             id_ = document
+            cls = self.document_type
+
+        id_field_name = cls._meta['id_field']
+        id_field = cls._fields[id_field_name]
 
         id_ = id_field.to_mongo(id_)
-        if self.dbref:
-            collection = self.document_type._get_collection_name()
+        if self.document_type._meta.get('abstract'):
+            collection = cls._get_collection_name()
+            return DBRef(collection, id_, cls=cls._class_name)
+        elif self.dbref:
+            collection = cls._get_collection_name()
             return DBRef(collection, id_)
 
         return id_
@@ -982,6 +999,14 @@ class ReferenceField(BaseField):
             self.error('You can only reference documents once they have been '
                        'saved to the database')
 
+        if self.document_type._meta.get('abstract') and \
+                not isinstance(value, self.document_type):
+            self.error('%s is not an instance of abstract reference'
+                    ' type %s' % (value._class_name,
+                        self.document_type._class_name)
+                    )
+
+
     def lookup_member(self, member_name):
         return self.document_type._fields.get(member_name)
 

mongoengine/queryset/base.py

@@ -471,6 +471,32 @@ class BaseQuerySet(object):
                 raise OperationError(message)
             raise OperationError(u'Update failed (%s)' % unicode(err))
 
+
+    def upsert_one(self, write_concern=None, **update):
+        """Overwrite or add the first document matched by the query.
+
+        :param write_concern: Extra keyword arguments are passed down which
+            will be used as options for the resultant
+            ``getLastError`` command.  For example,
+            ``save(..., write_concern={w: 2, fsync: True}, ...)`` will
+            wait until at least two servers have recorded the write and
+            will force an fsync on the primary server.
+        :param update: Django-style update keyword arguments
+
+        :returns the new or overwritten document
+
+        .. versionadded:: 0.10.2
+        """
+
+        atomic_update = self.update(multi=False, upsert=True, write_concern=write_concern,
+                             full_result=True,**update)
+
+        if atomic_update['updatedExisting']:
+            document = self.get()
+        else:
+            document = self._document.objects.with_id(atomic_update['upserted'])
+        return document
+
     def update_one(self, upsert=False, write_concern=None, **update):
         """Perform an atomic update on the fields of the first document
         matched by the query.
@@ -930,6 +956,7 @@ class BaseQuerySet(object):
         validate_read_preference('read_preference', read_preference)
         queryset = self.clone()
         queryset._read_preference = read_preference
+        queryset._cursor_obj = None  # we need to re-create the cursor object whenever we apply read_preference
         return queryset
 
     def scalar(self, *fields):
@@ -1443,8 +1470,16 @@ class BaseQuerySet(object):
     def _cursor(self):
         if self._cursor_obj is None:
 
-            self._cursor_obj = self._collection.find(self._query,
-                                                     **self._cursor_args)
+            # In PyMongo 3+, we define the read preference on a collection
+            # level, not a cursor level. Thus, we need to get a cloned
+            # collection object using `with_options` first.
+            if IS_PYMONGO_3 and self._read_preference is not None:
+                self._cursor_obj = self._collection\
+                    .with_options(read_preference=self._read_preference)\
+                    .find(self._query, **self._cursor_args)
+            else:
+                self._cursor_obj = self._collection.find(self._query,
+                                                         **self._cursor_args)
             # Apply where clauses to cursor
             if self._where_clause:
                 where_clause = self._sub_js_fields(self._where_clause)

tests/document/instance.py

@@ -571,6 +571,28 @@ class InstanceTest(unittest.TestCase):
         except Exception:
             self.assertFalse("Threw wrong exception")
 
+    def test_reload_of_non_strict_with_special_field_name(self):
+        """Ensures reloading works for documents with meta strict == False
+        """
+        class Post(Document):
+            meta = {
+                'strict': False
+            }
+            title = StringField()
+            items = ListField()
+
+        Post.drop_collection()
+
+        Post._get_collection().insert_one({
+            "title": "Items eclipse",
+            "items": ["more lorem", "even more ipsum"]
+        })
+
+        post = Post.objects.first()
+        post.reload()
+        self.assertEqual(post.title, "Items eclipse")
+        self.assertEqual(post.items, ["more lorem", "even more ipsum"])
+
     def test_dictionary_access(self):
         """Ensure that dictionary-style field access works properly.
         """

tests/fields/fields.py

@@ -2281,6 +2281,81 @@ class FieldTest(unittest.TestCase):
         Member.drop_collection()
         BlogPost.drop_collection()
 
+    def test_reference_class_with_abstract_parent(self):
+        """Ensure that a class with an abstract parent can be referenced.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Sister(Sibling):
+            pass
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+        sister = Sister(name="Alice")
+        sister.save()
+        brother = Brother(name="Bob", sibling=sister)
+        brother.save()
+
+        self.assertEquals(Brother.objects[0].sibling.name, sister.name)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+    def test_reference_abstract_class(self):
+        """Ensure that an abstract class instance cannot be used in the
+        reference of that abstract class.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Sister(Sibling):
+            pass
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+        sister = Sibling(name="Alice")
+        brother = Brother(name="Bob", sibling=sister)
+        self.assertRaises(ValidationError, brother.save)
+
+        Sister.drop_collection()
+        Brother.drop_collection()
+
+    def test_abstract_reference_base_type(self):
+        """Ensure that an an abstract reference fails validation when given a
+        Document that does not inherit from the abstract type.
+        """
+        class Sibling(Document):
+            name = StringField()
+            meta = {"abstract": True}
+
+        class Brother(Sibling):
+            sibling = ReferenceField(Sibling)
+
+        class Mother(Document):
+            name = StringField()
+
+        Brother.drop_collection()
+        Mother.drop_collection()
+
+        mother = Mother(name="Carol")
+        mother.save()
+        brother = Brother(name="Bob", sibling=mother)
+        self.assertRaises(ValidationError, brother.save)
+
+        Brother.drop_collection()
+        Mother.drop_collection()
+
     def test_generic_reference(self):
         """Ensure that a GenericReferenceField properly dereferences items.
         """

tests/queryset/queryset.py

@@ -680,12 +680,21 @@ class QuerySetTest(unittest.TestCase):
     def test_upsert_one(self):
         self.Person.drop_collection()
 
-        self.Person.objects(name="Bob", age=30).update_one(upsert=True)
+        bob = self.Person.objects(name="Bob", age=30).upsert_one()
 
-        bob = self.Person.objects.first()
         self.assertEqual("Bob", bob.name)
         self.assertEqual(30, bob.age)
 
+        bob.name = "Bobby"
+        bob.save()
+
+        bobby = self.Person.objects(name="Bobby", age=30).upsert_one()
+
+        self.assertEqual("Bobby", bobby.name)
+        self.assertEqual(30, bobby.age)
+        self.assertEqual(bob.id, bobby.id)
+
+
     def test_set_on_insert(self):
         self.Person.drop_collection()
 
@@ -4165,7 +4174,11 @@ class QuerySetTest(unittest.TestCase):
 
     def test_read_preference(self):
         class Bar(Document):
-            pass
+            txt = StringField()
+
+            meta = {
+                'indexes': [ 'txt' ]
+            }
 
         Bar.drop_collection()
         bars = list(Bar.objects(read_preference=ReadPreference.PRIMARY))
@@ -4177,9 +4190,51 @@ class QuerySetTest(unittest.TestCase):
             error_class = TypeError
         self.assertRaises(error_class, Bar.objects, read_preference='Primary')
 
+        # read_preference as a kwarg
         bars = Bar.objects(read_preference=ReadPreference.SECONDARY_PREFERRED)
         self.assertEqual(
             bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
+
+        # read_preference as a query set method
+        bars = Bar.objects.read_preference(ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(
+            bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
+
+        # read_preference after skip
+        bars = Bar.objects.skip(1) \
+            .read_preference(ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(
+            bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
+
+        # read_preference after limit
+        bars = Bar.objects.limit(1) \
+            .read_preference(ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(
+            bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
+
+        # read_preference after order_by
+        bars = Bar.objects.order_by('txt') \
+            .read_preference(ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(
+            bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
+
+        # read_preference after hint
+        bars = Bar.objects.hint([('txt', 1)]) \
+            .read_preference(ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(
+            bars._read_preference, ReadPreference.SECONDARY_PREFERRED)
+        self.assertEqual(bars._cursor._Cursor__read_preference,
+            ReadPreference.SECONDARY_PREFERRED)
 
     def test_json_simple(self):