Merge branch 'master' of github.com:MongoEngine/mongoengine

Sorry Jorge!

.travis.yml

@@ -26,4 +26,4 @@ notifications:
 branches:
   only:
     - master
-    - 0.7
+    - 0.8

AUTHORS

@@ -106,7 +106,7 @@ that much better:
  * Adam Reeve
  * Anthony Nemitz
  * deignacio
- * shaunduncan
+ * Shaun Duncan
  * Meir Kriheli
  * Andrey Fedoseev
  * aparajita
@@ -124,3 +124,7 @@ that much better:
  * Stefan Wójcik
  * dimonb
  * Garry Polley
+ * Adrian Scott
+ * Peter Teichman
+ * Jakub Kot
+ * Jorge Bastida

README.rst

@@ -14,7 +14,7 @@ About
 MongoEngine is a Python Object-Document Mapper for working with MongoDB.
 Documentation available at http://mongoengine-odm.rtfd.org - there is currently
 a `tutorial <http://readthedocs.org/docs/mongoengine-odm/en/latest/tutorial.html>`_, a `user guide
-<http://readthedocs.org/docs/mongoengine-odm/en/latest/userguide.html>`_ and an `API reference
+<https://mongoengine-odm.readthedocs.org/en/latest/guide/index.html>`_ and an `API reference
 <http://readthedocs.org/docs/mongoengine-odm/en/latest/apireference.html>`_.
 
 Installation

docs/changelog.rst

@@ -2,7 +2,25 @@
 Changelog
 =========
 
-Changes in 0.7.X
+Changes in 0.7.9
+================
+- Better fix handling for old style _types
+- Embedded SequenceFields follow collection naming convention
+
+Changes in 0.7.8
+================
+- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166)
+- Fix query chaining with .order_by() (MongoEngine/mongoengine#176)
+- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183)
+- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187)
+- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192)
+- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193)
+
+Changes in 0.7.7
+================
+- Fix handling for old style _types
+
+Changes in 0.7.6
 ================
 - Unicode fix for repr (MongoEngine/mongoengine#133)
 - Allow updates with match operators (MongoEngine/mongoengine#144)

mongoengine/__init__.py

@@ -12,7 +12,7 @@ from signals import *
 __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
            queryset.__all__ + signals.__all__)
 
-VERSION = (0, 7, 5)
+VERSION = (0, 7, 9)
 
 
 def get_version():

mongoengine/base.py

@@ -121,10 +121,11 @@ class ValidationError(AssertionError):
 def get_document(name):
     doc = _document_registry.get(name, None)
     if not doc:
-        # Possible old style names
-        end = ".%s" % name
+        # Possible old style name
+        single_end = name.split('.')[-1]
+        compound_end = '.%s' % single_end
         possible_match = [k for k in _document_registry.keys()
-                          if k.endswith(end)]
+                          if k.endswith(compound_end) or k == single_end]
         if len(possible_match) == 1:
             doc = _document_registry.get(possible_match.pop(), None)
     if not doc:

mongoengine/django/sessions.py

@@ -15,13 +15,23 @@ MONGOENGINE_SESSION_DB_ALIAS = getattr(
     settings, 'MONGOENGINE_SESSION_DB_ALIAS',
     DEFAULT_CONNECTION_NAME)
 
+# a setting for the name of the collection used to store sessions
+MONGOENGINE_SESSION_COLLECTION = getattr(
+    settings, 'MONGOENGINE_SESSION_COLLECTION',
+    'django_session')
+
+# a setting for whether session data is stored encoded or not
+MONGOENGINE_SESSION_DATA_ENCODE = getattr(
+    settings, 'MONGOENGINE_SESSION_DATA_ENCODE',
+    True)
 
 class MongoSession(Document):
     session_key = fields.StringField(primary_key=True, max_length=40)
-    session_data = fields.StringField()
+    session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \
+                                        else fields.DictField()
     expire_date = fields.DateTimeField()
 
-    meta = {'collection': 'django_session',
+    meta = {'collection': MONGOENGINE_SESSION_COLLECTION,
             'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
             'allow_inheritance': False}
 
@@ -34,7 +44,10 @@ class SessionStore(SessionBase):
         try:
             s = MongoSession.objects(session_key=self.session_key,
                                      expire_date__gt=datetime.now())[0]
-            return self.decode(force_unicode(s.session_data))
+            if MONGOENGINE_SESSION_DATA_ENCODE:
+                return self.decode(force_unicode(s.session_data))
+            else:
+                return s.session_data
         except (IndexError, SuspiciousOperation):
             self.create()
             return {}
@@ -57,7 +70,10 @@ class SessionStore(SessionBase):
         if self.session_key is None:
             self._session_key = self._get_new_session_key()
         s = MongoSession(session_key=self.session_key)
-        s.session_data = self.encode(self._get_session(no_load=must_create))
+        if MONGOENGINE_SESSION_DATA_ENCODE:
+            s.session_data = self.encode(self._get_session(no_load=must_create))
+        else:
+            s.session_data = self._get_session(no_load=must_create)
         s.expire_date = self.get_expiry_date()
         try:
             s.save(force_insert=must_create, safe=True)

mongoengine/fields.py

@@ -149,6 +149,7 @@ class EmailField(StringField):
     def validate(self, value):
         if not EmailField.EMAIL_REGEX.match(value):
             self.error('Invalid Mail-address: %s' % value)
+        super(EmailField, self).validate(value)
 
 
 class IntField(BaseField):
@@ -777,7 +778,7 @@ class ReferenceField(BaseField):
     def to_mongo(self, document):
         if isinstance(document, DBRef):
             if not self.dbref:
-                return DBRef.id
+                return document.id
             return document
         elif not self.dbref and isinstance(document, basestring):
             return document
@@ -1337,7 +1338,7 @@ class SequenceField(IntField):
 
     .. versionadded:: 0.5
     """
-    def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs):
+    def __init__(self, collection_name=None, db_alias=None, sequence_name=None, *args, **kwargs):
         self.collection_name = collection_name or 'mongoengine.counters'
         self.db_alias = db_alias or DEFAULT_CONNECTION_NAME
         self.sequence_name = sequence_name
@@ -1347,7 +1348,7 @@ class SequenceField(IntField):
         """
         Generate and Increment the counter
         """
-        sequence_name = self.sequence_name or self.owner_document._get_collection_name()
+        sequence_name = self.get_sequence_name()
         sequence_id = "%s.%s" % (sequence_name, self.name)
         collection = get_db(alias=self.db_alias)[self.collection_name]
         counter = collection.find_and_modify(query={"_id": sequence_id},
@@ -1356,6 +1357,16 @@ class SequenceField(IntField):
                                              upsert=True)
         return counter['next']
 
+    def get_sequence_name(self):
+        if self.sequence_name:
+            return self.sequence_name
+        owner = self.owner_document
+        if issubclass(owner, Document):
+            return owner._get_collection_name()
+        else:
+            return ''.join('_%s' % c if c.isupper() else c
+                            for c in owner._class_name).strip('_').lower()
+
     def __get__(self, instance, owner):
 
         if instance is None:

mongoengine/queryset.py

@@ -353,6 +353,8 @@ class QuerySet(object):
         self._slave_okay = False
         self._iter = False
         self._scalar = []
+        self._as_pymongo = False
+        self._as_pymongo_coerce = False
 
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -608,11 +610,13 @@ class QuerySet(object):
             if self._where_clause:
                 self._cursor_obj.where(self._where_clause)
 
-            # apply default ordering
             if self._ordering:
+                # Apply query ordering
                 self._cursor_obj.sort(self._ordering)
             elif self._document._meta['ordering']:
+                # Otherwise, apply the ordering from the document model
                 self.order_by(*self._document._meta['ordering'])
+                self._cursor_obj.sort(self._ordering)
 
             if self._limit is not None:
                 self._cursor_obj.limit(self._limit - (self._skip or 0))
@@ -927,7 +931,7 @@ class QuerySet(object):
             if not isinstance(doc, self._document):
                 msg = "Some documents inserted aren't instances of %s" % str(self._document)
                 raise OperationError(msg)
-            if doc.pk:
+            if doc.pk and not doc._created:
                 msg = "Some documents have ObjectIds use doc.update() instead"
                 raise OperationError(msg)
             raw.append(doc.to_mongo())
@@ -986,6 +990,9 @@ class QuerySet(object):
             for doc in docs:
                 doc_map[doc['_id']] = self._get_scalar(
                         self._document._from_son(doc))
+        elif self._as_pymongo:
+            for doc in docs:
+                doc_map[doc['_id']] = self._get_as_pymongo(doc)
         else:
             for doc in docs:
                 doc_map[doc['_id']] = self._document._from_son(doc)
@@ -1002,6 +1009,9 @@ class QuerySet(object):
             if self._scalar:
                 return self._get_scalar(self._document._from_son(
                         self._cursor.next()))
+            if self._as_pymongo:
+                return self._get_as_pymongo(self._cursor.next())
+
             return self._document._from_son(self._cursor.next())
         except StopIteration, e:
             self.rewind()
@@ -1184,6 +1194,8 @@ class QuerySet(object):
             if self._scalar:
                 return self._get_scalar(self._document._from_son(
                         self._cursor[key]))
+            if self._as_pymongo:
+                return self._get_as_pymongo(self._cursor.next())
             return self._document._from_son(self._cursor[key])
         raise AttributeError
 
@@ -1302,7 +1314,7 @@ class QuerySet(object):
             key_list.append((key, direction))
 
         self._ordering = key_list
-        self._cursor.sort(key_list)
+
         return self
 
     def explain(self, format=False):
@@ -1580,6 +1592,48 @@ class QuerySet(object):
 
         return tuple(data)
 
+    def _get_as_pymongo(self, row):
+        # Extract which fields paths we should follow if .fields(...) was
+        # used. If not, handle all fields.
+        if not getattr(self, '__as_pymongo_fields', None):
+            self.__as_pymongo_fields = []
+            for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']):
+                self.__as_pymongo_fields.append(field)
+                while '.' in field:
+                    field, _ = field.rsplit('.', 1)
+                    self.__as_pymongo_fields.append(field)
+
+        all_fields = not self.__as_pymongo_fields
+
+        def clean(data, path=None):
+            path = path or ''
+
+            if isinstance(data, dict):
+                new_data = {}
+                for key, value in data.iteritems():
+                    new_path = '%s.%s' % (path, key) if path else key
+                    if all_fields or new_path in self.__as_pymongo_fields:
+                        new_data[key] = clean(value, path=new_path)
+                data = new_data
+            elif isinstance(data, list):
+                data = [clean(d, path=path) for d in data]
+            else:
+                if self._as_pymongo_coerce:
+                    # If we need to coerce types, we need to determine the
+                    # type of this field and use the corresponding .to_python(...)
+                    from mongoengine.fields import EmbeddedDocumentField
+                    obj = self._document
+                    for chunk in path.split('.'):
+                        obj = getattr(obj, chunk, None)
+                        if obj is None:
+                            break
+                        elif isinstance(obj, EmbeddedDocumentField):
+                            obj = obj.document_type
+                    if obj and data is not None:
+                        data = obj.to_python(data)
+            return data
+        return clean(row)
+
     def scalar(self, *fields):
         """Instead of returning Document instances, return either a specific
         value or a tuple of values in order.
@@ -1602,6 +1656,16 @@ class QuerySet(object):
         """An alias for scalar"""
         return self.scalar(*fields)
 
+    def as_pymongo(self, coerce_types=False):
+        """Instead of returning Document instances, return raw values from
+        pymongo.
+
+        :param coerce_type: Field types (if applicable) would be use to coerce types.
+        """
+        self._as_pymongo = True
+        self._as_pymongo_coerce = coerce_types
+        return self
+
     def _sub_js_fields(self, code):
         """When fields are specified with [~fieldname] syntax, where
         *fieldname* is the Python name of a field, *fieldname* will be

python-mongoengine.spec

@@ -5,7 +5,7 @@
 %define srcname mongoengine
 
 Name:           python-%{srcname}
-Version:        0.7.5
+Version:        0.7.9
 Release:        1%{?dist}
 Summary:        A Python Document-Object Mapper for working with MongoDB
 

tests/test_all_warnings.py

@@ -53,7 +53,7 @@ class TestWarnings(unittest.TestCase):
         p2.parent = p1
         p2.save(cascade=False)
 
-        self.assertEqual(len(self.warning_list), 1)
+        self.assertTrue(len(self.warning_list) > 0)
         warning = self.warning_list[0]
         self.assertEqual(FutureWarning, warning["category"])
         self.assertTrue("ReferenceFields will default to using ObjectId"
@@ -77,6 +77,8 @@ class TestWarnings(unittest.TestCase):
         p2.save()
 
         self.assertEqual(len(self.warning_list), 1)
+        if len(self.warning_list) > 1:
+            print self.warning_list
         warning = self.warning_list[0]
         self.assertEqual(FutureWarning, warning["category"])
         self.assertTrue("Cascading saves will default to off in 0.8"

tests/test_dereference.py

@@ -42,6 +42,12 @@ class FieldTest(unittest.TestCase):
             group_obj = Group.objects.first()
             self.assertEqual(q, 1)
 
+            len(group_obj._data['members'])
+            self.assertEqual(q, 1)
+
+            len(group_obj.members)
+            self.assertEqual(q, 2)
+
             [m for m in group_obj.members]
             self.assertEqual(q, 2)
 

tests/test_document.py

@@ -15,7 +15,7 @@ from datetime import datetime
 from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest
 
 from mongoengine import *
-from mongoengine.base import NotRegistered, InvalidDocumentError
+from mongoengine.base import NotRegistered, InvalidDocumentError, get_document
 from mongoengine.queryset import InvalidQueryError
 from mongoengine.connection import get_db, get_connection
 
@@ -1336,7 +1336,6 @@ class DocumentTest(unittest.TestCase):
 
         User.drop_collection()
 
-
     def test_document_not_registered(self):
 
         class Place(Document):
@@ -1361,6 +1360,19 @@ class DocumentTest(unittest.TestCase):
             print Place.objects.all()
         self.assertRaises(NotRegistered, query_without_importing_nice_place)
 
+    def test_document_registry_regressions(self):
+
+        class Location(Document):
+            name = StringField()
+            meta = {'allow_inheritance': True}
+
+        class Area(Location):
+            location = ReferenceField('Location', dbref=True)
+
+        Location.drop_collection()
+
+        self.assertEquals(Area, get_document("Area"))
+        self.assertEquals(Area, get_document("Location.Area"))
 
     def test_creation(self):
         """Ensure that document may be created using keyword arguments.

tests/test_fields.py

@@ -1105,6 +1105,16 @@ class FieldTest(unittest.TestCase):
         p = Person.objects.get(name="Ross")
         self.assertEqual(p.parent, p1)
 
+    def test_dbref_to_mongo(self):
+        class Person(Document):
+            name = StringField()
+            parent = ReferenceField('self', dbref=False)
+
+        p1 = Person._from_son({'name': "Yakxxx",
+                               'parent': "50a234ea469ac1eda42d347d"})
+        mongoed = p1.to_mongo()
+        self.assertTrue(isinstance(mongoed['parent'], ObjectId))
+
     def test_objectid_reference_fields(self):
 
         class Person(Document):
@@ -2175,6 +2185,28 @@ class FieldTest(unittest.TestCase):
         c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'})
         self.assertEqual(c['next'], 10)
 
+    def test_embedded_sequence_field(self):
+        class Comment(EmbeddedDocument):
+            id = SequenceField()
+            content = StringField(required=True)
+
+        class Post(Document):
+            title = StringField(required=True)
+            comments = ListField(EmbeddedDocumentField(Comment))
+
+        self.db['mongoengine.counters'].drop()
+        Post.drop_collection()
+
+        Post(title="MongoEngine",
+             comments=[Comment(content="NoSQL Rocks"),
+                       Comment(content="MongoEngine Rocks")]).save()
+
+        c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'})
+        self.assertEqual(c['next'], 2)
+        post = Post.objects.first()
+        self.assertEqual(1, post.comments[0].id)
+        self.assertEqual(2, post.comments[1].id)
+
     def test_generic_embedded_document(self):
         class Car(EmbeddedDocument):
             name = StringField()
@@ -2299,6 +2331,18 @@ class FieldTest(unittest.TestCase):
         post.comments[1].content = 'here we go'
         post.validate()
 
+    def test_email_field_honors_regex(self):
+        class User(Document):
+            email = EmailField(regex=r'\w+@example.com')
+
+        # Fails regex validation
+        user = User(email='me@foo.com')
+        self.assertRaises(ValidationError, user.validate)
+
+        # Passes regex validation
+        user = User(email='me@example.com')
+        self.assertTrue(user.validate() is None)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_queryset.py

@@ -591,6 +591,10 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(OperationError, throw_operation_error)
 
+        # Test can insert new doc
+        new_post = Blog(title="code", id=ObjectId())
+        Blog.objects.insert(new_post)
+
         # test handles other classes being inserted
         def throw_operation_error_wrong_doc():
             class Author(Document):
@@ -1933,6 +1937,22 @@ class QuerySetTest(unittest.TestCase):
         ages = [p.age for p in self.Person.objects.order_by('-name')]
         self.assertEqual(ages, [30, 40, 20])
 
+    def test_order_by_chaining(self):
+        """Ensure that an order_by query chains properly and allows .only()
+        """
+        self.Person(name="User A", age=20).save()
+        self.Person(name="User B", age=40).save()
+        self.Person(name="User C", age=30).save()
+
+        only_age = self.Person.objects.order_by('-age').only('age')
+
+        names = [p.name for p in only_age]
+        ages = [p.age for p in only_age]
+
+        # The .only('age') clause should mean that all names are None
+        self.assertEqual(names, [None, None, None])
+        self.assertEqual(ages, [40, 30, 20])
+
     def test_confirm_order_by_reference_wont_work(self):
         """Ordering by reference is not possible.  Use map / reduce.. or
         denormalise"""
@@ -3691,6 +3711,38 @@ class QueryFieldListTest(unittest.TestCase):
         ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"}))
         self.assertEqual([b1], ak)
 
+    def test_as_pymongo(self):
+
+        from decimal import Decimal
+
+        class User(Document):
+            id = ObjectIdField('_id')
+            name = StringField()
+            age = IntField()
+            price = DecimalField()
+
+        User.drop_collection()
+        User(name="Bob Dole", age=89, price=Decimal('1.11')).save()
+        User(name="Barack Obama", age=51, price=Decimal('2.22')).save()
+
+        users = User.objects.only('name', 'price').as_pymongo()
+        results = list(users)
+        self.assertTrue(isinstance(results[0], dict))
+        self.assertTrue(isinstance(results[1], dict))
+        self.assertEqual(results[0]['name'], 'Bob Dole')
+        self.assertEqual(results[0]['price'], '1.11')
+        self.assertEqual(results[1]['name'], 'Barack Obama')
+        self.assertEqual(results[1]['price'], '2.22')
+
+        # Test coerce_types
+        users = User.objects.only('name', 'price').as_pymongo(coerce_types=True)
+        results = list(users)
+        self.assertTrue(isinstance(results[0], dict))
+        self.assertTrue(isinstance(results[1], dict))
+        self.assertEqual(results[0]['name'], 'Bob Dole')
+        self.assertEqual(results[0]['price'], Decimal('1.11'))
+        self.assertEqual(results[1]['name'], 'Barack Obama')
+        self.assertEqual(results[1]['price'], Decimal('2.22'))
 
 if __name__ == '__main__':
     unittest.main()