diff --git a/AUTHORS b/AUTHORS index 5801981a..23afe6f6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -106,7 +106,7 @@ that much better: * Adam Reeve * Anthony Nemitz * deignacio - * shaunduncan + * Shaun Duncan * Meir Kriheli * Andrey Fedoseev * aparajita @@ -125,3 +125,7 @@ that much better: * dimonb * Garry Polley * James Slagle + * Adrian Scott + * Peter Teichman + * Jakub Kot + * Jorge Bastida diff --git a/README.rst b/README.rst index ae6bd0ec..5eab5021 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ About MongoEngine is a Python Object-Document Mapper for working with MongoDB. Documentation available at http://mongoengine-odm.rtfd.org - there is currently a `tutorial `_, a `user guide -`_ and an `API reference +`_ and an `API reference `_. Installation diff --git a/docs/changelog.rst b/docs/changelog.rst index a56f33ac..109940ea 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,20 @@ Changes in 0.8 - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) +Changes in 0.7.9 +================ +- Better fix handling for old style _types +- Embedded SequenceFields follow collection naming convention + +Changes in 0.7.8 +================ +- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166) +- Fix query chaining with .order_by() (MongoEngine/mongoengine#176) +- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) +- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) +- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) +- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) + Changes in 0.7.7 ================ - Fix handling for old style _types diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py index dc43d405..3a966c79 100644 --- a/mongoengine/base/common.py +++ b/mongoengine/base/common.py @@ -9,10 +9,12 @@ _document_registry = {} def get_document(name): doc = _document_registry.get(name, None) - if not doc and '.' in name: + if not doc: # Possible old style name - end = name.split('.')[-1] - possible_match = [k for k in _document_registry.keys() if k == end] + single_end = name.split('.')[-1] + compound_end = '.%s' % single_end + possible_match = [k for k in _document_registry.keys() + if k.endswith(compound_end) or k == single_end] if len(possible_match) == 1: doc = _document_registry.get(possible_match.pop(), None) if not doc: diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index f1783429..810b6265 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -15,13 +15,23 @@ MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', DEFAULT_CONNECTION_NAME) +# a setting for the name of the collection used to store sessions +MONGOENGINE_SESSION_COLLECTION = getattr( + settings, 'MONGOENGINE_SESSION_COLLECTION', + 'django_session') + +# a setting for whether session data is stored encoded or not +MONGOENGINE_SESSION_DATA_ENCODE = getattr( + settings, 'MONGOENGINE_SESSION_DATA_ENCODE', + True) class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) - session_data = fields.StringField() + session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \ + else fields.DictField() expire_date = fields.DateTimeField() - meta = {'collection': 'django_session', + meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} @@ -34,7 +44,10 @@ class SessionStore(SessionBase): try: s = MongoSession.objects(session_key=self.session_key, expire_date__gt=datetime.now())[0] - return self.decode(force_unicode(s.session_data)) + if MONGOENGINE_SESSION_DATA_ENCODE: + return self.decode(force_unicode(s.session_data)) + else: + return s.session_data except (IndexError, SuspiciousOperation): self.create() return {} @@ -57,7 +70,10 @@ class SessionStore(SessionBase): if self.session_key is None: self._session_key = self._get_new_session_key() s = MongoSession(session_key=self.session_key) - s.session_data = self.encode(self._get_session(no_load=must_create)) + if MONGOENGINE_SESSION_DATA_ENCODE: + s.session_data = self.encode(self._get_session(no_load=must_create)) + else: + s.session_data = self._get_session(no_load=must_create) s.expire_date = self.get_expiry_date() try: s.save(force_insert=must_create, safe=True) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e2ce33cd..65996a4b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -149,6 +149,7 @@ class EmailField(StringField): def validate(self, value): if not EmailField.EMAIL_REGEX.match(value): self.error('Invalid Mail-address: %s' % value) + super(EmailField, self).validate(value) class IntField(BaseField): @@ -782,7 +783,7 @@ class ReferenceField(BaseField): def to_mongo(self, document): if isinstance(document, DBRef): if not self.dbref: - return DBRef.id + return document.id return document elif not self.dbref and isinstance(document, basestring): return document @@ -1377,6 +1378,16 @@ class SequenceField(BaseField): upsert=True) return self.value_decorator(counter['next']) + def get_sequence_name(self): + if self.sequence_name: + return self.sequence_name + owner = self.owner_document + if issubclass(owner, Document): + return owner._get_collection_name() + else: + return ''.join('_%s' % c if c.isupper() else c + for c in owner._class_name).strip('_').lower() + def __get__(self, instance, owner): value = super(SequenceField, self).__get__(instance, owner) if value is None and instance._initialised: diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 3acee36e..58ebb568 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -58,6 +58,8 @@ class QuerySet(object): self._read_preference = None self._iter = False self._scalar = [] + self._as_pymongo = False + self._as_pymongo_coerce = False # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -178,11 +180,13 @@ class QuerySet(object): if self._where_clause: self._cursor_obj.where(self._where_clause) - # apply default ordering if self._ordering: + # Apply query ordering self._cursor_obj.sort(self._ordering) elif self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model self.order_by(*self._document._meta['ordering']) + self._cursor_obj.sort(self._ordering) if self._limit is not None: self._cursor_obj.limit(self._limit - (self._skip or 0)) @@ -328,7 +332,7 @@ class QuerySet(object): msg = ("Some documents inserted aren't instances of %s" % str(self._document)) raise OperationError(msg) - if doc.pk: + if doc.pk and not doc._created: msg = "Some documents have ObjectIds use doc.update() instead" raise OperationError(msg) raw.append(doc.to_mongo()) @@ -388,6 +392,9 @@ class QuerySet(object): for doc in docs: doc_map[doc['_id']] = self._get_scalar( self._document._from_son(doc)) + elif self._as_pymongo: + for doc in docs: + doc_map[doc['_id']] = self._get_as_pymongo(doc) else: for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) @@ -404,6 +411,9 @@ class QuerySet(object): if self._scalar: return self._get_scalar(self._document._from_son( self._cursor.next())) + if self._as_pymongo: + return self._get_as_pymongo(self._cursor.next()) + return self._document._from_son(self._cursor.next()) except StopIteration, e: self.rewind() @@ -592,6 +602,8 @@ class QuerySet(object): if self._scalar: return self._get_scalar(self._document._from_son( self._cursor[key])) + if self._as_pymongo: + return self._get_as_pymongo(self._cursor.next()) return self._document._from_son(self._cursor[key]) raise AttributeError @@ -714,7 +726,7 @@ class QuerySet(object): key_list.append((key, direction)) self._ordering = key_list - self._cursor.sort(key_list) + return self def explain(self, format=False): @@ -887,6 +899,48 @@ class QuerySet(object): return tuple(data) + def _get_as_pymongo(self, row): + # Extract which fields paths we should follow if .fields(...) was + # used. If not, handle all fields. + if not getattr(self, '__as_pymongo_fields', None): + self.__as_pymongo_fields = [] + for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']): + self.__as_pymongo_fields.append(field) + while '.' in field: + field, _ = field.rsplit('.', 1) + self.__as_pymongo_fields.append(field) + + all_fields = not self.__as_pymongo_fields + + def clean(data, path=None): + path = path or '' + + if isinstance(data, dict): + new_data = {} + for key, value in data.iteritems(): + new_path = '%s.%s' % (path, key) if path else key + if all_fields or new_path in self.__as_pymongo_fields: + new_data[key] = clean(value, path=new_path) + data = new_data + elif isinstance(data, list): + data = [clean(d, path=path) for d in data] + else: + if self._as_pymongo_coerce: + # If we need to coerce types, we need to determine the + # type of this field and use the corresponding .to_python(...) + from mongoengine.fields import EmbeddedDocumentField + obj = self._document + for chunk in path.split('.'): + obj = getattr(obj, chunk, None) + if obj is None: + break + elif isinstance(obj, EmbeddedDocumentField): + obj = obj.document_type + if obj and data is not None: + data = obj.to_python(data) + return data + return clean(row) + def scalar(self, *fields): """Instead of returning Document instances, return either a specific value or a tuple of values in order. @@ -909,6 +963,16 @@ class QuerySet(object): """An alias for scalar""" return self.scalar(*fields) + def as_pymongo(self, coerce_types=False): + """Instead of returning Document instances, return raw values from + pymongo. + + :param coerce_type: Field types (if applicable) would be use to coerce types. + """ + self._as_pymongo = True + self._as_pymongo_coerce = coerce_types + return self + def _sub_js_fields(self, code): """When fields are specified with [~fieldname] syntax, where *fieldname* is the Python name of a field, *fieldname* will be diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 9a376ec7..b1ec3361 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.7 +Version: 0.7.9 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/document/instance.py b/tests/document/instance.py index de677e21..5e29dc31 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -17,6 +17,7 @@ from mongoengine.errors import (NotRegistered, InvalidDocumentError, InvalidQueryError) from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db +from mongoengine.base import get_document TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -281,7 +282,6 @@ class InstanceTest(unittest.TestCase): User.drop_collection() - def test_document_not_registered(self): class Place(Document): @@ -306,6 +306,19 @@ class InstanceTest(unittest.TestCase): print Place.objects.all() self.assertRaises(NotRegistered, query_without_importing_nice_place) + def test_document_registry_regressions(self): + + class Location(Document): + name = StringField() + meta = {'allow_inheritance': True} + + class Area(Location): + location = ReferenceField('Location', dbref=True) + + Location.drop_collection() + + self.assertEquals(Area, get_document("Area")) + self.assertEquals(Area, get_document("Location.Area")) def test_creation(self): """Ensure that document may be created using keyword arguments. diff --git a/tests/test_fields.py b/tests/test_fields.py index 69cce871..6e3dc8b8 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1118,6 +1118,16 @@ class FieldTest(unittest.TestCase): p = Person.objects.get(name="Ross") self.assertEqual(p.parent, p1) + def test_dbref_to_mongo(self): + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=False) + + p1 = Person._from_son({'name': "Yakxxx", + 'parent': "50a234ea469ac1eda42d347d"}) + mongoed = p1.to_mongo() + self.assertTrue(isinstance(mongoed['parent'], ObjectId)) + def test_objectid_reference_fields(self): class Person(Document): @@ -2216,6 +2226,29 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + def test_embedded_sequence_field(self): + class Comment(EmbeddedDocument): + id = SequenceField() + content = StringField(required=True) + + class Post(Document): + title = StringField(required=True) + comments = ListField(EmbeddedDocumentField(Comment)) + + self.db['mongoengine.counters'].drop() + Post.drop_collection() + + Post(title="MongoEngine", + comments=[Comment(content="NoSQL Rocks"), + Comment(content="MongoEngine Rocks")]).save() + import ipdb; ipdb.set_trace(); + c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'}) + self.assertEqual(c['next'], 2) + post = Post.objects.first() + self.assertEqual(1, post.comments[0].id) + self.assertEqual(2, post.comments[1].id) + + def test_generic_embedded_document(self): class Car(EmbeddedDocument): name = StringField() @@ -2339,6 +2372,18 @@ class FieldTest(unittest.TestCase): post.comments[1].content = 'here we go' post.validate() + def test_email_field_honors_regex(self): + class User(Document): + email = EmailField(regex=r'\w+@example.com') + + # Fails regex validation + user = User(email='me@foo.com') + self.assertRaises(ValidationError, user.validate) + + # Passes regex validation + user = User(email='me@example.com') + self.assertTrue(user.validate() is None) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index a86920e9..f73cde22 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -625,6 +625,10 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, throw_operation_error) + # Test can insert new doc + new_post = Blog(title="code", id=ObjectId()) + Blog.objects.insert(new_post) + # test handles other classes being inserted def throw_operation_error_wrong_doc(): class Author(Document): @@ -1967,6 +1971,22 @@ class QuerySetTest(unittest.TestCase): ages = [p.age for p in self.Person.objects.order_by('-name')] self.assertEqual(ages, [30, 40, 20]) + def test_order_by_chaining(self): + """Ensure that an order_by query chains properly and allows .only() + """ + self.Person(name="User A", age=20).save() + self.Person(name="User B", age=40).save() + self.Person(name="User C", age=30).save() + + only_age = self.Person.objects.order_by('-age').only('age') + + names = [p.name for p in only_age] + ages = [p.age for p in only_age] + + # The .only('age') clause should mean that all names are None + self.assertEqual(names, [None, None, None]) + self.assertEqual(ages, [40, 30, 20]) + def test_confirm_order_by_reference_wont_work(self): """Ordering by reference is not possible. Use map / reduce.. or denormalise""" @@ -3761,5 +3781,38 @@ class QueryFieldListTest(unittest.TestCase): Test.objects(test='foo').update_one(upsert=True, set__test='foo') self.assertTrue('_cls' in Test._collection.find_one()) + def test_as_pymongo(self): + + from decimal import Decimal + + class User(Document): + id = ObjectIdField('_id') + name = StringField() + age = IntField() + price = DecimalField() + + User.drop_collection() + User(name="Bob Dole", age=89, price=Decimal('1.11')).save() + User(name="Barack Obama", age=51, price=Decimal('2.22')).save() + + users = User.objects.only('name', 'price').as_pymongo() + results = list(users) + self.assertTrue(isinstance(results[0], dict)) + self.assertTrue(isinstance(results[1], dict)) + self.assertEqual(results[0]['name'], 'Bob Dole') + self.assertEqual(results[0]['price'], '1.11') + self.assertEqual(results[1]['name'], 'Barack Obama') + self.assertEqual(results[1]['price'], '2.22') + + # Test coerce_types + users = User.objects.only('name', 'price').as_pymongo(coerce_types=True) + results = list(users) + self.assertTrue(isinstance(results[0], dict)) + self.assertTrue(isinstance(results[1], dict)) + self.assertEqual(results[0]['name'], 'Bob Dole') + self.assertEqual(results[0]['price'], Decimal('1.11')) + self.assertEqual(results[1]['name'], 'Barack Obama') + self.assertEqual(results[1]['price'], Decimal('2.22')) + if __name__ == '__main__': unittest.main()