From c38faebc25a4a4bfa749d97005a43e6bbe6c0be3 Mon Sep 17 00:00:00 2001 From: James Punteney Date: Sat, 16 Jan 2010 13:21:16 -0500 Subject: [PATCH 01/21] Adding a get method to the queryset that raises exceptions if more or less than one item is returned --- mongoengine/queryset.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 06c82cf9..7d89a447 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -18,6 +18,11 @@ class InvalidQueryError(Exception): class OperationError(Exception): pass +class MultipleObjectsReturned(Exception): + pass + +class DoesNotExist(Exception): + pass class Q(object): @@ -290,6 +295,20 @@ class QuerySet(object): return mongo_query + def get(self): + """Retrieve the the matching object raising + 'MultipleObjectsReturned' or 'DoesNotExist' exceptions + if multiple or no results are found. + """ + count = self.count() + if count == 1: + return self[0] + elif count > 1: + raise MultipleObjectsReturned + else: + raise DoesNotExist + + def first(self): """Retrieve the first object matching the query. """ From 47e4dd40cd69f314f59703abb124f58268355f1f Mon Sep 17 00:00:00 2001 From: James Punteney Date: Sat, 16 Jan 2010 13:24:10 -0500 Subject: [PATCH 02/21] Making the query actually get called for get --- mongoengine/queryset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7d89a447..92d20315 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -295,11 +295,12 @@ class QuerySet(object): return mongo_query - def get(self): + def get(self, *q_objs, **query): """Retrieve the the matching object raising 'MultipleObjectsReturned' or 'DoesNotExist' exceptions if multiple or no results are found. """ + self.__call__(*q_objs, **query) count = self.count() if count == 1: return self[0] From e0becc109d1f6a236e500e65b4f8ff30b34b9e6d Mon Sep 17 00:00:00 2001 From: James Punteney Date: Sat, 16 Jan 2010 14:51:13 -0500 Subject: [PATCH 03/21] Adding tests to test the get query --- mongoengine/queryset.py | 1 - tests/queryset.py | 23 ++++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 92d20315..4a563b53 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -14,7 +14,6 @@ REPR_OUTPUT_SIZE = 20 class InvalidQueryError(Exception): pass - class OperationError(Exception): pass diff --git a/tests/queryset.py b/tests/queryset.py index 93dc0747..bf8af758 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2,7 +2,7 @@ import unittest import pymongo from datetime import datetime -from mongoengine.queryset import QuerySet +from mongoengine.queryset import QuerySet, MultipleObjectsReturned, DoesNotExist from mongoengine import * @@ -135,6 +135,27 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.with_id(person1.id) self.assertEqual(person.name, "User A") + def test_find_only_one(self): + """Ensure that a query using find_one returns a valid result. + """ + # Try retrieving when no objects exists + self.assertRaises(DoesNotExist, self.Person.objects.get) + + person1 = self.Person(name="User A", age=20) + person1.save() + person2 = self.Person(name="User B", age=30) + person2.save() + + # Retrieve the first person from the database + self.assertRaises(MultipleObjectsReturned, self.Person.objects.get) + + # Use a query to filter the people found to just person2 + person = self.Person.objects.get(age=30) + self.assertEqual(person.name, "User B") + + person = self.Person.objects.get(age__lt=30) + self.assertEqual(person.name, "User A") + def test_filter_chaining(self): """Ensure filters can be chained together. """ From 2585f1b724e77fb63eb18c614fb5e59413e2eb6c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 23 Jan 2010 17:16:01 +0000 Subject: [PATCH 04/21] queryset_manager funcs now accept doc as arg --- docs/guide/defining-documents.rst | 22 ++++++------ docs/guide/document-instances.rst | 4 +-- docs/guide/querying.rst | 57 ++++++++++++++++++++++++++----- mongoengine/queryset.py | 11 +++--- tests/queryset.py | 4 +-- 5 files changed, 69 insertions(+), 29 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 7683387e..0a12cb6d 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -44,7 +44,7 @@ are as follows: * :class:`~mongoengine.ReferenceField` List fields -^^^^^^^^^^^ +----------- MongoDB allows the storage of lists of items. To add a list of items to a :class:`~mongoengine.Document`, use the :class:`~mongoengine.ListField` field type. :class:`~mongoengine.ListField` takes another field object as its first @@ -54,7 +54,7 @@ argument, which specifies which type elements may be stored within the list:: tags = ListField(StringField(max_length=50)) Embedded documents -^^^^^^^^^^^^^^^^^^ +------------------ MongoDB has the ability to embed documents within other documents. Schemata may be defined for these embedded documents, just as they may be for regular documents. To create an embedded document, just define a document as usual, but @@ -76,7 +76,7 @@ document class as the first argument:: page = Page(comments=[comment1, comment2]) Reference fields -^^^^^^^^^^^^^^^^ +---------------- References may be stored to other documents in the database using the :class:`~mongoengine.ReferenceField`. Pass in another document class as the first argument to the constructor, then simply assign document objects to the @@ -100,7 +100,7 @@ The :class:`User` object is automatically turned into a reference behind the scenes, and dereferenced when the :class:`Page` object is retrieved. Uniqueness constraints -^^^^^^^^^^^^^^^^^^^^^^ +---------------------- MongoEngine allows you to specify that a field should be unique across a collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's constructor. If you try to save a document that has the same value for a unique @@ -130,7 +130,7 @@ document class to use:: meta = {'collection': 'cmsPage'} Capped collections -^^^^^^^^^^^^^^^^^^ +------------------ A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary. :attr:`max_documents` is the maximum number of documents that is allowed to be @@ -179,13 +179,13 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. :: } blog_post_1 = BlogPost(title="Blog Post #1") - blog_post_1.published_date = datetime(2010, 1, 5, 0, 0 ,0)) + blog_post_1.published_date = datetime(2010, 1, 5, 0, 0 ,0) blog_post_2 = BlogPost(title="Blog Post #2") - blog_post_2.published_date = datetime(2010, 1, 6, 0, 0 ,0)) + blog_post_2.published_date = datetime(2010, 1, 6, 0, 0 ,0) blog_post_3 = BlogPost(title="Blog Post #3") - blog_post_3.published_date = datetime(2010, 1, 7, 0, 0 ,0)) + blog_post_3.published_date = datetime(2010, 1, 7, 0, 0 ,0) blog_post_1.save() blog_post_2.save() @@ -194,11 +194,11 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. :: # get the "first" BlogPost using default ordering # from BlogPost.meta.ordering latest_post = BlogPost.objects.first() - self.assertEqual(latest_post.title, "Blog Post #3") + assert latest_post.title == "Blog Post #3" # override default ordering, order BlogPosts by "published_date" first_post = BlogPost.objects.order_by("+published_date").first() - self.assertEqual(first_post.title, "Blog Post #1") + assert first_post.title == "Blog Post #1" Document inheritance ==================== @@ -218,7 +218,7 @@ convenient and efficient retrieval of related documents:: date = DateTimeField() Working with existing data -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- To enable correct retrieval of documents involved in this kind of heirarchy, two extra attributes are stored on each document in the database: :attr:`_cls` and :attr:`_types`. These are hidden from the user through the MongoEngine diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 756bc3d5..b5a1f029 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -17,7 +17,7 @@ attribute syntax:: 'Example Page' Saving and deleting documents ------------------------------ +============================= To save the document to the database, call the :meth:`~mongoengine.Document.save` method. If the document does not exist in the database, it will be created. If it does already exist, it will be @@ -31,7 +31,7 @@ valide :attr:`id`. :ref:`guide-atomic-updates` Document IDs ------------- +============ Each document in the database has a unique id. This may be accessed through the :attr:`id` attribute on :class:`~mongoengine.Document` objects. Usually, the id will be generated automatically by the database server when the object is save, diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 3b700322..10712047 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -14,7 +14,7 @@ fetch documents from the database:: print user.name Filtering queries ------------------ +================= The query may be filtered by calling the :class:`~mongoengine.queryset.QuerySet` object with field lookup keyword arguments. The keys in the keyword arguments correspond to fields on the @@ -33,7 +33,7 @@ syntax:: uk_pages = Page.objects(author__country='uk') Querying lists -^^^^^^^^^^^^^^ +-------------- On most fields, this syntax will look up documents where the field specified matches the given value exactly, but when the field refers to a :class:`~mongoengine.ListField`, a single item may be provided, in which case @@ -47,7 +47,7 @@ lists that contain that item will be matched:: Page.objects(tags='coding') Query operators ---------------- +=============== Operators other than equality may also be used in queries; just attach the operator name to a key with a double-underscore:: @@ -69,7 +69,7 @@ Available operators are as follows: * ``exists`` -- value for field exists Limiting and skipping results ------------------------------ +============================= Just as with traditional ORMs, you may limit the number of results returned, or skip a number or results in you query. :meth:`~mongoengine.queryset.QuerySet.limit` and @@ -86,15 +86,54 @@ achieving this is using array-slicing syntax:: # 5 users, starting from the 10th user found users = User.objects[10:15] +Default Document queries +======================== +By default, the objects :attr:`~mongoengine.Document.objects` attribute on a +document returns a :class:`~mongoengine.queryset.QuerySet` that doesn't filter +the collection -- it returns all objects. This may be changed by defining a +method on a document that modifies a queryset. The method should accept two +arguments -- :attr:`doc_cls` and :attr:`queryset`. The first argument is the +:class:`~mongoengine.Document` class that the method is defined on (in this +sense, the method is more like a :func:`classmethod` than a regular method), +and the second argument is the initial queryset. The method needs to be +decorated with :func:`~mongoengine.queryset.queryset_manager` in order for it +to be recognised. :: + + class BlogPost(Document): + title = StringField() + date = DateTimeField() + + @queryset_manager + def objects(doc_cls, queryset): + # This may actually also be done by defining a default ordering for + # the document, but this illustrates the use of manager methods + return queryset.order_by('-date') + +You don't need to call your method :attr:`objects` -- you may define as many +custom manager methods as you like:: + + class BlogPost(Document): + title = StringField() + published = BooleanField() + + @queryset_manager + def live_posts(doc_cls, queryset): + return queryset.order_by('-date') + + BlogPost(title='test1', published=False).save() + BlogPost(title='test2', published=True).save() + assert len(BlogPost.objects) == 2 + assert len(BlogPost.live_posts) == 1 + Aggregation ------------ +=========== MongoDB provides some aggregation methods out of the box, but there are not as many as you typically get with an RDBMS. MongoEngine provides a wrapper around the built-in methods and provides some of its own, which are implemented as Javascript code that is executed on the database server. Counting results -^^^^^^^^^^^^^^^^ +---------------- Just as with limiting and skipping results, there is a method on :class:`~mongoengine.queryset.QuerySet` objects -- :meth:`~mongoengine.queryset.QuerySet.count`, but there is also a more Pythonic @@ -103,7 +142,7 @@ way of achieving this:: num_users = len(User.objects) Further aggregation -^^^^^^^^^^^^^^^^^^^ +------------------- You may sum over the values of a specific field on documents using :meth:`~mongoengine.queryset.QuerySet.sum`:: @@ -133,7 +172,7 @@ would be generating "tag-clouds":: top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10] Advanced queries ----------------- +================ Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword arguments can't fully express the query you want to use -- for example if you need to combine a number of constraints using *and* and *or*. This is made @@ -161,7 +200,7 @@ calling it with keyword arguments:: .. _guide-atomic-updates: Atomic updates --------------- +============== Documents may be updated atomically by using the :meth:`~mongoengine.queryset.QuerySet.update_one` and :meth:`~mongoengine.queryset.QuerySet.update` methods on a diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bb0090ea..6c8572c6 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -660,14 +660,15 @@ class QuerySetManager(object): # owner is the document that contains the QuerySetManager queryset = QuerySet(owner, self._collection) if self._manager_func: - queryset = self._manager_func(queryset) + queryset = self._manager_func(owner, queryset) return queryset def queryset_manager(func): - """Decorator that allows you to define custom QuerySet managers on + """Decorator that allows you to define custom QuerySet managers on :class:`~mongoengine.Document` classes. The manager must be a function that - accepts a :class:`~mongoengine.queryset.QuerySet` as its only argument, and - returns a :class:`~mongoengine.queryset.QuerySet`, probably the same one - but modified in some way. + accepts a :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. The method + function should return a :class:`~mongoengine.queryset.QuerySet`, probably + the same one that was passed in, but modified in some way. """ return QuerySetManager(func) diff --git a/tests/queryset.py b/tests/queryset.py index 10507e00..1e06615a 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -146,7 +146,7 @@ class QuerySetTest(unittest.TestCase): published_date = DateTimeField() @queryset_manager - def published(queryset): + def published(doc_cls, queryset): return queryset(is_published=True) blog_post_1 = BlogPost(title="Blog Post #1", @@ -444,7 +444,7 @@ class QuerySetTest(unittest.TestCase): tags = ListField(StringField()) @queryset_manager - def music_posts(queryset): + def music_posts(doc_cls, queryset): return queryset(tags='music') BlogPost.drop_collection() From 568000805f40236e5417584e392559e197312cb0 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 25 Jan 2010 01:00:04 +0000 Subject: [PATCH 05/21] EmbeddedDocuments may now be non-polymorphic --- mongoengine/base.py | 48 ++++++++++++++++++++++++++++----------------- tests/document.py | 16 ++++++++++++++- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 62bf80b5..83fd34ee 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -102,6 +102,7 @@ class DocumentMetaclass(type): doc_fields = {} class_name = [name] superclasses = {} + simple_class = True for base in bases: # Include all fields present in superclasses if hasattr(base, '_fields'): @@ -110,6 +111,29 @@ class DocumentMetaclass(type): # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) + + if hasattr(base, '_meta'): + # Ensure that the Document class may be subclassed - + # inheritance may be disabled to remove dependency on + # additional fields _cls and _types + if base._meta.get('allow_inheritance', True) == False: + raise ValueError('Document %s may not be subclassed' % + base.__name__) + else: + simple_class = False + + meta = attrs.get('_meta', attrs.get('meta', {})) + + if 'allow_inheritance' not in meta: + meta['allow_inheritance'] = True + + # Only simple classes - direct subclasses of Document - may set + # allow_inheritance to False + if not simple_class and not meta['allow_inheritance']: + raise ValueError('Only direct subclasses of Document may set ' + '"allow_inheritance" to False') + attrs['_meta'] = meta + attrs['_class_name'] = '.'.join(reversed(class_name)) attrs['_superclasses'] = superclasses @@ -142,21 +166,12 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): collection = name.lower() - simple_class = True id_field = None base_indexes = [] # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta') and 'collection' in base._meta: - # Ensure that the Document class may be subclassed - - # inheritance may be disabled to remove dependency on - # additional fields _cls and _types - if base._meta.get('allow_inheritance', True) == False: - raise ValueError('Document %s may not be subclassed' % - base.__name__) - else: - simple_class = False collection = base._meta['collection'] id_field = id_field or base._meta.get('id_field') @@ -164,7 +179,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta = { 'collection': collection, - 'allow_inheritance': True, 'max_documents': None, 'max_size': None, 'ordering': [], # default ordering applied at runtime @@ -174,12 +188,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Apply document-defined meta options meta.update(attrs.get('meta', {})) - - # Only simple classes - direct subclasses of Document - may set - # allow_inheritance to False - if not simple_class and not meta['allow_inheritance']: - raise ValueError('Only direct subclasses of Document may set ' - '"allow_inheritance" to False') attrs['_meta'] = meta # Set up collection manager, needs the class to have fields so use @@ -337,8 +345,12 @@ class BaseDocument(object): if value is not None: data[field.name] = field.to_mongo(value) # Only add _cls and _types if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', True) == False): + #if not (hasattr(self, '_meta') and + # self._meta.get('allow_inheritance', True) == False): + ah = True + if hasattr(self, '_meta'): + ah = self._meta.get('allow_inheritance', True) + if ah: data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] return data diff --git a/tests/document.py b/tests/document.py index 0c0b220b..1b58781c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -156,6 +156,20 @@ class DocumentTest(unittest.TestCase): class Employee(self.Person): meta = {'allow_inheritance': False} self.assertRaises(ValueError, create_employee_class) + + # Test the same for embedded documents + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': False} + + def create_special_comment(): + class SpecialComment(Comment): + pass + self.assertRaises(ValueError, create_special_comment) + + comment = Comment(content='test') + self.assertFalse('_cls' in comment.to_mongo()) + self.assertFalse('_types' in comment.to_mongo()) def test_collection_name(self): """Ensure that a collection with a specified name may be used. @@ -391,7 +405,7 @@ class DocumentTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) - self.assertFalse(hasattr(Comment, '_meta')) + self.assertFalse('collection' in Comment._meta) def test_embedded_document_validation(self): """Ensure that embedded documents may be validated. From bca6119db8c855d120b8a7aae478c895ee54e31c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 26 Jan 2010 19:36:19 +0000 Subject: [PATCH 06/21] Minor tidyup --- mongoengine/base.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 83fd34ee..f5a171aa 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -345,19 +345,15 @@ class BaseDocument(object): if value is not None: data[field.name] = field.to_mongo(value) # Only add _cls and _types if allow_inheritance is not False - #if not (hasattr(self, '_meta') and - # self._meta.get('allow_inheritance', True) == False): - ah = True - if hasattr(self, '_meta'): - ah = self._meta.get('allow_inheritance', True) - if ah: + if not (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] return data @classmethod def _from_son(cls, son): - """Create an instance of a Document (subclass) from a PyMongo SOM. + """Create an instance of a Document (subclass) from a PyMongo SON. """ # get the class name from the document, falling back to the given # class if unavailable From 5cac52720ca256c15919f0ae9e4d0e4ef9db3e44 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 27 Jan 2010 15:57:11 +0000 Subject: [PATCH 07/21] Fixed querying on ReferenceFields using primary key --- mongoengine/fields.py | 15 +++++++-------- tests/fields.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 13fa6689..beb8ae00 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -209,20 +209,19 @@ class ReferenceField(BaseField): return super(ReferenceField, self).__get__(instance, owner) def to_mongo(self, document): - if isinstance(document, (str, unicode, pymongo.objectid.ObjectId)): - # document may already be an object id - id_ = document - else: + id_field_name = self.document_type._meta['id_field'] + id_field = self.document_type._fields[id_field_name] + + if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.id if id_ is None: raise ValidationError('You can only reference documents once ' 'they have been saved to the database') + else: + id_ = document - # id may be a string rather than an ObjectID object - if not isinstance(id_, pymongo.objectid.ObjectId): - id_ = pymongo.objectid.ObjectId(id_) - + id_ = id_field.to_mongo(id_) collection = self.document_type._meta['collection'] return pymongo.dbref.DBRef(collection, id_) diff --git a/tests/fields.py b/tests/fields.py index affb0c9b..0ebb143d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -259,6 +259,40 @@ class FieldTest(unittest.TestCase): User.drop_collection() BlogPost.drop_collection() + def test_reference_query_conversion(self): + """Ensure that ReferenceFields can be queried using objects and values + of the type of the primary key of the referenced object. + """ + class Member(Document): + user_num = IntField(primary_key=True) + + class BlogPost(Document): + title = StringField() + author = ReferenceField(Member) + + Member.drop_collection() + BlogPost.drop_collection() + + m1 = Member(user_num=1) + m1.save() + m2 = Member(user_num=2) + m2.save() + + post1 = BlogPost(title='post 1', author=m1) + post1.save() + + post2 = BlogPost(title='post 2', author=m2) + post2.save() + + post = BlogPost.objects(author=m1.id).first() + self.assertEqual(post.id, post1.id) + + post = BlogPost.objects(author=m2.id).first() + self.assertEqual(post.id, post2.id) + + Member.drop_collection() + BlogPost.drop_collection() + if __name__ == '__main__': unittest.main() From 8d953f0bcb2484c1dce937efac07ae949f732052 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sat, 30 Jan 2010 21:12:46 +0100 Subject: [PATCH 08/21] Added get_or_create-method --- mongoengine/queryset.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6c8572c6..c40523ae 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -10,6 +10,9 @@ __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 +class MultipleObjectsReturned(Exception): + pass + class InvalidQueryError(Exception): pass @@ -292,6 +295,23 @@ class QuerySet(object): return mongo_query + def get_or_create(self, **kwargs): + """Retreive unique object or create with paras, if it doesn't exist + """ + dataset = self.filter(**kwargs) + cnt = dataset.count() + if cnt == 0: + if kwargs.has_key('defaults'): + kwargs.update(kwargs.get('defaults')) + del kwargs['defaults'] + doc = self._document(**kwargs) + doc.save() + return doc + elif cnt == 1: + return dataset.first() + else: + raise MultipleObjectsReturned(u'%d items returned, instead of 1' % cnt) + def first(self): """Retrieve the first object matching the query. """ From b3cc2f990a9ad612f66f4f7def23972074d77292 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sat, 30 Jan 2010 22:01:43 +0100 Subject: [PATCH 09/21] improved get_or_create --- mongoengine/queryset.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c40523ae..bec6da2c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -296,14 +296,16 @@ class QuerySet(object): return mongo_query def get_or_create(self, **kwargs): - """Retreive unique object or create with paras, if it doesn't exist + """Retreive unique object or create, if it doesn't exist """ + defaults = kwargs.get('defaults', {}) + if kwargs.has_key('defaults'): + del kwargs['defaults'] + dataset = self.filter(**kwargs) cnt = dataset.count() if cnt == 0: - if kwargs.has_key('defaults'): - kwargs.update(kwargs.get('defaults')) - del kwargs['defaults'] + kwargs.update(defaults) doc = self._document(**kwargs) doc.save() return doc From 7d6e117f687462709e19fd8ce343ac3534d3a825 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sun, 31 Jan 2010 01:11:37 +0100 Subject: [PATCH 10/21] added get-method to fetch exactly one document from the collection. catching pymongo's ObjectId-errors and raising mongoengine's ValidationError instead. --- mongoengine/base.py | 6 ++++-- mongoengine/queryset.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 83fd34ee..8c038c8c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -6,7 +6,6 @@ import pymongo class ValidationError(Exception): pass - class BaseField(object): """A base class for fields in a MongoDB document. Instances of this class may be added to subclasses of `Document` to define a document's schema. @@ -76,7 +75,10 @@ class ObjectIdField(BaseField): def to_mongo(self, value): if not isinstance(value, pymongo.objectid.ObjectId): - return pymongo.objectid.ObjectId(str(value)) + try: + return pymongo.objectid.ObjectId(str(value)) + except Exception, e: + raise ValidationError(e.message) return value def prepare_query_value(self, value): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bec6da2c..69bec002 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -10,6 +10,10 @@ __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 +class DoesNotExist(Exception): + pass + + class MultipleObjectsReturned(Exception): pass @@ -313,7 +317,17 @@ class QuerySet(object): return dataset.first() else: raise MultipleObjectsReturned(u'%d items returned, instead of 1' % cnt) - + + def get(self, **kwargs): + dataset = self.filter(**kwargs) + cnt = dataset.count() + if cnt == 1: + return dataset.first() + elif cnt > 1: + raise MultipleObjectsReturned(u'%d items returned, instead of 1' % cnt) + else: + raise DoesNotExist('Document not found') + def first(self): """Retrieve the first object matching the query. """ From 431f0067510154a32008239ef53147a1c472c588 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sun, 31 Jan 2010 14:40:00 +0100 Subject: [PATCH 11/21] new save() method updates only dirty fields. fixes issue #18 --- mongoengine/base.py | 13 +++++++++---- mongoengine/document.py | 24 ++++++++++++++++++++++-- mongoengine/fields.py | 1 + mongoengine/queryset.py | 6 ++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8c038c8c..b4920eec 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -15,13 +15,14 @@ class BaseField(object): _index_with_types = True def __init__(self, name=None, required=False, default=None, unique=False, - unique_with=None, primary_key=False): + unique_with=None, primary_key=False, modified=False): self.name = name if not primary_key else '_id' self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) self.unique_with = unique_with self.primary_key = primary_key + self.modified = modified def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do @@ -44,6 +45,7 @@ class BaseField(object): """Descriptor for assigning a value to a field in a document. """ instance._data[self.name] = value + self.modified = True def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. @@ -252,8 +254,11 @@ class BaseDocument(object): def __init__(self, **values): self._data = {} + + modified = 'id' in values.keys() # Assign initial values to instance for attr_name, attr_value in self._fields.items(): + attr_value.modified = modified if attr_name in values: setattr(self, attr_name, values.pop(attr_name)) else: @@ -381,9 +386,9 @@ class BaseDocument(object): # that has been queried to return this SON return None cls = subclasses[class_name] - + for field_name, field in cls._fields.items(): if field.name in data: data[field_name] = field.to_python(data[field.name]) - - return cls(**data) + + return cls(**data) \ No newline at end of file diff --git a/mongoengine/document.py b/mongoengine/document.py index 62f9ecce..eec31e09 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,8 +1,9 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError) -from queryset import OperationError +from queryset import OperationError, QuerySet from connection import _get_db + import pymongo @@ -75,12 +76,30 @@ class Document(BaseDocument): if force_insert: object_id = collection.insert(doc, safe=safe) else: - object_id = collection.save(doc, safe=safe) + if getattr(self, 'id', None) == None: + # new document + object_id = collection.save(doc, safe=safe) + else: + # update document + modified_fields = map(lambda obj: obj[0], filter(lambda obj: obj[1].modified, self._fields.items())) + modified_doc = dict(filter(lambda k: k[0] in modified_fields, doc.items())) + try: + id_field = self._meta['id_field'] + idObj = self._fields[id_field].to_mongo(self['id']) + collection.update({'_id': idObj}, {'$set': modified_doc}, safe=safe) + except pymongo.errors.OperationFailure, err: + if str(err) == 'multi not coded yet': + raise OperationError('update() method requires MongoDB 1.1.3+') + raise OperationError('Update failed (%s)' % str(err)) + object_id = self['id'] + + for field in self._fields.values(): field.modified = False except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if 'duplicate key' in str(err): message = 'Tried to save duplicate unique keys (%s)' raise OperationError(message % str(err)) + id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) @@ -106,6 +125,7 @@ class Document(BaseDocument): obj = self.__class__.objects(**{id_field: self[id_field]}).first() for field in self._fields: setattr(self, field, obj[field]) + obj.modified = False @classmethod def drop_collection(cls): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index beb8ae00..90a3b3d2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -71,6 +71,7 @@ class FloatField(BaseField): return float(value) def validate(self, value): + if isinstance(value, int): value = float(value) assert isinstance(value, float) if self.min_value is not None and value < self.min_value: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 69bec002..e71f0598 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -316,15 +316,17 @@ class QuerySet(object): elif cnt == 1: return dataset.first() else: - raise MultipleObjectsReturned(u'%d items returned, instead of 1' % cnt) + raise MultipleObjectsReturned(u'%d items returned, expected exactly one' % cnt) def get(self, **kwargs): + """Retreive exactly one document. Raise DoesNotExist if it's not found. + """ dataset = self.filter(**kwargs) cnt = dataset.count() if cnt == 1: return dataset.first() elif cnt > 1: - raise MultipleObjectsReturned(u'%d items returned, instead of 1' % cnt) + raise MultipleObjectsReturned(u'%d items returned, expected exactly one' % cnt) else: raise DoesNotExist('Document not found') From e05d31eaaf4e275b0f1f3af2bdd36ee74181f105 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 31 Jan 2010 13:47:27 +0000 Subject: [PATCH 12/21] Added get{,_or_create} docs --- docs/guide/querying.rst | 35 +++++++++++++++++++++++++++++++++++ docs/index.rst | 5 +++++ mongoengine/queryset.py | 7 ++----- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 10712047..0742244d 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -86,6 +86,41 @@ achieving this is using array-slicing syntax:: # 5 users, starting from the 10th user found users = User.objects[10:15] +You may also index the query to retrieve a single result. If an item at that +index does not exists, an :class:`IndexError` will be raised. A shortcut for +retrieving the first result and returning :attr:`None` if no result exists is +provided (:meth:`~mongoengine.queryset.QuerySet.first`):: + + >>> # Make sure there are no users + >>> User.drop_collection() + >>> User.objects[0] + IndexError: list index out of range + >>> User.objects.first() == None + True + >>> User(name='Test User').save() + >>> User.objects[0] == User.objects.first() + True + +Retrieving unique results +------------------------- +To retrieve a result that should be unique in the collection, use +:meth:`~mongoengine.queryset.QuerySet.get`. This will raise +:class:`~mongoengine.queryset.DoesNotExist` if no document matches the query, +and :class:`~mongoengine.queryset.MultipleObjectsReturned` if more than one +document matched the query. + +A variation of this method exists, +:meth:`~mongoengine.queryset.Queryset.get_or_create`, that will create a new +document with the query arguments if no documents match the query. An +additional keyword argument, :attr:`defaults` may be provided, which will be +used as default values for the new document, in the case that it should need +to be created:: + + >>> a = User.objects.get_or_create(name='User A', defaults={'age': 30}) + >>> b = User.objects.get_or_create(name='User A', defaults={'age': 40}) + >>> a.name == b.name and a.age == b.age + True + Default Document queries ======================== By default, the objects :attr:`~mongoengine.Document.objects` attribute on a diff --git a/docs/index.rst b/docs/index.rst index 1205e4b9..6e0db8b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,11 @@ MongoDB. To install it, simply run The source is available on `GitHub `_. +If you are interested in contributing, join the developers' `mailing list +`_. Some of us also like to +hang out at `#mongoengine IRC channel `_. + + .. toctree:: :maxdepth: 2 diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9764c699..f2405b82 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -10,6 +10,7 @@ __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 + class DoesNotExist(Exception): pass @@ -21,14 +22,10 @@ class MultipleObjectsReturned(Exception): class InvalidQueryError(Exception): pass + class OperationError(Exception): pass -class MultipleObjectsReturned(Exception): - pass - -class DoesNotExist(Exception): - pass class Q(object): From bbfc2f416eb4ace4d18008b845a7433dd570a62e Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sun, 31 Jan 2010 15:43:40 +0100 Subject: [PATCH 13/21] keep track of dirty fields is still work in progress; EmbeddedDocuments still aren't tracked (TBD) --- mongoengine/document.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index eec31e09..36216e0d 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -84,6 +84,10 @@ class Document(BaseDocument): modified_fields = map(lambda obj: obj[0], filter(lambda obj: obj[1].modified, self._fields.items())) modified_doc = dict(filter(lambda k: k[0] in modified_fields, doc.items())) try: + # + # WORK IN PROGRESS + # - EmbeddedDocuments still aren't tracked + # id_field = self._meta['id_field'] idObj = self._fields[id_field].to_mongo(self['id']) collection.update({'_id': idObj}, {'$set': modified_doc}, safe=safe) From 73aff806f329ace77bccca4001d8d8b15a44a7a3 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sun, 31 Jan 2010 18:00:01 +0100 Subject: [PATCH 14/21] reset to master, keep working on the dirty-fields-patch in another branch --- mongoengine/base.py | 13 ++++--------- mongoengine/document.py | 28 ++-------------------------- mongoengine/fields.py | 1 - 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index b658f326..024602a9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -15,14 +15,13 @@ class BaseField(object): _index_with_types = True def __init__(self, name=None, required=False, default=None, unique=False, - unique_with=None, primary_key=False, modified=False): + unique_with=None, primary_key=False): self.name = name if not primary_key else '_id' self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) self.unique_with = unique_with self.primary_key = primary_key - self.modified = modified def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do @@ -45,7 +44,6 @@ class BaseField(object): """Descriptor for assigning a value to a field in a document. """ instance._data[self.name] = value - self.modified = True def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. @@ -254,11 +252,8 @@ class BaseDocument(object): def __init__(self, **values): self._data = {} - - modified = 'id' in values.keys() # Assign initial values to instance for attr_name, attr_value in self._fields.items(): - attr_value.modified = modified if attr_name in values: setattr(self, attr_name, values.pop(attr_name)) else: @@ -382,9 +377,9 @@ class BaseDocument(object): # that has been queried to return this SON return None cls = subclasses[class_name] - + for field_name, field in cls._fields.items(): if field.name in data: data[field_name] = field.to_python(data[field.name]) - - return cls(**data) \ No newline at end of file + + return cls(**data) diff --git a/mongoengine/document.py b/mongoengine/document.py index 36216e0d..62f9ecce 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,9 +1,8 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError) -from queryset import OperationError, QuerySet +from queryset import OperationError from connection import _get_db - import pymongo @@ -76,34 +75,12 @@ class Document(BaseDocument): if force_insert: object_id = collection.insert(doc, safe=safe) else: - if getattr(self, 'id', None) == None: - # new document - object_id = collection.save(doc, safe=safe) - else: - # update document - modified_fields = map(lambda obj: obj[0], filter(lambda obj: obj[1].modified, self._fields.items())) - modified_doc = dict(filter(lambda k: k[0] in modified_fields, doc.items())) - try: - # - # WORK IN PROGRESS - # - EmbeddedDocuments still aren't tracked - # - id_field = self._meta['id_field'] - idObj = self._fields[id_field].to_mongo(self['id']) - collection.update({'_id': idObj}, {'$set': modified_doc}, safe=safe) - except pymongo.errors.OperationFailure, err: - if str(err) == 'multi not coded yet': - raise OperationError('update() method requires MongoDB 1.1.3+') - raise OperationError('Update failed (%s)' % str(err)) - object_id = self['id'] - - for field in self._fields.values(): field.modified = False + object_id = collection.save(doc, safe=safe) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if 'duplicate key' in str(err): message = 'Tried to save duplicate unique keys (%s)' raise OperationError(message % str(err)) - id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) @@ -129,7 +106,6 @@ class Document(BaseDocument): obj = self.__class__.objects(**{id_field: self[id_field]}).first() for field in self._fields: setattr(self, field, obj[field]) - obj.modified = False @classmethod def drop_collection(cls): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 90a3b3d2..beb8ae00 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -71,7 +71,6 @@ class FloatField(BaseField): return float(value) def validate(self, value): - if isinstance(value, int): value = float(value) assert isinstance(value, float) if self.min_value is not None and value < self.min_value: From 10de19d38b316b7727abd5615b1e74c95a4b52eb Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sun, 31 Jan 2010 18:06:25 +0100 Subject: [PATCH 15/21] be kind and also accept an integer for a float field (so e.g. mymodel.floatfield = 9 is possible, instead of mymodel.floatfield = 9.0) --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index beb8ae00..90a3b3d2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -71,6 +71,7 @@ class FloatField(BaseField): return float(value) def validate(self, value): + if isinstance(value, int): value = float(value) assert isinstance(value, float) if self.min_value is not None and value < self.min_value: From 0bbf3a3d76a11a44e1f76007860155f10273ade5 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 2 Feb 2010 17:37:09 +0000 Subject: [PATCH 16/21] Fixed EmbeddedDocument validation bug --- mongoengine/fields.py | 1 + tests/fields.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index beb8ae00..6a7e95a4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -129,6 +129,7 @@ class EmbeddedDocumentField(BaseField): if not isinstance(value, self.document): raise ValidationError('Invalid embedded document instance ' 'provided to an EmbeddedDocumentField') + self.document.validate(value) def lookup_member(self, member_name): return self.document._fields.get(member_name) diff --git a/tests/fields.py b/tests/fields.py index 0ebb143d..599e3d75 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -184,7 +184,7 @@ class FieldTest(unittest.TestCase): content = StringField() class PersonPreferences(EmbeddedDocument): - food = StringField() + food = StringField(required=True) number = IntField() class Person(Document): @@ -195,9 +195,14 @@ class FieldTest(unittest.TestCase): person.preferences = 'My Preferences' self.assertRaises(ValidationError, person.validate) + # Check that only the right embedded doc works person.preferences = Comment(content='Nice blog post...') self.assertRaises(ValidationError, person.validate) + # Check that the embedded doc is valid + person.preferences = PersonPreferences() + self.assertRaises(ValidationError, person.validate) + person.preferences = PersonPreferences(food='Cheese', number=47) self.assertEqual(person.preferences.food, 'Cheese') person.validate() From 69e9b5d55e26f7d758803c8f6c06a3970fecf68b Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 2 Feb 2010 21:44:11 +0100 Subject: [PATCH 17/21] fixed unicode-bug; replaced str(err) with err.message --- mongoengine/document.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 62f9ecce..357c8bc7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -78,9 +78,9 @@ class Document(BaseDocument): object_id = collection.save(doc, safe=safe) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' - if 'duplicate key' in str(err): - message = 'Tried to save duplicate unique keys (%s)' - raise OperationError(message % str(err)) + if u'duplicate key' in err.message: + message = u'Tried to save duplicate unique keys (%s)' + raise OperationError(message % err.message) id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) @@ -95,7 +95,7 @@ class Document(BaseDocument): try: self.__class__.objects(**{id_field: object_id}).delete(safe=safe) except pymongo.errors.OperationFailure, err: - raise OperationError('Could not delete document (%s)' % str(err)) + raise OperationError(u'Could not delete document (%s)' % err.message) def reload(self): """Reloads all attributes from the database. From 59f8c9f38e3f2dea3f33f32610c6e979d7a845c0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 2 Feb 2010 21:48:47 +0100 Subject: [PATCH 18/21] make mongoengine more international :) using unicode-strings; str(err) raises errors if it contains non-ascii chars/umlauts --- mongoengine/queryset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f2405b82..a42690a5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -520,9 +520,9 @@ class QuerySet(object): self._collection.update(self._query, update, safe=safe_update, multi=True) except pymongo.errors.OperationFailure, err: - if str(err) == 'multi not coded yet': - raise OperationError('update() method requires MongoDB 1.1.3+') - raise OperationError('Update failed (%s)' % str(err)) + if err.message == u'multi not coded yet': + raise OperationError(u'update() method requires MongoDB 1.1.3+') + raise OperationError(u'Update failed (%s)' % err.message) def update_one(self, safe_update=True, **update): """Perform an atomic update on first field matched by the query. From 755deb3ffec07878b2ee56550fdf73e339bcb87b Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 3 Feb 2010 01:22:25 +0000 Subject: [PATCH 19/21] Added DictField --- docs/apireference.rst | 2 ++ docs/guide/defining-documents.rst | 18 ++++++++++++++++++ mongoengine/fields.py | 24 +++++++++++++++++++++++- tests/fields.py | 22 ++++++++++++++++++++++ tests/queryset.py | 16 ++++++++++++++++ 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 03e44e63..1a243d2e 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -46,6 +46,8 @@ Fields .. autoclass:: mongoengine.EmbeddedDocumentField +.. autoclass:: mongoengine.DictField + .. autoclass:: mongoengine.ListField .. autoclass:: mongoengine.ObjectIdField diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 0a12cb6d..0862ffd0 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -39,6 +39,7 @@ are as follows: * :class:`~mongoengine.FloatField` * :class:`~mongoengine.DateTimeField` * :class:`~mongoengine.ListField` +* :class:`~mongoengine.DictField` * :class:`~mongoengine.ObjectIdField` * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` @@ -75,6 +76,23 @@ document class as the first argument:: comment2 = Comment('Nice article!') page = Page(comments=[comment1, comment2]) +Dictionary Fields +----------------- +Often, an embedded document may be used instead of a dictionary -- generally +this is recommended as dictionaries don't support validation or custom field +types. However, sometimes you will not know the structure of what you want to +store; in this situation a :class:`~mongoengine.DictField` is appropriate:: + + class SurveyResponse(Document): + date = DateTimeField() + user = ReferenceField(User) + answers = DictField() + + survey_response = SurveyResponse(date=datetime.now(), user=request.user) + response_form = ResponseForm(request.POST) + survey_response.answers = response_form.cleaned_data() + survey_response.save() + Reference fields ---------------- References may be stored to other documents in the database using the diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 6a7e95a4..67ac69c3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,7 +8,7 @@ import datetime __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', - 'DateTimeField', 'EmbeddedDocumentField', 'ListField', + 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError'] @@ -179,6 +179,28 @@ class ListField(BaseField): return self.field.lookup_member(member_name) +class DictField(BaseField): + """A dictionary field that wraps a standard Python dictionary. This is + similar to an embedded document, but the structure is not defined. + + .. versionadded:: 0.2.3 + """ + + def validate(self, value): + """Make sure that a list of valid fields is being used. + """ + if not isinstance(value, dict): + raise ValidationError('Only dictionaries may be used in a ' + 'DictField') + + if any(('.' in k or '$' in k) for k in value): + raise ValidationError('Invalid dictionary key name - keys may not ' + 'contain "." or "$" characters') + + def lookup_member(self, member_name): + return BaseField(name=member_name) + + class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on access (lazily). diff --git a/tests/fields.py b/tests/fields.py index 599e3d75..7949cc36 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -176,6 +176,28 @@ class FieldTest(unittest.TestCase): post.comments = 'yay' self.assertRaises(ValidationError, post.validate) + def test_dict_validation(self): + """Ensure that dict types work as expected. + """ + class BlogPost(Document): + info = DictField() + + post = BlogPost() + post.info = 'my post' + self.assertRaises(ValidationError, post.validate) + + post.info = ['test', 'test'] + self.assertRaises(ValidationError, post.validate) + + post.info = {'$title': 'test'} + self.assertRaises(ValidationError, post.validate) + + post.info = {'the.title': 'test'} + self.assertRaises(ValidationError, post.validate) + + post.info = {'title': 'test'} + post.validate() + def test_embedded_document_validation(self): """Ensure that invalid embedded documents cannot be assigned to embedded document fields. diff --git a/tests/queryset.py b/tests/queryset.py index 00f3e461..6d45bdae 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -277,6 +277,22 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_find_dict_item(self): + """Ensure that DictField items may be found. + """ + class BlogPost(Document): + info = DictField() + + BlogPost.drop_collection() + + post = BlogPost(info={'title': 'test'}) + post.save() + + post_obj = BlogPost.objects(info__title='test').first() + self.assertEqual(post_obj.id, post.id) + + BlogPost.drop_collection() + def test_q(self): class BlogPost(Document): publish_date = DateTimeField() From 89f505bb13cef72a804d09867a68bfdd2c46e98d Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Thu, 4 Feb 2010 01:44:52 +0000 Subject: [PATCH 20/21] Removed pool_size from connect, minor tidyup --- mongoengine/connection.py | 1 - mongoengine/document.py | 7 ++++--- mongoengine/fields.py | 3 ++- mongoengine/queryset.py | 7 ++++--- tests/fields.py | 2 +- tests/queryset.py | 6 +++++- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index ee8d735b..da8f2baf 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -7,7 +7,6 @@ __all__ = ['ConnectionError', 'connect'] _connection_settings = { 'host': 'localhost', 'port': 27017, - 'pool_size': 1, } _connection = None diff --git a/mongoengine/document.py b/mongoengine/document.py index 357c8bc7..8fdb88db 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -78,9 +78,9 @@ class Document(BaseDocument): object_id = collection.save(doc, safe=safe) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' - if u'duplicate key' in err.message: + if u'duplicate key' in unicode(err): message = u'Tried to save duplicate unique keys (%s)' - raise OperationError(message % err.message) + raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) @@ -95,7 +95,8 @@ class Document(BaseDocument): try: self.__class__.objects(**{id_field: object_id}).delete(safe=safe) except pymongo.errors.OperationFailure, err: - raise OperationError(u'Could not delete document (%s)' % err.message) + message = u'Could not delete document (%s)' % err.message + raise OperationError(message) def reload(self): """Reloads all attributes from the database. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a0374c4e..4695bf81 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -71,7 +71,8 @@ class FloatField(BaseField): return float(value) def validate(self, value): - if isinstance(value, int): value = float(value) + if isinstance(value, int): + value = float(value) assert isinstance(value, float) if self.min_value is not None and value < self.min_value: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a42690a5..e88804f9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -520,9 +520,10 @@ class QuerySet(object): self._collection.update(self._query, update, safe=safe_update, multi=True) except pymongo.errors.OperationFailure, err: - if err.message == u'multi not coded yet': - raise OperationError(u'update() method requires MongoDB 1.1.3+') - raise OperationError(u'Update failed (%s)' % err.message) + if unicode(err) == u'multi not coded yet': + message = u'update() method requires MongoDB 1.1.3+' + raise OperationError(message) + raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, safe_update=True, **update): """Perform an atomic update on first field matched by the query. diff --git a/tests/fields.py b/tests/fields.py index 7949cc36..b35a9142 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -106,7 +106,7 @@ class FieldTest(unittest.TestCase): person.height = 1.89 person.validate() - person.height = 2 + person.height = '2.0' self.assertRaises(ValidationError, person.validate) person.height = 0.01 self.assertRaises(ValidationError, person.validate) diff --git a/tests/queryset.py b/tests/queryset.py index 6d45bdae..b6e958a2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -21,7 +21,7 @@ class QuerySetTest(unittest.TestCase): """Ensure that a QuerySet is correctly initialised by QuerySetManager. """ self.assertTrue(isinstance(self.Person.objects, QuerySet)) - self.assertEqual(self.Person.objects._collection.name(), + self.assertEqual(self.Person.objects._collection.name, self.Person._meta['collection']) self.assertTrue(isinstance(self.Person.objects._collection, pymongo.collection.Collection)) @@ -294,6 +294,8 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() def test_q(self): + """Ensure that Q objects may be used to query for documents. + """ class BlogPost(Document): publish_date = DateTimeField() published = BooleanField() @@ -618,6 +620,8 @@ class QuerySetTest(unittest.TestCase): class QTest(unittest.TestCase): def test_or_and(self): + """Ensure that Q objects may be combined correctly. + """ q1 = Q(name='test') q2 = Q(age__gte=18) From b2588d1c4fe50baa7553189b94a55c5fb8518c54 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 10 Feb 2010 12:35:41 +0000 Subject: [PATCH 21/21] Changed neq to ne, fixed Q object in and nin --- mongoengine/queryset.py | 10 +++++----- tests/queryset.py | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e88804f9..8257e5f4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -33,14 +33,14 @@ class Q(object): AND = '&&' OPERATORS = { 'eq': 'this.%(field)s == %(value)s', - 'neq': 'this.%(field)s != %(value)s', + 'ne': 'this.%(field)s != %(value)s', 'gt': 'this.%(field)s > %(value)s', 'gte': 'this.%(field)s >= %(value)s', 'lt': 'this.%(field)s < %(value)s', 'lte': 'this.%(field)s <= %(value)s', 'lte': 'this.%(field)s <= %(value)s', - 'in': 'this.%(field)s.indexOf(%(value)s) != -1', - 'nin': 'this.%(field)s.indexOf(%(value)s) == -1', + 'in': '%(value)s.indexOf(this.%(field)s) != -1', + 'nin': '%(value)s.indexOf(this.%(field)s) == -1', 'mod': '%(field)s %% %(value)s', 'all': ('%(value)s.every(function(a){' 'return this.%(field)s.indexOf(a) != -1 })'), @@ -265,7 +265,7 @@ class QuerySet(object): def _transform_query(cls, _doc_cls=None, **query): """Transform a query from Django-style format to Mongo format. """ - operators = ['neq', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', + operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists'] mongo_query = {} @@ -283,7 +283,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'neq', 'gt', 'gte', 'lt', 'lte'): + if op in (None, 'ne', 'gt', 'gte', 'lt', 'lte'): value = field.prepare_query_value(value) elif op in ('in', 'nin', 'all'): # 'in', 'nin' and 'all' require a list of values diff --git a/tests/queryset.py b/tests/queryset.py index b6e958a2..d05fac29 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -331,6 +331,15 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + # Check the 'in' operator + self.Person(name='user1', age=20).save() + self.Person(name='user2', age=20).save() + self.Person(name='user3', age=30).save() + self.Person(name='user4', age=40).save() + + self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) + self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. """