From 86e2797c577d007b1883164867eaca7548703ea9 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 9 Jun 2010 22:28:30 -0300 Subject: [PATCH 01/35] added a TestCase for tests that uses mongoDB --- mongoengine/django/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 mongoengine/django/tests.py diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py new file mode 100644 index 00000000..a8d7c7ff --- /dev/null +++ b/mongoengine/django/tests.py @@ -0,0 +1,21 @@ +#coding: utf-8 +from django.test import TestCase +from django.conf import settings + +from mongoengine import connect + +class MongoTestCase(TestCase): + """ + TestCase class that clear the collection between the tests + """ + db_name = 'test_%s' % settings.MONGO_DATABASE_NAME + def __init__(self, methodName='runtest'): + self.db = connect(self.db_name) + super(MongoTestCase, self).__init__(methodName) + + def _post_teardown(self): + super(MongoTestCase, self)._post_teardown() + for collection in self.db.collection_names(): + if collection == 'system.indexes': + continue + self.db.drop_collection(collection) From f5e39c0064e52c03194e09a72c204c409a9ab5b2 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 10:25:31 +0200 Subject: [PATCH 02/35] Allowed _id to be missing when converting to mongo --- mongoengine/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..fe942b10 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -407,11 +407,15 @@ class BaseDocument(object): value = getattr(self, field_name, None) if value is not None: data[field.db_field] = field.to_mongo(value) + else: + data[field.db_field] = None # Only add _cls and _types if allow_inheritance is not False if not (hasattr(self, '_meta') and self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] + if hasattr(self, '_id') and not data['_id']: + del data['_id'] return data @classmethod From 3179c4e4aca20cdaa3daafb7cbc1204d847e2e2f Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 11:25:49 +0200 Subject: [PATCH 03/35] Now only removes _id if none, for real --- mongoengine/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index fe942b10..253c758a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -414,8 +414,11 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if hasattr(self, '_id') and not data['_id']: - del data['_id'] + try: + if not data['_id']: + del data['_id'] + except KeyError: + pass return data @classmethod From b89d71bfa5ffea3d3aff6c23ab0be4bcfefc930e Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 14:17:30 +0200 Subject: [PATCH 04/35] Do not convert None objects --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 253c758a..8137eddf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -450,7 +450,8 @@ class BaseDocument(object): for field_name, field in cls._fields.items(): if field.db_field in data: - data[field_name] = field.to_python(data[field.db_field]) + value = data[field.db_field] + data[field_name] = value if value is None else field.to_python(value) obj = cls(**data) obj._present_fields = present_fields From acbc741037c00736ce1dbdcd39bd77ddf6f663b3 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Thu, 15 Jul 2010 18:20:29 +0200 Subject: [PATCH 05/35] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..6bfcff9f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -261,6 +261,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 51065e7a4dda96943dadfcbdfcca6111dfa1f9f5 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:33:33 +0200 Subject: [PATCH 06/35] Closes #46 by instantiating a new default instance for every field by request. --- mongoengine/base.py | 5 ++++- mongoengine/fields.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 086c7874..10ff1219 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,7 +52,10 @@ class BaseField(object): # Get value from document instance if available, if not use default value = instance._data.get(self.name) if value is None: - value = self.default + if callable(self.default): # fixes #46 + value = self.default() + else: + value = self.default # Allow callable default values if callable(value): value = value() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..670e3cd3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From 9d82911f6338747411788291175739f436d95709 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:38:24 +0200 Subject: [PATCH 07/35] Added tests for #46. --- tests/fields.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 386c48b116191a754e3cec550ccdbd5246efdc97 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:43:11 +0200 Subject: [PATCH 08/35] Typo. --- tests/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields.py b/tests/fields.py index ef776d4a..6e42ea4d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -694,7 +694,7 @@ class FieldTest(unittest.TestCase): Event.drop_collection() def test_ensure_unique_default_instances(self): - """Ensure that every document has it's own unique default instance.""" + """Ensure that every field has it's own unique default instance.""" class D(Document): data = DictField() data2 = DictField(default=lambda: {}) From 9411b38508ac261f166dac1f8f8adc7f01084334 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:45:49 +0200 Subject: [PATCH 09/35] Removed unnecessary comment. --- mongoengine/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 00a7f7a2..be29ae83 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -135,7 +135,6 @@ class Q(object): # Handle DBRef if isinstance(value, pymongo.dbref.DBRef): - # this.created_user.$id == "4c4c56f8cc1831418c000000" op_js = '(this.%(field)s.$id == "%(id)s" &&'\ ' this.%(field)s.$ref == "%(ref)s")' % { 'field': key, From 2f991ac6f18172008ed71f1d97ab6ee71ea360ba Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 19:02:15 +0200 Subject: [PATCH 10/35] Added all() method to get all document instances from a document. Extended the FileField's tests with testing on empty filefield. --- mongoengine/queryset.py | 4 ++++ tests/fields.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index be29ae83..ffbe5255 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -238,6 +238,10 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) + + def all(self): + """Returns all documents.""" + return self.__call__() @property def _collection(self): diff --git a/tests/fields.py b/tests/fields.py index 6e42ea4d..1e53d23d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -674,7 +674,12 @@ class FieldTest(unittest.TestCase): PutFile.drop_collection() StreamFile.drop_collection() SetFile.drop_collection() - + + # Make sure FileField is optional and not required + class DemoFile(Document): + file = FileField() + d = DemoFile.objects.create() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 7ab2e21c106fc89875b1ae4ae79ecd7e0988ded3 Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:42:10 +0200 Subject: [PATCH 11/35] Handle unsafe expressions when using startswith/endswith/contains with unsafe expressions. Closes #58 --- mongoengine/fields.py | 3 +++ tests/queryset.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 670e3cd3..bd81d3a8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,6 +66,9 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' + + # escape unsafe characters which could lead to a re.error + value = re.escape(value) value = re.compile(regex % value, flags) return value diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..1efd034c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -288,6 +288,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() self.assertEqual(obj, None) + + # Test unsafe expressions + person = self.Person(name='Guido van Rossum [.\'Geek\']') + person.save() + + obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. From 6791f205af3d15a0f4ccaf357692ba868b6ebfff Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:50:09 +0200 Subject: [PATCH 12/35] Style fix. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bd81d3a8..30a11f20 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,7 +66,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - + # escape unsafe characters which could lead to a re.error value = re.escape(value) value = re.compile(regex % value, flags) From 21d267cb112e9030a0f17271fecd052dbe35b50a Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 17:28:59 +0200 Subject: [PATCH 13/35] Now order_by() works like queries for referencing deeper fields (replacing . with __). old: order_by('mydoc.myattr') / new: order_by('mydoc__myattr'). Closes #45 --- mongoengine/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ffbe5255..0b9218af 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -670,11 +670,13 @@ class QuerySet(object): """ key_list = [] for key in keys: + if not key: continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING if key[0] in ('-', '+'): key = key[1:] + key = key.replace('__', '.') key_list.append((key, direction)) self._ordering = key_list From 1147ac43506283a37554d09242894e9088f2133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 23 May 2010 00:19:50 +0800 Subject: [PATCH 14/35] ignore virtualenv directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42dcc6e6..57cf1b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ docs/.build docs/_build build/ dist/ -mongoengine.egg-info/ \ No newline at end of file +mongoengine.egg-info/ +env/ \ No newline at end of file From b96e27a7e4675c0564cff05bcba66f7662b6099d Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Fri, 30 Jul 2010 22:09:00 +1000 Subject: [PATCH 15/35] Allow documents to override the 'objects' QuerySetManager --- mongoengine/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..ed8214af 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - new_class.objects = QuerySetManager() + + # Provide a default queryset unless one has been manually provided + if not 'objects' in dir(new_class): + new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes From 17addbefe2e7befe45edcc764add45c397d4202b Mon Sep 17 00:00:00 2001 From: sp Date: Wed, 18 Aug 2010 16:50:52 -0400 Subject: [PATCH 16/35] made it more like Django's user model --- mongoengine/django/auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d4b0ff0b..da0005c8 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -32,6 +32,9 @@ class User(Document): last_login = DateTimeField(default=datetime.datetime.now) date_joined = DateTimeField(default=datetime.datetime.now) + def __unicode__(self): + return self.username + def get_full_name(self): """Returns the users first and last names, separated by a space. """ From 7de9adc6b1677b5e9c2b7b2b6d5b670effdd5ead Mon Sep 17 00:00:00 2001 From: Richard Henry Date: Sat, 28 Aug 2010 09:16:02 +0100 Subject: [PATCH 17/35] Adding support for pop operations to QuerySet.update and QuerySet.update_one --- docs/guide/querying.rst | 1 + mongoengine/queryset.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..7de6c5c8 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -399,6 +399,7 @@ that you may use with these methods: * ``unset`` -- delete a particular value (since MongoDB v1.3+) * ``inc`` -- increment a value by a given amount * ``dec`` -- decrement a value by a given amount +* ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list * ``pull`` -- remove a value from a list diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab113..182dfd52 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -661,8 +661,8 @@ class QuerySet(object): def _transform_update(cls, _doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ - operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', + 'pull', 'pull_all'] mongo_update = {} for key, value in update.items(): @@ -688,7 +688,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'pop', 'push', 'pull'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From d99c5973c3cb6da18670c6592ff396d18bb0e946 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 12:52:24 +0100 Subject: [PATCH 18/35] Fixed Q object DBRef test bug --- tests/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..b1f16576 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,8 @@ class QTest(unittest.TestCase): def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + class User(Document): pass From 3b62cf80cde0b83715662ce9bb01053dc13a7060 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:00:34 +0100 Subject: [PATCH 19/35] Fixed {Dict,List}Field default issue. Closes #46. --- mongoengine/fields.py | 6 +++--- tests/fields.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..d8d310dd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): @@ -623,4 +623,4 @@ class GeoPointField(BaseField): raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int.') \ No newline at end of file + raise ValidationError('Both values in point must be float or int.') diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 4fb6fcabefa04f9ab4b410ce1cf3b6293e0ec52c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:54:20 +0100 Subject: [PATCH 20/35] Added test for overriding objects --- tests/queryset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/queryset.py b/tests/queryset.py index b339d613..0c7ba05a 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -992,10 +992,15 @@ class QuerySetTest(unittest.TestCase): """ class BlogPost(Document): tags = ListField(StringField()) + deleted = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(deleted=False) @queryset_manager def music_posts(doc_cls, queryset): - return queryset(tags='music') + return queryset(tags='music', deleted=False) BlogPost.drop_collection() @@ -1005,6 +1010,8 @@ class QuerySetTest(unittest.TestCase): post2.save() post3 = BlogPost(tags=['film', 'actors']) post3.save() + post4 = BlogPost(tags=['film', 'actors'], deleted=True) + post4.save() self.assertEqual([p.id for p in BlogPost.objects], [post1.id, post2.id, post3.id]) From 95efa39b5229081deef179ae4b08299a50b18ccb Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:56:18 +0200 Subject: [PATCH 21/35] Added *.egg to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57cf1b7b..51a9ca1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*.egg docs/.build docs/_build build/ From e0911a5fe00d77bb74b0b8b2d8e5d6d963afc687 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:58:58 +0200 Subject: [PATCH 22/35] Replaced slow exception handling with has_key. --- mongoengine/base.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4523a268..e0e71d31 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -415,11 +415,8 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - try: - if not data['_id']: - del data['_id'] - except KeyError: - pass + if data.has_key('_id') and not data['_id']: + del data['_id'] return data @classmethod From 1ed9a36d0a15eef4038568ccc974d14bfb425c42 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 14:02:02 +0100 Subject: [PATCH 23/35] Added test for pop operation --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 0c7ba05a..2f86953d 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -671,6 +671,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + tags = post.tags[:-1] + BlogPost.objects.update(pop__tags=1) + post.reload() + self.assertEqual(post.tags, tags) + BlogPost.drop_collection() def test_update_pull(self): From 5b230b90b984b20d09322d17dd6b43b93db050f7 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 15:34:29 +0100 Subject: [PATCH 24/35] Doc fix (all operator) --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 86563082..bef19bc5 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -71,7 +71,7 @@ Available operators are as follows: * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values -* ``all`` -- every item in array is in list of values provided +* ``all`` -- every item in list of values provided is in array * ``size`` -- the size of the array is * ``exists`` -- value for field exists From 185e7a6a7e27884ac4896bf5c6a6881db8d419a0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 18:38:41 +0200 Subject: [PATCH 25/35] Better way of checking if new_class has an 'objects' attribute. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3172e9a3..9505d416 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -257,7 +257,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) # Provide a default queryset unless one has been manually provided - if not 'objects' in dir(new_class): + if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) From dcc8d22cec6d59392c8470a9325d0d8fedc6d4fa Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 16:22:56 -0700 Subject: [PATCH 26/35] Proper unique index name when using db_field. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9505d416..92a450de 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -269,7 +269,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True - unique_fields = [field_name] + unique_fields = [field.db_field] # Add any unique_with fields to the back of the index spec if field.unique_with: From 266f33adc4730859c272d9a8233769b0d9e54205 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 19:16:54 -0700 Subject: [PATCH 27/35] Bug fix for gridfs FileField. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index dc5dbe0e..49e9d056 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -600,7 +600,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - if self.gridfs.grid_id is not None: + if value is not None: return self.gridfs.get(id=value) return None From 3e30d71263356082ef5d391a2ea614d74790bb32 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:25:07 -0700 Subject: [PATCH 28/35] Support for background and drop_dups indexing options. --- mongoengine/queryset.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dd7fcef6..7969c196 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -172,7 +172,8 @@ class QuerySet(object): self._limit = None self._skip = None - def ensure_index(self, key_or_list): + def ensure_index(self, key_or_list, drop_dups=False, background=False, + **kwargs): """Ensure that the given indexes are in place. :param key_or_list: a single index key or a list of index keys (to @@ -180,7 +181,8 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_list = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index(index_list) + self._collection.ensure_index(index_list, drop_dups=drop_dups, + background=background) return self @classmethod @@ -250,25 +252,33 @@ class QuerySet(object): """ if not self._accessed_collection: self._accessed_collection = True + + background = self._document._meta.get('index_background', False) + drop_dups = self._document._meta.get('index_drop_dups', False) + index_opts = self._document._meta.get('index_options', {}) # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - self._collection.ensure_index(key_or_list) + self._collection.ensure_index(key_or_list, + background=background, **index_opts) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - self._collection.ensure_index(index, unique=True) + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: - self._collection.ensure_index('_types') + self._collection.ensure_index('_types', + background=background, **index_opts) # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec) + self._collection.ensure_index(index_spec, + background=background, **index_opts) return self._collection_obj From f1aec68f239d77edff296e05764c419c4187b2d0 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:46:44 -0700 Subject: [PATCH 29/35] Inherit index options. --- mongoengine/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92a450de..836817da 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -230,12 +230,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_field = None base_indexes = [] + base_meta = {} # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta') and 'collection' in base._meta: collection = base._meta['collection'] + # Propagate index options. + for key in ('index_background', 'index_drop_dups', 'index_opts'): + if key in base._meta: + base_meta[key] = base._meta[key] + id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) @@ -246,7 +252,11 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime 'id_field': id_field, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': {}, } + meta.update(base_meta) # Apply document-defined meta options meta.update(attrs.get('meta', {})) From 17642c8a8cc2e37741a1d10be04d704770679193 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:48:17 +0100 Subject: [PATCH 30/35] Fixed QuerySet.average issue that ignored 0 --- mongoengine/queryset.py | 2 +- tests/queryset.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7969c196..662fa8c3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -892,7 +892,7 @@ class QuerySet(object): var total = 0.0; var num = 0; db[collection].find(query).forEach(function(doc) { - if (doc[averageField]) { + if (doc[averageField] !== undefined) { total += doc[averageField]; num += 1; } diff --git a/tests/queryset.py b/tests/queryset.py index 2f86953d..0c6c3ca4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -960,11 +960,14 @@ class QuerySetTest(unittest.TestCase): def test_average(self): """Ensure that field can be averaged correctly. """ + self.Person(name='person', age=0).save() + self.assertEqual(int(self.Person.objects.average('age')), 0) + ages = [23, 54, 12, 94, 27] for i, age in enumerate(ages): self.Person(name='test%s' % i, age=age).save() - avg = float(sum(ages)) / len(ages) + avg = float(sum(ages)) / (len(ages) + 1) # take into account the 0 self.assertAlmostEqual(int(self.Person.objects.average('age')), avg) self.Person(name='ageless person').save() @@ -1340,5 +1343,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + if __name__ == '__main__': unittest.main() From 69012e8ad11a7e4312450ff71fc80445e007270f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:59:49 +0100 Subject: [PATCH 31/35] Fixed incorrect $pull test --- tests/queryset.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 0c6c3ca4..e3912246 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -681,23 +681,17 @@ class QuerySetTest(unittest.TestCase): def test_update_pull(self): """Ensure that the 'pull' update operation works correctly. """ - class Comment(EmbeddedDocument): - content = StringField() - class BlogPost(Document): slug = StringField() - comments = ListField(EmbeddedDocumentField(Comment)) + tags = ListField(StringField()) - comment1 = Comment(content="test1") - comment2 = Comment(content="test2") - - post = BlogPost(slug="test", comments=[comment1, comment2]) + post = BlogPost(slug="test", tags=['code', 'mongodb', 'code']) post.save() - self.assertTrue(comment2 in post.comments) - BlogPost.objects(slug="test").update(pull__comments__content="test2") + BlogPost.objects(slug="test").update(pull__tags="code") post.reload() - self.assertTrue(comment2 not in post.comments) + self.assertTrue('code' not in post.tags) + self.assertEqual(len(post.tags), 1) def test_order_by(self): """Ensure that QuerySets may be ordered. From 32e66b29f44f3015be099851201241caee92054f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 22:12:05 +0100 Subject: [PATCH 32/35] Fixed FileField problem caused by shared objects --- mongoengine/fields.py | 50 ++++++++++++++++++++++++++++++------------- tests/fields.py | 22 +++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 49e9d056..ffcfb53d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -510,14 +510,15 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ - def __init__(self): + def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file + self.grid_id = grid_id # Store GridFS id for file def __getattr__(self, name): obj = self.get() @@ -528,9 +529,12 @@ class GridFSProxy(object): return self def get(self, id=None): - if id: self.grid_id = id - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted + if id: + self.grid_id = id + try: + return self.fs.get(id or self.grid_id) + except: + return None # File has been deleted def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -552,8 +556,10 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - try: return self.get().read() - except: return None + try: + return self.get().read() + except: + return None def delete(self): # Delete file from GridFS, FileField still remains @@ -571,38 +577,52 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) + class FileField(BaseField): """A GridFS storage field. """ def __init__(self, **kwargs): - self.gridfs = GridFSProxy() super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self - return self.gridfs + # Check if a file already exists for this model + grid_file = instance._data.get(self.name) + if grid_file: + return grid_file + return GridFSProxy() def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation - self.gridfs.put(value) + grid_file = instance._data.get(self.name) + # If a file already exists, delete it + if grid_file: + try: + grid_file.delete() + except: + pass + # Create a new file with the new data + grid_file.put(value) + else: + # Create a new proxy object as we don't already have one + instance._data[self.name] = GridFSProxy() + instance._data[self.name].put(value) else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB - if self.gridfs.grid_id is not None: - return self.gridfs.grid_id + if isinstance(value, GridFSProxy) and value.grid_id is not None: + return value.grid_id return None def to_python(self, value): - # Use stored value (id) to lookup file in GridFS if value is not None: - return self.gridfs.get(id=value) - return None + return GridFSProxy(value) def validate(self, value): if value.grid_id is not None: diff --git a/tests/fields.py b/tests/fields.py index 1e53d23d..8c727196 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -680,6 +680,28 @@ class FieldTest(unittest.TestCase): file = FileField() d = DemoFile.objects.create() + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + file = FileField() + + # First instance + testfile = TestFile() + testfile.name = "Hello, World!" + testfile.file.put('Hello, World!') + testfile.save() + + # Second instance + testfiledupe = TestFile() + data = testfiledupe.file.read() # Should be None + + self.assertTrue(testfile.name != testfiledupe.name) + self.assertTrue(testfile.file.read() != data) + + TestFile.drop_collection() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 1849f75ad04a8a50c73862a6d0794fe65016343c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:23:59 +0100 Subject: [PATCH 33/35] Made GridFSProxy a bit stricter / safer --- mongoengine/fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ffcfb53d..418f57cc 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -511,6 +511,10 @@ class BinaryField(BaseField): raise ValidationError('Binary value is too long') +class GridFSError(Exception): + pass + + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ @@ -541,12 +545,18 @@ class GridFSProxy(object): self.grid_id = self.newfile._id def put(self, file, **kwargs): + if self.grid_id: + raise GridFSError('This document alreay has a file. Either delete ' + 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): - if not self.newfile: + if self.grid_id: + if not self.newfile: + raise GridFSError('This document alreay has a file. Either ' + 'delete it or call replace to overwrite it') + else: self.new_file() - self.grid_id = self.newfile._id self.newfile.write(string) def writelines(self, lines): From 2af5f3c56ebc7feff6a0a1cd95a77e12b425efed Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:24:30 +0100 Subject: [PATCH 34/35] Added support for querying by array position. Closes #36. --- mongoengine/queryset.py | 6 +++++- tests/queryset.py | 43 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c3..b8ca125d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -344,6 +344,8 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if parts[-1] in operators + match_operators + geo_operators: @@ -381,7 +383,9 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + + for i, part in indices: + parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value diff --git a/tests/queryset.py b/tests/queryset.py index e3912246..3d714be2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -165,8 +165,49 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") + def test_find_array_position(self): + """Ensure that query by array position works. + """ + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() - + Blog.objects.create(tags=['a', 'b']) + self.assertEqual(len(Blog.objects(tags__0='a')), 1) + self.assertEqual(len(Blog.objects(tags__0='b')), 0) + self.assertEqual(len(Blog.objects(tags__1='a')), 0) + self.assertEqual(len(Blog.objects(tags__1='b')), 1) + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + blog = Blog.objects(posts__0__comments__0__name='testa').get() + self.assertEqual(blog, blog1) + + query = Blog.objects(posts__1__comments__1__name='testb') + self.assertEqual(len(query), 2) + + query = Blog.objects(posts__1__comments__1__name='testa') + self.assertEqual(len(query), 0) + + query = Blog.objects(posts__0__comments__1__name='testa') + self.assertEqual(len(query), 0) + + Blog.drop_collection() def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new From f11ee1f9cf60979fbb5c6768219225202d510951 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 15 Sep 2010 09:47:13 +0100 Subject: [PATCH 35/35] Added support for using custom QuerySet classes --- mongoengine/base.py | 1 + mongoengine/queryset.py | 3 ++- tests/queryset.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817da..0cbd707d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -255,6 +255,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_background': False, 'index_drop_dups': False, 'index_opts': {}, + 'queryset_class': QuerySet, } meta.update(base_meta) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125d..8b486093 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -992,7 +992,8 @@ class QuerySetManager(object): self._collection = db[collection] # owner is the document that contains the QuerySetManager - queryset = QuerySet(owner, self._collection) + queryset_class = owner._meta['queryset_class'] or QuerySet + queryset = queryset_class(owner, self._collection) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/queryset.py b/tests/queryset.py index 3d714be2..4491be8c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1378,6 +1378,26 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + if __name__ == '__main__': unittest.main()