From a4d2f22fd2016e47987eb8c48f541db99eafb067 Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:14:01 -0500 Subject: [PATCH 01/98] added 'geo_indexes' to TopLevelDocumentMetaclass; added GeoPointField, a glorified [lat float, lng float] container; added geo lookup operators to QuerySet; added initial geo tests --- mongoengine/base.py | 1 + mongoengine/fields.py | 17 ++++++++++++- mongoengine/queryset.py | 26 ++++++++++++++++---- tests/queryset.py | 54 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0a904467..e4b45f26 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -206,6 +206,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'max_size': None, 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime + 'geo_indexes': [], 'id_field': id_field, } diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9a9f4e0e..bb61f1da 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField'] + 'BinaryField', 'GeoPointField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -443,6 +443,7 @@ class GenericReferenceField(BaseField): def prepare_query_value(self, op, value): return self.to_mongo(value)['_ref'] + class BinaryField(BaseField): """A binary data field. """ @@ -462,3 +463,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + + +class GeoPointField(BaseField): + """A list storing a latitude and longitude. + """ + + def validate(self, value): + assert isinstance(value, (list, tuple)) + + if not len(value) == 2: + raise ValidationError('Value must be a two-dimensional point.') + if not isinstance(value[0], (float, int)) and \ + not isinstance(value[1], (float, int)): + raise ValidationError('Both values in point must be float or int.') diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11dc2bcb..f03f97a5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -232,12 +232,17 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - #self.ensure_index(key_or_list) self._collection.ensure_index(key_or_list) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True) + + if self._document._meta['geo_indexes'] and \ + pymongo.version >= "1.5.1": + from pymongo import GEO2D + for index in self._document._meta['geo_indexes']: + self._collection.ensure_index([(index, GEO2D)]) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: @@ -298,6 +303,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists'] + geo_operators = ['within_distance', 'within_box', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith'] @@ -306,7 +312,7 @@ class QuerySet(object): parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in operators + match_operators: + if parts[-1] in operators + match_operators + geo_operators: op = parts.pop() if _doc_cls: @@ -320,15 +326,25 @@ class QuerySet(object): singular_ops += match_operators if op in singular_ops: value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all'): + elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] if field.__class__.__name__ == 'GenericReferenceField': parts.append('_ref') - if op and op not in match_operators: - value = {'$' + op: value} + # if op and op not in match_operators: + if op: + if op in geo_operators: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "near": + value = {'$near': value} + else: + raise NotImplmenetedError, \ + "Geo method has been implemented" + elif op not in match_operators: + value = {'$' + op: value} key = '.'.join(parts) if op is None or key not in mongo_query: diff --git a/tests/queryset.py b/tests/queryset.py index c0bd8fa9..004c4160 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,6 +1070,60 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() + def test_near(self): + """Ensure that "near" queries work with and without radii. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + def __unicode__(self): + return self.title + + meta = {'geo_indexes': ["location"]} + + Event.drop_collection() + + event1 = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + location=[37.7749295, -122.4194155]) + event3 = Event(title="Coltrane Motion @ Empty Bottle", + location=[41.900474, -87.686638]) + + event1.save() + event2.save() + event3.save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(location__near=[41.9120459, -87.67892]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + # find events within 5 miles of pitchfork office, chicago + point_and_distance = [[41.9120459, -87.67892], 5] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 2) + events = list(events) + self.assertTrue(event2 not in events) + self.assertTrue(event1 in events) + self.assertTrue(event3 in events) + + # find events around san francisco + point_and_distance = [[37.7566023, -122.415579], 10] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 1 mile of greenpoint, broolyn, nyc, ny + point_and_distance = [[40.7237134, -73.9509714], 1] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 0) + + Event.drop_collection() + class QTest(unittest.TestCase): From 600ca3bcf9d9f87129f8504286f7e2108154b3cd Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:57:26 -0500 Subject: [PATCH 02/98] renamed 'test_near' to 'test_geospatial_operators', updated added ordering checks to test --- tests/queryset.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 004c4160..c8db29a2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,11 +1070,12 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() - def test_near(self): - """Ensure that "near" queries work with and without radii. + def test_geospatial_operators(self): + """Ensure that geospatial queries are working. """ class Event(Document): title = StringField() + date = DateTimeField() location = GeoPointField() def __unicode__(self): @@ -1085,10 +1086,13 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", + date=datetime.now() - timedelta(days=1), location=[41.909889, -87.677137]) event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + date=datetime.now() - timedelta(days=10), location=[37.7749295, -122.4194155]) event3 = Event(title="Coltrane Motion @ Empty Bottle", + date=datetime.now(), location=[41.900474, -87.686638]) event1.save() @@ -1111,6 +1115,12 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(event1 in events) self.assertTrue(event3 in events) + # ensure ordering is respected by "near" + events = Event.objects(location__near=[41.9120459, -87.67892]) + events = events.order_by("-date") + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event3, event1, event2]) + # find events around san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -1122,6 +1132,13 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 0) + # ensure ordering is respected by "within_distance" + point_and_distance = [[41.9120459, -87.67892], 10] + events = Event.objects(location__within_distance=point_and_distance) + events = events.order_by("-date") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0], event3) + Event.drop_collection() From 0ad343484f932722bdbe5b00b68e2659b8c0ddc4 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Wed, 2 Jun 2010 20:53:39 +0100 Subject: [PATCH 03/98] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22aa..4fff317a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..7d9c47f4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e264..e22e6ddd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 39b749432a7fe11a899ad3602038217c6301a8dd Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 08:27:21 +0100 Subject: [PATCH 04/98] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..7b8dcd5b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..b6d5a63b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f4..a276399e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6ddd..8dddcb3e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 86e2797c577d007b1883164867eaca7548703ea9 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 9 Jun 2010 22:28:30 -0300 Subject: [PATCH 05/98] added a TestCase for tests that uses mongoDB --- mongoengine/django/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 mongoengine/django/tests.py diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py new file mode 100644 index 00000000..a8d7c7ff --- /dev/null +++ b/mongoengine/django/tests.py @@ -0,0 +1,21 @@ +#coding: utf-8 +from django.test import TestCase +from django.conf import settings + +from mongoengine import connect + +class MongoTestCase(TestCase): + """ + TestCase class that clear the collection between the tests + """ + db_name = 'test_%s' % settings.MONGO_DATABASE_NAME + def __init__(self, methodName='runtest'): + self.db = connect(self.db_name) + super(MongoTestCase, self).__init__(methodName) + + def _post_teardown(self): + super(MongoTestCase, self)._post_teardown() + for collection in self.db.collection_names(): + if collection == 'system.indexes': + continue + self.db.drop_collection(collection) From f5e39c0064e52c03194e09a72c204c409a9ab5b2 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 10:25:31 +0200 Subject: [PATCH 06/98] Allowed _id to be missing when converting to mongo --- mongoengine/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..fe942b10 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -407,11 +407,15 @@ class BaseDocument(object): value = getattr(self, field_name, None) if value is not None: data[field.db_field] = field.to_mongo(value) + else: + data[field.db_field] = None # Only add _cls and _types if allow_inheritance is not False if not (hasattr(self, '_meta') and self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] + if hasattr(self, '_id') and not data['_id']: + del data['_id'] return data @classmethod From 3179c4e4aca20cdaa3daafb7cbc1204d847e2e2f Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 11:25:49 +0200 Subject: [PATCH 07/98] Now only removes _id if none, for real --- mongoengine/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index fe942b10..253c758a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -414,8 +414,11 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if hasattr(self, '_id') and not data['_id']: - del data['_id'] + try: + if not data['_id']: + del data['_id'] + except KeyError: + pass return data @classmethod From b89d71bfa5ffea3d3aff6c23ab0be4bcfefc930e Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 14:17:30 +0200 Subject: [PATCH 08/98] Do not convert None objects --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 253c758a..8137eddf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -450,7 +450,8 @@ class BaseDocument(object): for field_name, field in cls._fields.items(): if field.db_field in data: - data[field_name] = field.to_python(data[field.db_field]) + value = data[field.db_field] + data[field_name] = value if value is None else field.to_python(value) obj = cls(**data) obj._present_fields = present_fields From 71689fcf234ec3cc9349850de4e4afd7af574650 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:00:46 +0100 Subject: [PATCH 09/98] Got within_box working for Geo fields --- docs/changelog.rst | 14 ++++++++++++++ mongoengine/queryset.py | 2 ++ tests/queryset.py | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 479ea21c..6f2d6f1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog ========= +Changes in v0.4 +=============== +- Added ``SortedListField`` +- Added ``EmailField`` +- Added ``GeoLocationField`` +- Added ``exact`` and ``iexact`` match operators to ``QuerySet`` +- Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts +- Fixed bug in Q-objects +- Fixed document inheritance primary key issue +- Base class can now be defined for ``DictField`` +- Fixed MRO error that occured on document inheritance +- Introduced ``min_length`` for ``StringField`` +- Other minor fixes + Changes in v0.3 =============== - Added MapReduce support diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a837c4f..4840b537 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -351,6 +351,8 @@ class QuerySet(object): value = {'$within': {'$center': value}} elif op == "near": value = {'$near': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) diff --git a/tests/queryset.py b/tests/queryset.py index a7719b88..a84c8c50 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1232,6 +1232,12 @@ class QuerySetTest(unittest.TestCase): events = events.order_by("-date") self.assertEqual(events.count(), 2) self.assertEqual(events[0], event3) + + # check that within_box works + box = [(35.0, -125.0), (40.0, -100.0)] + events = Event.objects(location__within_box=box) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event2.id) Event.drop_collection() From c2163ecee5674e4379d6db7ad6b46254e59e2ee4 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:12:14 +0100 Subject: [PATCH 10/98] Added test for Geo indexes --- docs/changelog.rst | 2 +- tests/fields.py | 18 ++++++++++++++++++ tests/queryset.py | 2 -- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f2d6f1a..8dd5b00d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changes in v0.4 =============== - Added ``SortedListField`` - Added ``EmailField`` -- Added ``GeoLocationField`` +- Added ``GeoPointField`` - Added ``exact`` and ``iexact`` match operators to ``QuerySet`` - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts - Fixed bug in Q-objects diff --git a/tests/fields.py b/tests/fields.py index 4050e264..80ce3b67 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -607,6 +607,24 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_geo_indexes(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + Event.drop_collection() + event = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event.save() + + info = Event.objects._collection.index_information() + self.assertTrue(u'location_2d' in info) + self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + + Event.drop_collection() + if __name__ == '__main__': diff --git a/tests/queryset.py b/tests/queryset.py index a84c8c50..4187d550 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1175,8 +1175,6 @@ class QuerySetTest(unittest.TestCase): def __unicode__(self): return self.title - meta = {'geo_indexes': ["location"]} - Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", From acbc741037c00736ce1dbdcd39bd77ddf6f663b3 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Thu, 15 Jul 2010 18:20:29 +0200 Subject: [PATCH 11/98] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..6bfcff9f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -261,6 +261,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 0512dd4c25e0f8395c317725fa149a64025faf3e Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 03:53:39 +0800 Subject: [PATCH 12/98] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22aa..4fff317a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..7d9c47f4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e264..e22e6ddd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 6bfd6c322b166d55eb151de33e9eaee7ca214b8a Mon Sep 17 00:00:00 2001 From: martin Date: Fri, 18 Jun 2010 10:41:23 +0800 Subject: [PATCH 13/98] Fixed bug with GeoLocationField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f4..5893049d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -388,7 +388,7 @@ class GeoLocationField(DictField): return {'x': value[0], 'y': value[1]} def to_python(self, value): - return value.keys() + return (value['x'], value['y']) class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on From 47bfeec115a297c1a859543e842797751ab9c14a Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 15:27:21 +0800 Subject: [PATCH 14/98] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..7b8dcd5b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..b6d5a63b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5893049d..ebefbb75 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6ddd..8dddcb3e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 9596a25bb92212806e98fd77b6ced1b79c2ed1bf Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 00:56:16 +0200 Subject: [PATCH 15/98] Fixed documentation bug. --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..1fd2ed57 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -174,7 +174,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset(published=True).filter(published=True) + return queryset.filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From f9057e1a288dea4b99ac1936a75696071476870d Mon Sep 17 00:00:00 2001 From: martin Date: Thu, 24 Jun 2010 00:56:51 +0800 Subject: [PATCH 16/98] Fixed bug in FileField, proxy was not getting the grid_id set --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ebefbb75..d2b41ab9 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -542,6 +542,7 @@ class GridFSProxy(object): return self def get(self, id=None): + if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: return None # File has been deleted @@ -611,7 +612,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() + return self.gridfs.get(id=value) def validate(self, value): assert isinstance(value, GridFSProxy) From ec519f20fa8e1c769ffb6ae766dd00bb36f9eabc Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 01:32:28 +0200 Subject: [PATCH 17/98] Makes the tests compatible to pymongo 1.7+. Not backwards compatible! --- tests/document.py | 14 ++++++++------ tests/fields.py | 2 +- tests/queryset.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/document.py b/tests/document.py index 8bc907c5..1160b353 100644 --- a/tests/document.py +++ b/tests/document.py @@ -264,11 +264,12 @@ class DocumentTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info.values()) + self.assertTrue([('tags', 1)] in info) class ExtendedBlogPost(BlogPost): title = StringField() @@ -278,10 +279,11 @@ class DocumentTest(unittest.TestCase): list(ExtendedBlogPost.objects) info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) - self.assertTrue([('_types', 1), ('title', 1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) + self.assertTrue([('_types', 1), ('title', 1)] in info) BlogPost.drop_collection() diff --git a/tests/fields.py b/tests/fields.py index d95f4d3f..136437b8 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -689,7 +689,7 @@ class FieldTest(unittest.TestCase): info = Event.objects._collection.index_information() self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) Event.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index 4187d550..0424d323 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1087,8 +1087,9 @@ class QuerySetTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() - self.assertTrue([('_types', 1)] in info.values()) - self.assertTrue([('_types', 1), ('date', -1)] in info.values()) + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1)] in info) + self.assertTrue([('_types', 1), ('date', -1)] in info) BlogPost.drop_collection() From 6093e88eebf932d04741364f53de77878198c1f1 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Fri, 16 Jul 2010 00:20:29 +0800 Subject: [PATCH 18/98] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 24c5b569..8e8a97c8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,6 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 03c0fd9ada5eb1c90a9213fe4579e64d9a6a111c Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 19:01:53 +0200 Subject: [PATCH 19/98] Make default value of DictField an empty dict instead of None. --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8e8a97c8..759697ac 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault("default", []) + kwargs.setdefault('default', []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,6 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) + kwargs.setdefault('default', {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From aa00feb6a55eba7f720969828242436103817e64 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 20 Jul 2010 22:46:00 +0200 Subject: [PATCH 20/98] FileField's values are now optional. When no value is applied, no File object is created and referenced. --- mongoengine/base.py | 4 ++-- mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 806d83bb..086c7874 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,8 +339,8 @@ class BaseDocument(object): try: field._validate(value) except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field of type "' + - field.__class__.__name__ + '"') + raise ValidationError('Invalid value for field of type "%s": %s' + % (field.__class__.__name__, value)) elif field.required: raise ValidationError('Field "%s" is required' % field.name) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 759697ac..f84f751b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -591,15 +591,20 @@ class FileField(BaseField): def to_mongo(self, value): # Store the GridFS file id in MongoDB - return self.gridfs.grid_id + if self.gridfs.grid_id is not None: + return self.gridfs.grid_id + return None def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get(id=value) + if self.gridfs.grid_id is not None: + return self.gridfs.get(id=value) + return None def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + if value.grid_id is not None: + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) class GeoPointField(BaseField): """A list storing a latitude and longitude. From be651caa688c31b434d641c270e1bf073b3c576a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:02:37 +0100 Subject: [PATCH 21/98] Removed a couple of sneaky print statements --- mongoengine/queryset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4840b537..f81adb5c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -114,13 +114,11 @@ class Q(object): value, field_js = self._build_op_js(op, key, value, value_name) js_scope[value_name] = value js.append(field_js) - print ' && '.join(js) return ' && '.join(js) def _build_op_js(self, op, key, value, value_name): """Substitute the values in to the correct chunk of Javascript. """ - print op, key, value, value_name if isinstance(value, RE_TYPE): # Regexes are handled specially if op.strip('$') == 'ne': From 9f98025b8c13d9e845f5de68e29e825f5517b245 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:29:02 +0100 Subject: [PATCH 22/98] Added QuerySet.distinct. Closes #44. --- mongoengine/queryset.py | 9 +++++++++ tests/queryset.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f81adb5c..1e42cd1d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -604,6 +604,15 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) + + def distinct(self, field): + """Return a list of distinct values for a given field. + + :param field: the field to select distinct values from + + .. versionadded:: 0.4 + """ + return self._collection.distinct(field) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index 0424d323..3691d89e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -970,6 +970,16 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + def test_distinct(self): + """Ensure that the QuerySet.distinct method works. + """ + self.Person(name='Mr Orange', age=20).save() + self.Person(name='Mr White', age=20).save() + self.Person(name='Mr Orange', age=30).save() + self.assertEqual(self.Person.objects.distinct('name'), + ['Mr Orange', 'Mr White']) + self.assertEqual(self.Person.objects.distinct('age'), [20, 30]) + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From 13316e5380f9886b0d8c65458096311b8382233a Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 17:35:09 +0200 Subject: [PATCH 23/98] Introduced new Document.objects.create, like django has. It creates a new object, saves it and returns the new object instance. --- mongoengine/queryset.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1e42cd1d..e176c54c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,5 +1,6 @@ from connection import _get_db +import pprint import pymongo import re import copy @@ -414,6 +415,14 @@ class QuerySet(object): message = u'%d items returned, instead of 1' % count raise self._document.MultipleObjectsReturned(message) + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + .. versionadded:: 0.4 + """ + doc = self._document(**kwargs) + doc.save() + return doc + def first(self): """Retrieve the first object matching the query. """ @@ -667,7 +676,6 @@ class QuerySet(object): plan = self._cursor.explain() if format: - import pprint plan = pprint.pformat(plan) return plan From 327452622e3081c81b05916c61454a9fd97d992d Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:22:26 +0200 Subject: [PATCH 24/98] Handle DBRefs correctly within Q objects. Closes #55 --- mongoengine/queryset.py | 11 +++++++++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e176c54c..00a7f7a2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -133,6 +133,17 @@ class Q(object): if isinstance(value, pymongo.objectid.ObjectId): value = unicode(value) + # Handle DBRef + if isinstance(value, pymongo.dbref.DBRef): + # this.created_user.$id == "4c4c56f8cc1831418c000000" + op_js = '(this.%(field)s.$id == "%(id)s" &&'\ + ' this.%(field)s.$ref == "%(ref)s")' % { + 'field': key, + 'id': unicode(value.id), + 'ref': unicode(value.collection) + } + value = None + # Perform the substitution operation_js = op_js % { 'field': key, diff --git a/tests/queryset.py b/tests/queryset.py index 3691d89e..8cbd9a40 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1304,6 +1304,20 @@ class QTest(unittest.TestCase): query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) + + def test_q_with_dbref(self): + """Ensure Q objects handle DBRefs correctly""" + class User(Document): + pass + + class Post(Document): + created_user = ReferenceField(User) + + user = User.objects.create() + Post.objects.create(created_user=user) + + self.assertEqual(Post.objects.filter(created_user=user).count(), 1) + self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) if __name__ == '__main__': unittest.main() From 51065e7a4dda96943dadfcbdfcca6111dfa1f9f5 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:33:33 +0200 Subject: [PATCH 25/98] Closes #46 by instantiating a new default instance for every field by request. --- mongoengine/base.py | 5 ++++- mongoengine/fields.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 086c7874..10ff1219 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,7 +52,10 @@ class BaseField(object): # Get value from document instance if available, if not use default value = instance._data.get(self.name) if value is None: - value = self.default + if callable(self.default): # fixes #46 + value = self.default() + else: + value = self.default # Allow callable default values if callable(value): value = value() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..670e3cd3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From 9d82911f6338747411788291175739f436d95709 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:38:24 +0200 Subject: [PATCH 26/98] Added tests for #46. --- tests/fields.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 386c48b116191a754e3cec550ccdbd5246efdc97 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:43:11 +0200 Subject: [PATCH 27/98] Typo. --- tests/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields.py b/tests/fields.py index ef776d4a..6e42ea4d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -694,7 +694,7 @@ class FieldTest(unittest.TestCase): Event.drop_collection() def test_ensure_unique_default_instances(self): - """Ensure that every document has it's own unique default instance.""" + """Ensure that every field has it's own unique default instance.""" class D(Document): data = DictField() data2 = DictField(default=lambda: {}) From 9411b38508ac261f166dac1f8f8adc7f01084334 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:45:49 +0200 Subject: [PATCH 28/98] Removed unnecessary comment. --- mongoengine/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 00a7f7a2..be29ae83 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -135,7 +135,6 @@ class Q(object): # Handle DBRef if isinstance(value, pymongo.dbref.DBRef): - # this.created_user.$id == "4c4c56f8cc1831418c000000" op_js = '(this.%(field)s.$id == "%(id)s" &&'\ ' this.%(field)s.$ref == "%(ref)s")' % { 'field': key, From 2f991ac6f18172008ed71f1d97ab6ee71ea360ba Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 19:02:15 +0200 Subject: [PATCH 29/98] Added all() method to get all document instances from a document. Extended the FileField's tests with testing on empty filefield. --- mongoengine/queryset.py | 4 ++++ tests/fields.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index be29ae83..ffbe5255 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -238,6 +238,10 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) + + def all(self): + """Returns all documents.""" + return self.__call__() @property def _collection(self): diff --git a/tests/fields.py b/tests/fields.py index 6e42ea4d..1e53d23d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -674,7 +674,12 @@ class FieldTest(unittest.TestCase): PutFile.drop_collection() StreamFile.drop_collection() SetFile.drop_collection() - + + # Make sure FileField is optional and not required + class DemoFile(Document): + file = FileField() + d = DemoFile.objects.create() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 7ab2e21c106fc89875b1ae4ae79ecd7e0988ded3 Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:42:10 +0200 Subject: [PATCH 30/98] Handle unsafe expressions when using startswith/endswith/contains with unsafe expressions. Closes #58 --- mongoengine/fields.py | 3 +++ tests/queryset.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 670e3cd3..bd81d3a8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,6 +66,9 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' + + # escape unsafe characters which could lead to a re.error + value = re.escape(value) value = re.compile(regex % value, flags) return value diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..1efd034c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -288,6 +288,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() self.assertEqual(obj, None) + + # Test unsafe expressions + person = self.Person(name='Guido van Rossum [.\'Geek\']') + person.save() + + obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. From 6791f205af3d15a0f4ccaf357692ba868b6ebfff Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:50:09 +0200 Subject: [PATCH 31/98] Style fix. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bd81d3a8..30a11f20 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,7 +66,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - + # escape unsafe characters which could lead to a re.error value = re.escape(value) value = re.compile(regex % value, flags) From 21d267cb112e9030a0f17271fecd052dbe35b50a Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 17:28:59 +0200 Subject: [PATCH 32/98] Now order_by() works like queries for referencing deeper fields (replacing . with __). old: order_by('mydoc.myattr') / new: order_by('mydoc__myattr'). Closes #45 --- mongoengine/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ffbe5255..0b9218af 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -670,11 +670,13 @@ class QuerySet(object): """ key_list = [] for key in keys: + if not key: continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING if key[0] in ('-', '+'): key = key[1:] + key = key.replace('__', '.') key_list.append((key, direction)) self._ordering = key_list From 1147ac43506283a37554d09242894e9088f2133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 23 May 2010 00:19:50 +0800 Subject: [PATCH 33/98] ignore virtualenv directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42dcc6e6..57cf1b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ docs/.build docs/_build build/ dist/ -mongoengine.egg-info/ \ No newline at end of file +mongoengine.egg-info/ +env/ \ No newline at end of file From b96e27a7e4675c0564cff05bcba66f7662b6099d Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Fri, 30 Jul 2010 22:09:00 +1000 Subject: [PATCH 34/98] Allow documents to override the 'objects' QuerySetManager --- mongoengine/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..ed8214af 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - new_class.objects = QuerySetManager() + + # Provide a default queryset unless one has been manually provided + if not 'objects' in dir(new_class): + new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes From 198ccc028a9d24e7879db583f8a263a061255742 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 6 Aug 2010 20:29:09 +1000 Subject: [PATCH 35/98] made list queries work with regexes (e.g. istartswith) --- mongoengine/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..73bc7562 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -319,8 +319,8 @@ class ListField(BaseField): def prepare_query_value(self, op, value): if op in ('set', 'unset'): - return [self.field.to_mongo(v) for v in value] - return self.field.to_mongo(value) + return [self.field.prepare_query_value(op, v) for v in value] + return self.field.prepare_query_value(op, value) def lookup_member(self, member_name): return self.field.lookup_member(member_name) From 809fe44b43decc6f271d3eedd1f637db4149f24c Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Thu, 12 Aug 2010 15:14:20 +1000 Subject: [PATCH 36/98] Added a __raw__ parameter for passing query dictionaries directly to pymongo --- mongoengine/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab113..b2460217 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -318,6 +318,11 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): + + if key == "__raw__": + mongo_query.update(value) + return mongo_query + parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None From 6373e20696533630979327ed0183ea2cbbc6cc48 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 13 Aug 2010 22:28:26 +1000 Subject: [PATCH 37/98] Better error reporting on a validation error for a list. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 73bc7562..866dc7e3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -315,7 +315,7 @@ class ListField(BaseField): try: [self.field.validate(item) for item in value] except Exception, err: - raise ValidationError('Invalid ListField item (%s)' % str(err)) + raise ValidationError('Invalid ListField item (%s)' % str(item)) def prepare_query_value(self, op, value): if op in ('set', 'unset'): From d274576b4739d77803cc4adfda5e069be865d909 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 13 Aug 2010 22:30:36 +1000 Subject: [PATCH 38/98] Fixed premature return for query gen --- mongoengine/queryset.py | 57 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b2460217..3f9b7c5c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -321,40 +321,39 @@ class QuerySet(object): if key == "__raw__": mongo_query.update(value) - return mongo_query + else: + parts = key.split('__') + # Check for an operator and transform to mongo-style if there is + op = None + if parts[-1] in operators + match_operators: + op = parts.pop() - parts = key.split('__') - # Check for an operator and transform to mongo-style if there is - op = None - if parts[-1] in operators + match_operators: - op = parts.pop() + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + fields = QuerySet._lookup_field(_doc_cls, parts) + parts = [field.db_field for field in fields] - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [field.db_field for field in fields] + # Convert value to proper value + field = fields[-1] + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] + singular_ops += match_operators + if op in singular_ops: + value = field.prepare_query_value(op, value) + elif op in ('in', 'nin', 'all'): + # 'in', 'nin' and 'all' require a list of values + value = [field.prepare_query_value(op, v) for v in value] - # Convert value to proper value - field = fields[-1] - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] - singular_ops += match_operators - if op in singular_ops: - value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all'): - # 'in', 'nin' and 'all' require a list of values - value = [field.prepare_query_value(op, v) for v in value] + if field.__class__.__name__ == 'GenericReferenceField': + parts.append('_ref') - if field.__class__.__name__ == 'GenericReferenceField': - parts.append('_ref') + if op and op not in match_operators: + value = {'$' + op: value} - if op and op not in match_operators: - value = {'$' + op: value} - - key = '.'.join(parts) - if op is None or key not in mongo_query: - mongo_query[key] = value - elif key in mongo_query and isinstance(mongo_query[key], dict): - mongo_query[key].update(value) + key = '.'.join(parts) + if op is None or key not in mongo_query: + mongo_query[key] = value + elif key in mongo_query and isinstance(mongo_query[key], dict): + mongo_query[key].update(value) return mongo_query From 17addbefe2e7befe45edcc764add45c397d4202b Mon Sep 17 00:00:00 2001 From: sp Date: Wed, 18 Aug 2010 16:50:52 -0400 Subject: [PATCH 39/98] made it more like Django's user model --- mongoengine/django/auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d4b0ff0b..da0005c8 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -32,6 +32,9 @@ class User(Document): last_login = DateTimeField(default=datetime.datetime.now) date_joined = DateTimeField(default=datetime.datetime.now) + def __unicode__(self): + return self.username + def get_full_name(self): """Returns the users first and last names, separated by a space. """ From 7de9adc6b1677b5e9c2b7b2b6d5b670effdd5ead Mon Sep 17 00:00:00 2001 From: Richard Henry Date: Sat, 28 Aug 2010 09:16:02 +0100 Subject: [PATCH 40/98] Adding support for pop operations to QuerySet.update and QuerySet.update_one --- docs/guide/querying.rst | 1 + mongoengine/queryset.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..7de6c5c8 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -399,6 +399,7 @@ that you may use with these methods: * ``unset`` -- delete a particular value (since MongoDB v1.3+) * ``inc`` -- increment a value by a given amount * ``dec`` -- decrement a value by a given amount +* ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list * ``pull`` -- remove a value from a list diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab113..182dfd52 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -661,8 +661,8 @@ class QuerySet(object): def _transform_update(cls, _doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ - operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', + 'pull', 'pull_all'] mongo_update = {} for key, value in update.items(): @@ -688,7 +688,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'pop', 'push', 'pull'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From d99c5973c3cb6da18670c6592ff396d18bb0e946 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 12:52:24 +0100 Subject: [PATCH 41/98] Fixed Q object DBRef test bug --- tests/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..b1f16576 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,8 @@ class QTest(unittest.TestCase): def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + class User(Document): pass From 3b62cf80cde0b83715662ce9bb01053dc13a7060 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:00:34 +0100 Subject: [PATCH 42/98] Fixed {Dict,List}Field default issue. Closes #46. --- mongoengine/fields.py | 6 +++--- tests/fields.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..d8d310dd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): @@ -623,4 +623,4 @@ class GeoPointField(BaseField): raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int.') \ No newline at end of file + raise ValidationError('Both values in point must be float or int.') diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 4fb6fcabefa04f9ab4b410ce1cf3b6293e0ec52c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:54:20 +0100 Subject: [PATCH 43/98] Added test for overriding objects --- tests/queryset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/queryset.py b/tests/queryset.py index b339d613..0c7ba05a 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -992,10 +992,15 @@ class QuerySetTest(unittest.TestCase): """ class BlogPost(Document): tags = ListField(StringField()) + deleted = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(deleted=False) @queryset_manager def music_posts(doc_cls, queryset): - return queryset(tags='music') + return queryset(tags='music', deleted=False) BlogPost.drop_collection() @@ -1005,6 +1010,8 @@ class QuerySetTest(unittest.TestCase): post2.save() post3 = BlogPost(tags=['film', 'actors']) post3.save() + post4 = BlogPost(tags=['film', 'actors'], deleted=True) + post4.save() self.assertEqual([p.id for p in BlogPost.objects], [post1.id, post2.id, post3.id]) From 95efa39b5229081deef179ae4b08299a50b18ccb Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:56:18 +0200 Subject: [PATCH 44/98] Added *.egg to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57cf1b7b..51a9ca1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*.egg docs/.build docs/_build build/ From e0911a5fe00d77bb74b0b8b2d8e5d6d963afc687 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:58:58 +0200 Subject: [PATCH 45/98] Replaced slow exception handling with has_key. --- mongoengine/base.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4523a268..e0e71d31 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -415,11 +415,8 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - try: - if not data['_id']: - del data['_id'] - except KeyError: - pass + if data.has_key('_id') and not data['_id']: + del data['_id'] return data @classmethod From 1ed9a36d0a15eef4038568ccc974d14bfb425c42 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 14:02:02 +0100 Subject: [PATCH 46/98] Added test for pop operation --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 0c7ba05a..2f86953d 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -671,6 +671,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + tags = post.tags[:-1] + BlogPost.objects.update(pop__tags=1) + post.reload() + self.assertEqual(post.tags, tags) + BlogPost.drop_collection() def test_update_pull(self): From 5b230b90b984b20d09322d17dd6b43b93db050f7 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 15:34:29 +0100 Subject: [PATCH 47/98] Doc fix (all operator) --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 86563082..bef19bc5 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -71,7 +71,7 @@ Available operators are as follows: * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values -* ``all`` -- every item in array is in list of values provided +* ``all`` -- every item in list of values provided is in array * ``size`` -- the size of the array is * ``exists`` -- value for field exists From 185e7a6a7e27884ac4896bf5c6a6881db8d419a0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 18:38:41 +0200 Subject: [PATCH 48/98] Better way of checking if new_class has an 'objects' attribute. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3172e9a3..9505d416 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -257,7 +257,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) # Provide a default queryset unless one has been manually provided - if not 'objects' in dir(new_class): + if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) From dcc8d22cec6d59392c8470a9325d0d8fedc6d4fa Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 16:22:56 -0700 Subject: [PATCH 49/98] Proper unique index name when using db_field. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9505d416..92a450de 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -269,7 +269,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True - unique_fields = [field_name] + unique_fields = [field.db_field] # Add any unique_with fields to the back of the index spec if field.unique_with: From 266f33adc4730859c272d9a8233769b0d9e54205 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 19:16:54 -0700 Subject: [PATCH 50/98] Bug fix for gridfs FileField. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index dc5dbe0e..49e9d056 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -600,7 +600,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - if self.gridfs.grid_id is not None: + if value is not None: return self.gridfs.get(id=value) return None From 3e30d71263356082ef5d391a2ea614d74790bb32 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:25:07 -0700 Subject: [PATCH 51/98] Support for background and drop_dups indexing options. --- mongoengine/queryset.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dd7fcef6..7969c196 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -172,7 +172,8 @@ class QuerySet(object): self._limit = None self._skip = None - def ensure_index(self, key_or_list): + def ensure_index(self, key_or_list, drop_dups=False, background=False, + **kwargs): """Ensure that the given indexes are in place. :param key_or_list: a single index key or a list of index keys (to @@ -180,7 +181,8 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_list = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index(index_list) + self._collection.ensure_index(index_list, drop_dups=drop_dups, + background=background) return self @classmethod @@ -250,25 +252,33 @@ class QuerySet(object): """ if not self._accessed_collection: self._accessed_collection = True + + background = self._document._meta.get('index_background', False) + drop_dups = self._document._meta.get('index_drop_dups', False) + index_opts = self._document._meta.get('index_options', {}) # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - self._collection.ensure_index(key_or_list) + self._collection.ensure_index(key_or_list, + background=background, **index_opts) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - self._collection.ensure_index(index, unique=True) + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: - self._collection.ensure_index('_types') + self._collection.ensure_index('_types', + background=background, **index_opts) # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec) + self._collection.ensure_index(index_spec, + background=background, **index_opts) return self._collection_obj From f1aec68f239d77edff296e05764c419c4187b2d0 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:46:44 -0700 Subject: [PATCH 52/98] Inherit index options. --- mongoengine/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92a450de..836817da 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -230,12 +230,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_field = None base_indexes = [] + base_meta = {} # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta') and 'collection' in base._meta: collection = base._meta['collection'] + # Propagate index options. + for key in ('index_background', 'index_drop_dups', 'index_opts'): + if key in base._meta: + base_meta[key] = base._meta[key] + id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) @@ -246,7 +252,11 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime 'id_field': id_field, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': {}, } + meta.update(base_meta) # Apply document-defined meta options meta.update(attrs.get('meta', {})) From 17642c8a8cc2e37741a1d10be04d704770679193 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:48:17 +0100 Subject: [PATCH 53/98] Fixed QuerySet.average issue that ignored 0 --- mongoengine/queryset.py | 2 +- tests/queryset.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7969c196..662fa8c3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -892,7 +892,7 @@ class QuerySet(object): var total = 0.0; var num = 0; db[collection].find(query).forEach(function(doc) { - if (doc[averageField]) { + if (doc[averageField] !== undefined) { total += doc[averageField]; num += 1; } diff --git a/tests/queryset.py b/tests/queryset.py index 2f86953d..0c6c3ca4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -960,11 +960,14 @@ class QuerySetTest(unittest.TestCase): def test_average(self): """Ensure that field can be averaged correctly. """ + self.Person(name='person', age=0).save() + self.assertEqual(int(self.Person.objects.average('age')), 0) + ages = [23, 54, 12, 94, 27] for i, age in enumerate(ages): self.Person(name='test%s' % i, age=age).save() - avg = float(sum(ages)) / len(ages) + avg = float(sum(ages)) / (len(ages) + 1) # take into account the 0 self.assertAlmostEqual(int(self.Person.objects.average('age')), avg) self.Person(name='ageless person').save() @@ -1340,5 +1343,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + if __name__ == '__main__': unittest.main() From 69012e8ad11a7e4312450ff71fc80445e007270f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:59:49 +0100 Subject: [PATCH 54/98] Fixed incorrect $pull test --- tests/queryset.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 0c6c3ca4..e3912246 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -681,23 +681,17 @@ class QuerySetTest(unittest.TestCase): def test_update_pull(self): """Ensure that the 'pull' update operation works correctly. """ - class Comment(EmbeddedDocument): - content = StringField() - class BlogPost(Document): slug = StringField() - comments = ListField(EmbeddedDocumentField(Comment)) + tags = ListField(StringField()) - comment1 = Comment(content="test1") - comment2 = Comment(content="test2") - - post = BlogPost(slug="test", comments=[comment1, comment2]) + post = BlogPost(slug="test", tags=['code', 'mongodb', 'code']) post.save() - self.assertTrue(comment2 in post.comments) - BlogPost.objects(slug="test").update(pull__comments__content="test2") + BlogPost.objects(slug="test").update(pull__tags="code") post.reload() - self.assertTrue(comment2 not in post.comments) + self.assertTrue('code' not in post.tags) + self.assertEqual(len(post.tags), 1) def test_order_by(self): """Ensure that QuerySets may be ordered. From 32e66b29f44f3015be099851201241caee92054f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 22:12:05 +0100 Subject: [PATCH 55/98] Fixed FileField problem caused by shared objects --- mongoengine/fields.py | 50 ++++++++++++++++++++++++++++++------------- tests/fields.py | 22 +++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 49e9d056..ffcfb53d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -510,14 +510,15 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ - def __init__(self): + def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file + self.grid_id = grid_id # Store GridFS id for file def __getattr__(self, name): obj = self.get() @@ -528,9 +529,12 @@ class GridFSProxy(object): return self def get(self, id=None): - if id: self.grid_id = id - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted + if id: + self.grid_id = id + try: + return self.fs.get(id or self.grid_id) + except: + return None # File has been deleted def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -552,8 +556,10 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - try: return self.get().read() - except: return None + try: + return self.get().read() + except: + return None def delete(self): # Delete file from GridFS, FileField still remains @@ -571,38 +577,52 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) + class FileField(BaseField): """A GridFS storage field. """ def __init__(self, **kwargs): - self.gridfs = GridFSProxy() super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self - return self.gridfs + # Check if a file already exists for this model + grid_file = instance._data.get(self.name) + if grid_file: + return grid_file + return GridFSProxy() def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation - self.gridfs.put(value) + grid_file = instance._data.get(self.name) + # If a file already exists, delete it + if grid_file: + try: + grid_file.delete() + except: + pass + # Create a new file with the new data + grid_file.put(value) + else: + # Create a new proxy object as we don't already have one + instance._data[self.name] = GridFSProxy() + instance._data[self.name].put(value) else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB - if self.gridfs.grid_id is not None: - return self.gridfs.grid_id + if isinstance(value, GridFSProxy) and value.grid_id is not None: + return value.grid_id return None def to_python(self, value): - # Use stored value (id) to lookup file in GridFS if value is not None: - return self.gridfs.get(id=value) - return None + return GridFSProxy(value) def validate(self, value): if value.grid_id is not None: diff --git a/tests/fields.py b/tests/fields.py index 1e53d23d..8c727196 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -680,6 +680,28 @@ class FieldTest(unittest.TestCase): file = FileField() d = DemoFile.objects.create() + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + file = FileField() + + # First instance + testfile = TestFile() + testfile.name = "Hello, World!" + testfile.file.put('Hello, World!') + testfile.save() + + # Second instance + testfiledupe = TestFile() + data = testfiledupe.file.read() # Should be None + + self.assertTrue(testfile.name != testfiledupe.name) + self.assertTrue(testfile.file.read() != data) + + TestFile.drop_collection() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 1849f75ad04a8a50c73862a6d0794fe65016343c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:23:59 +0100 Subject: [PATCH 56/98] Made GridFSProxy a bit stricter / safer --- mongoengine/fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ffcfb53d..418f57cc 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -511,6 +511,10 @@ class BinaryField(BaseField): raise ValidationError('Binary value is too long') +class GridFSError(Exception): + pass + + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ @@ -541,12 +545,18 @@ class GridFSProxy(object): self.grid_id = self.newfile._id def put(self, file, **kwargs): + if self.grid_id: + raise GridFSError('This document alreay has a file. Either delete ' + 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): - if not self.newfile: + if self.grid_id: + if not self.newfile: + raise GridFSError('This document alreay has a file. Either ' + 'delete it or call replace to overwrite it') + else: self.new_file() - self.grid_id = self.newfile._id self.newfile.write(string) def writelines(self, lines): From 2af5f3c56ebc7feff6a0a1cd95a77e12b425efed Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:24:30 +0100 Subject: [PATCH 57/98] Added support for querying by array position. Closes #36. --- mongoengine/queryset.py | 6 +++++- tests/queryset.py | 43 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c3..b8ca125d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -344,6 +344,8 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if parts[-1] in operators + match_operators + geo_operators: @@ -381,7 +383,9 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + + for i, part in indices: + parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value diff --git a/tests/queryset.py b/tests/queryset.py index e3912246..3d714be2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -165,8 +165,49 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") + def test_find_array_position(self): + """Ensure that query by array position works. + """ + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() - + Blog.objects.create(tags=['a', 'b']) + self.assertEqual(len(Blog.objects(tags__0='a')), 1) + self.assertEqual(len(Blog.objects(tags__0='b')), 0) + self.assertEqual(len(Blog.objects(tags__1='a')), 0) + self.assertEqual(len(Blog.objects(tags__1='b')), 1) + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + blog = Blog.objects(posts__0__comments__0__name='testa').get() + self.assertEqual(blog, blog1) + + query = Blog.objects(posts__1__comments__1__name='testb') + self.assertEqual(len(query), 2) + + query = Blog.objects(posts__1__comments__1__name='testa') + self.assertEqual(len(query), 0) + + query = Blog.objects(posts__0__comments__1__name='testa') + self.assertEqual(len(query), 0) + + Blog.drop_collection() def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new From 449f5a00dccce95b3efebd59438ee981ddfb7435 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Sat, 11 Sep 2010 17:45:57 +0200 Subject: [PATCH 58/98] added a 'validate' option to Document.save() +docs +tests --- docs/guide/defining-documents.rst | 14 ++++++++++++++ mongoengine/document.py | 6 ++++-- tests/document.py | 10 ++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..dff3ed60 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -214,6 +214,20 @@ either a single field name, or a list or tuple of field names:: first_name = StringField() last_name = StringField(unique_with='first_name') +Skipping Document validation on save +------------------------------------ +You can also skip the whole document validation process by setting +``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` +method:: + + class Recipient(Document): + name = StringField() + email = EmailField() + + recipient = Recipient(name='admin', email='root@localhost') + recipient.save() # will raise a ValidationError while + recipient.save(validate=False) # won't + Document collections ==================== Document classes that inherit **directly** from :class:`~mongoengine.Document` diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec145..af2a5e21 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class Document(BaseDocument): __metaclass__ = TopLevelDocumentMetaclass - def save(self, safe=True, force_insert=False): + def save(self, safe=True, force_insert=False, validate=True): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -67,8 +67,10 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents + :param validate: validates the document; set to ``False`` for skiping """ - self.validate() + if validate: + self.validate() doc = self.to_mongo() try: collection = self.__class__.objects._collection diff --git a/tests/document.py b/tests/document.py index 8bc907c5..80cf3f08 100644 --- a/tests/document.py +++ b/tests/document.py @@ -446,6 +446,16 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person_obj['name'], 'Test User') self.assertEqual(person_obj['age'], 30) self.assertEqual(person_obj['_id'], person.id) + # Test skipping validation on save + class Recipient(Document): + email = EmailField(required=True) + + recipient = Recipient(email='root@localhost') + self.assertRaises(ValidationError, recipient.save) + try: + recipient.save(validate=False) + except ValidationError: + fail() def test_delete(self): """Ensure that document may be deleted using the delete method. From f11ee1f9cf60979fbb5c6768219225202d510951 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 15 Sep 2010 09:47:13 +0100 Subject: [PATCH 59/98] Added support for using custom QuerySet classes --- mongoengine/base.py | 1 + mongoengine/queryset.py | 3 ++- tests/queryset.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817da..0cbd707d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -255,6 +255,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_background': False, 'index_drop_dups': False, 'index_opts': {}, + 'queryset_class': QuerySet, } meta.update(base_meta) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125d..8b486093 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -992,7 +992,8 @@ class QuerySetManager(object): self._collection = db[collection] # owner is the document that contains the QuerySetManager - queryset = QuerySet(owner, self._collection) + queryset_class = owner._meta['queryset_class'] or QuerySet + queryset = queryset_class(owner, self._collection) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/queryset.py b/tests/queryset.py index 3d714be2..4491be8c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1378,6 +1378,26 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + if __name__ == '__main__': unittest.main() From b7e84031e310561469b99d62c99dd354e4fa6fe5 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Thu, 16 Sep 2010 14:37:18 +1000 Subject: [PATCH 60/98] Escape strings for regex query. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 866dc7e3..79045898 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -64,7 +64,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - value = re.compile(regex % value, flags) + value = re.compile(regex % re.escape(value), flags) return value From 20dd7562e0e7307f3a53c77d33521cb2f826cf5c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Thu, 16 Sep 2010 17:19:58 -0400 Subject: [PATCH 61/98] Adding multiprocessing support to mongoengine by using the identity of the process to define the connection. Each 'thread' gets its own pymongo connection. --- mongoengine/connection.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index ec3bf784..94cc6ea1 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ from pymongo import Connection - +import multiprocessing __all__ = ['ConnectionError', 'connect'] @@ -8,12 +8,12 @@ _connection_settings = { 'host': 'localhost', 'port': 27017, } -_connection = None +_connection = {} _db_name = None _db_username = None _db_password = None -_db = None +_db = {} class ConnectionError(Exception): @@ -22,32 +22,39 @@ class ConnectionError(Exception): def _get_connection(): global _connection + identity = get_identity() # Connect to the database if not already connected - if _connection is None: + if _connection.get(identity) is None: try: - _connection = Connection(**_connection_settings) + _connection[identity] = Connection(**_connection_settings) except: raise ConnectionError('Cannot connect to the database') - return _connection + return _connection[identity] def _get_db(): global _db, _connection + identity = get_identity() # Connect if not already connected - if _connection is None: - _connection = _get_connection() + if _connection.get(identity) is None: + _connection[identity] = _get_connection() - if _db is None: + if _db.get(identity) is None: # _db_name will be None if the user hasn't called connect() if _db_name is None: raise ConnectionError('Not connected to the database') # Get DB from current connection and authenticate if necessary - _db = _connection[_db_name] + _db[identity] = _connection[identity][_db_name] if _db_username and _db_password: - _db.authenticate(_db_username, _db_password) + _db[identity].authenticate(_db_username, _db_password) - return _db + return _db[identity] +def get_identity(): + identity = multiprocessing.current_process()._identity + identity = 0 if not identity else identity[0] + return identity + def connect(db, username=None, password=None, **kwargs): """Connect to the database specified by the 'db' argument. Connection settings may be provided here as well if the database is not running on From 73092dcb33ed64e64aedc3d32fa84780c6e534bb Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 17:17:37 +0100 Subject: [PATCH 62/98] QuerySet update method returns num affected docs --- mongoengine/queryset.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125d..c199d64f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -766,7 +766,8 @@ class QuerySet(object): return mongo_update def update(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on the fields matched by the query. + """Perform an atomic update on the fields matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -778,8 +779,10 @@ class QuerySet(object): update = QuerySet._transform_update(self._document, **update) try: - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=True) + ret = self._collection.update(self._query, update, multi=True, + upsert=upsert, safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' @@ -787,7 +790,8 @@ class QuerySet(object): raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on first field matched by the query. + """Perform an atomic update on first field matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -799,11 +803,14 @@ class QuerySet(object): # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' if pymongo.version >= '1.1.1': - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=False) + ret = self._collection.update(self._query, update, multi=False, + upsert=upsert, safe=safe_update) else: # Older versions of PyMongo don't support 'multi' - self._collection.update(self._query, update, safe=safe_update) + ret = self._collection.update(self._query, update, + safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, e: raise OperationError(u'Update failed [%s]' % unicode(e)) From bb2487914987972c4d7ff6960de1012d53a7f566 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 20:00:53 +0100 Subject: [PATCH 63/98] Moved custom queryset test to correct place --- tests/queryset.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 4491be8c..59a8216e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,26 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + class QTest(unittest.TestCase): @@ -1378,26 +1398,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - def test_custom_querysets(self): - """Ensure that custom QuerySet classes may be used. - """ - class CustomQuerySet(QuerySet): - def not_empty(self): - return len(self) > 0 - - class Post(Document): - meta = {'queryset_class': CustomQuerySet} - - Post.drop_collection() - - self.assertTrue(isinstance(Post.objects, CustomQuerySet)) - self.assertFalse(Post.objects.not_empty()) - - Post().save() - self.assertTrue(Post.objects.not_empty()) - - Post.drop_collection() - if __name__ == '__main__': unittest.main() From 98bc0a7c10f23c4c7582c222ec0d1432ebe5b567 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 25 Sep 2010 22:47:09 +0100 Subject: [PATCH 64/98] Raise AttributeError when necessary on GridFSProxy --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 418f57cc..87d52fd6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -528,6 +528,7 @@ class GridFSProxy(object): obj = self.get() if name in dir(obj): return getattr(obj, name) + raise AttributeError def __get__(self, instance, value): return self From b5eb3ea1cded1bb4c93e13298e86e3d85241c081 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Wed, 29 Sep 2010 23:36:58 +0100 Subject: [PATCH 65/98] Added a Django storage backend. - New GridFSStorage storage backend - New FileDocument document for storing files in GridFS - Whitespace cleaned up in various files --- docs/changelog.rst | 2 + docs/django.rst | 41 ++++++++++++- mongoengine/base.py | 28 ++++----- mongoengine/django/storage.py | 112 ++++++++++++++++++++++++++++++++++ mongoengine/document.py | 12 ++-- mongoengine/fields.py | 78 ++++++----------------- mongoengine/queryset.py | 20 +++--- tests/connnection.py | 44 +++++++++++++ tests/fields.py | 2 +- 9 files changed, 248 insertions(+), 91 deletions(-) create mode 100644 mongoengine/django/storage.py create mode 100644 tests/connnection.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dd5b00d..29f49cf1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in v0.4 =============== +- Added ``GridFSStorage`` Django storage backend +- Added ``FileField`` for GridFS support - Added ``SortedListField`` - Added ``EmailField`` - Added ``GeoPointField`` diff --git a/docs/django.rst b/docs/django.rst index 92a8a52b..2cce3f02 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -19,7 +19,7 @@ MongoDB but still use many of the Django authentication infrastucture (such as the :func:`login_required` decorator and the :func:`authenticate` function). To enable the MongoEngine auth backend, add the following to you **settings.py** file:: - + AUTHENTICATION_BACKENDS = ( 'mongoengine.django.auth.MongoEngineBackend', ) @@ -44,3 +44,42 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' .. versionadded:: 0.2.1 + +Storage +======= +With MongoEngine's support for GridFS via the FileField, it is useful to have a +Django file storage backend that wraps this. The new storage module is called +GridFSStorage. Using it is very similar to using the default FileSystemStorage.:: + + fs = mongoengine.django.GridFSStorage() + + filename = fs.save('hello.txt', 'Hello, World!') + +All of the `Django Storage API methods +`_ have been +implemented except ``path()``. If the filename provided already exists, an +underscore and a number (before # the file extension, if one exists) will be +appended to the filename until the generated filename doesn't exist. The +``save()`` method will return the new filename.:: + + > fs.exists('hello.txt') + True + > fs.open('hello.txt').read() + 'Hello, World!' + > fs.size('hello.txt') + 13 + > fs.url('hello.txt') + 'http://your_media_url/hello.txt' + > fs.open('hello.txt').name + 'hello.txt' + > fs.listdir() + ([], [u'hello.txt']) + +All files will be saved and retrieved in GridFS via the ``FileDocument`` document, +allowing easy access to the files without the GridFSStorage backend.:: + + > from mongoengine.django.storage import FileDocument + > FileDocument.objects() + [] + +.. versionadded:: 0.4 diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817da..91a12ab3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -23,7 +23,7 @@ class BaseField(object): # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False - + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): @@ -89,7 +89,7 @@ class BaseField(object): if value not in self.choices: raise ValidationError("Value must be one of %s." % unicode(self.choices)) - + # check validation argument if self.validation is not None: if callable(self.validation): @@ -98,13 +98,13 @@ class BaseField(object): 'validation method.') else: raise ValueError('validation argument must be a callable.') - + self.validate(value) class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ - + def to_python(self, value): return value # return unicode(value) @@ -150,7 +150,7 @@ class DocumentMetaclass(type): # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) - + if hasattr(base, '_meta'): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on @@ -191,20 +191,20 @@ class DocumentMetaclass(type): field.owner_document = new_class module = attrs.get('__module__') - + base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) exc = subclass_exception('DoesNotExist', base_excs, module) new_class.add_to_class('DoesNotExist', exc) - + base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, 'MultipleObjectsReturned')) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception('MultipleObjectsReturned', base_excs, module) new_class.add_to_class('MultipleObjectsReturned', exc) - + return new_class - + def add_to_class(self, name, value): setattr(self, name, value) @@ -227,7 +227,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): return super_new(cls, name, bases, attrs) collection = name.lower() - + id_field = None base_indexes = [] base_meta = {} @@ -265,7 +265,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - + # Provide a default queryset unless one has been manually provided if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() @@ -273,7 +273,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes new_class._meta['indexes'] = user_indexes - + unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints @@ -431,7 +431,7 @@ class BaseDocument(object): if data.has_key('_id') and not data['_id']: del data['_id'] return data - + @classmethod def _from_son(cls, son): """Create an instance of a Document (subclass) from a PyMongo SON. @@ -468,7 +468,7 @@ class BaseDocument(object): obj = cls(**data) obj._present_fields = present_fields return obj - + def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): if self.id == other.id: diff --git a/mongoengine/django/storage.py b/mongoengine/django/storage.py new file mode 100644 index 00000000..341455cd --- /dev/null +++ b/mongoengine/django/storage.py @@ -0,0 +1,112 @@ +import os +import itertools +import urlparse + +from mongoengine import * +from django.conf import settings +from django.core.files.storage import Storage +from django.core.exceptions import ImproperlyConfigured + + +class FileDocument(Document): + """A document used to store a single file in GridFS. + """ + file = FileField() + + +class GridFSStorage(Storage): + """A custom storage backend to store files in GridFS + """ + + def __init__(self, base_url=None): + + if base_url is None: + base_url = settings.MEDIA_URL + self.base_url = base_url + self.document = FileDocument + self.field = 'file' + + def delete(self, name): + """Deletes the specified file from the storage system. + """ + if self.exists(name): + doc = self.document.objects.first() + field = getattr(doc, self.field) + self._get_doc_with_name(name).delete() # Delete the FileField + field.delete() # Delete the FileDocument + + def exists(self, name): + """Returns True if a file referened by the given name already exists in the + storage system, or False if the name is available for a new file. + """ + doc = self._get_doc_with_name(name) + if doc: + field = getattr(doc, self.field) + return bool(field.name) + else: + return False + + def listdir(self, path=None): + """Lists the contents of the specified path, returning a 2-tuple of lists; + the first item being directories, the second item being files. + """ + def name(doc): + return getattr(doc, self.field).name + docs = self.document.objects + return [], [name(d) for d in docs if name(d)] + + def size(self, name): + """Returns the total size, in bytes, of the file specified by name. + """ + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field).length + else: + raise ValueError("No such file or directory: '%s'" % name) + + def url(self, name): + """Returns an absolute URL where the file's contents can be accessed + directly by a web browser. + """ + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + return urlparse.urljoin(self.base_url, name).replace('\\', '/') + + def _get_doc_with_name(self, name): + """Find the documents in the store with the given name + """ + docs = self.document.objects + doc = [d for d in docs if getattr(d, self.field).name == name] + if doc: + return doc[0] + else: + return None + + def _open(self, name, mode='rb'): + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field) + else: + raise ValueError("No file found with the name '%s'." % name) + + def get_available_name(self, name): + """Returns a filename that's free on the target storage system, and + available for new content to be written to. + """ + file_root, file_ext = os.path.splitext(name) + # If the filename already exists, add an underscore and a number (before + # the file extension, if one exists) to the filename until the generated + # filename doesn't exist. + count = itertools.count(1) + while self.exists(name): + # file_ext includes the dot. + name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext)) + + return name + + def _save(self, name, content): + doc = self.document() + getattr(doc, self.field).put(content, filename=name) + doc.save() + + return name diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec145..368b5805 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -15,7 +15,7 @@ class EmbeddedDocument(BaseDocument): fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. """ - + __metaclass__ = DocumentMetaclass @@ -119,23 +119,23 @@ class Document(BaseDocument): class MapReduceDocument(object): """A document returned from a map/reduce query. - + :param collection: An instance of :class:`~pymongo.Collection` :param key: Document/result key, often an instance of :class:`~pymongo.objectid.ObjectId`. If supplied as an ``ObjectId`` found in the given ``collection``, the object can be accessed via the ``object`` property. :param value: The result(s) for this key. - + .. versionadded:: 0.3 """ - + def __init__(self, document, collection, key, value): self._document = document self._collection = collection self.key = key self.value = value - + @property def object(self): """Lazy-load the object referenced by ``self.key``. ``self.key`` @@ -143,7 +143,7 @@ class MapReduceDocument(object): """ id_field = self._document()._meta['id_field'] id_field_type = type(id_field) - + if not isinstance(self.key, id_field_type): try: self.key = id_field_type(self.key) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 76bc4fbe..1b689f28 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,11 +16,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', -<<<<<<< HEAD - 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] -======= 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField'] ->>>>>>> 32e66b29f44f3015be099851201241caee92054f RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -42,7 +38,7 @@ class StringField(BaseField): if self.max_length is not None and len(value) > self.max_length: raise ValidationError('String value is too long') - + if self.min_length is not None and len(value) < self.min_length: raise ValidationError('String value is too short') @@ -350,7 +346,8 @@ class SortedListField(ListField): def to_mongo(self, value): if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value], + key=itemgetter(self._ordering)) return sorted([self.field.to_mongo(item) for item in value]) class DictField(BaseField): @@ -514,25 +511,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS + + .. versionadded:: 0.4 """ -<<<<<<< HEAD - def __init__(self): - self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file -======= def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __getattr__(self, name): obj = self.get() @@ -543,17 +532,13 @@ class GridFSProxy(object): return self def get(self, id=None): -<<<<<<< HEAD - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted -======= if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: - return None # File has been deleted ->>>>>>> 32e66b29f44f3015be099851201241caee92054f + # File has been deleted + return None def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -575,20 +560,19 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): -<<<<<<< HEAD - try: return self.get().read() - except: return None -======= try: return self.get().read() except: return None ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def delete(self): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) - self.grid_id = None + + #self.grid_id = None + # Doesn't make a difference because will be put back in when + # reinstantiated We should delete all the metadata stored with the + # file too def replace(self, file, **kwargs): self.delete() @@ -601,41 +585,30 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class FileField(BaseField): """A GridFS storage field. + + .. versionadded:: 0.4 """ def __init__(self, **kwargs): -<<<<<<< HEAD - self.gridfs = GridFSProxy() -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self -<<<<<<< HEAD - return self.gridfs -======= # Check if a file already exists for this model grid_file = instance._data.get(self.name) - if grid_file: - return grid_file + self.grid_file = grid_file + if self.grid_file: + return self.grid_file return GridFSProxy() ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation -<<<<<<< HEAD - self.gridfs.put(value) -======= grid_file = instance._data.get(self.name) # If a file already exists, delete it if grid_file: @@ -649,24 +622,11 @@ class FileField(BaseField): # Create a new proxy object as we don't already have one instance._data[self.name] = GridFSProxy() instance._data[self.name].put(value) ->>>>>>> 32e66b29f44f3015be099851201241caee92054f else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB -<<<<<<< HEAD - return self.gridfs.grid_id - - def to_python(self, value): - # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() - - def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) - -======= if isinstance(value, GridFSProxy) and value.grid_id is not None: return value.grid_id return None @@ -680,6 +640,7 @@ class FileField(BaseField): assert isinstance(value, GridFSProxy) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + class GeoPointField(BaseField): """A list storing a latitude and longitude. """ @@ -692,10 +653,9 @@ class GeoPointField(BaseField): if not isinstance(value, (list, tuple)): raise ValidationError('GeoPointField can only accept tuples or ' 'lists of (x, y)') - + if not len(value) == 2: raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') ->>>>>>> 32e66b29f44f3015be099851201241caee92054f diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c3..2fb8a9d8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -163,7 +163,7 @@ class QuerySet(object): self._where_clause = None self._loaded_fields = [] self._ordering = [] - + # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance'): @@ -240,7 +240,7 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) - + def all(self): """Returns all documents.""" return self.__call__() @@ -256,7 +256,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_options', {}) - + # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: @@ -267,12 +267,12 @@ class QuerySet(object): for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) - + # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: self._collection.ensure_index('_types', background=background, **index_opts) - + # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: @@ -471,7 +471,7 @@ class QuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - + :param object_ids: a list or tuple of ``ObjectId``\ s :rtype: dict of ObjectIds as keys and collection-specific Document subclasses as values. @@ -483,7 +483,7 @@ class QuerySet(object): docs = self._collection.find({'_id': {'$in': object_ids}}) for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) - + return doc_map def next(self): @@ -637,7 +637,7 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) - + def distinct(self, field): """Return a list of distinct values for a given field. @@ -649,9 +649,9 @@ class QuerySet(object): def only(self, *fields): """Load only a subset of this document's fields. :: - + post = BlogPost.objects(...).only("title") - + :param fields: fields to include .. versionadded:: 0.3 diff --git a/tests/connnection.py b/tests/connnection.py new file mode 100644 index 00000000..1903a5f4 --- /dev/null +++ b/tests/connnection.py @@ -0,0 +1,44 @@ +import unittest +import datetime +import pymongo + +import mongoengine.connection +from mongoengine import * +from mongoengine.connection import _get_db, _get_connection + + +class ConnectionTest(unittest.TestCase): + + def tearDown(self): + mongoengine.connection._connection_settings = {} + mongoengine.connection._connections = {} + mongoengine.connection._dbs = {} + + def test_connect(self): + """Ensure that the connect() method works properly. + """ + connect('mongoenginetest') + + conn = _get_connection() + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + + def test_register_connection(self): + """Ensure that connections with different aliases may be registered. + """ + register_connection('testdb', 'mongoenginetest2') + + self.assertRaises(ConnectionError, _get_connection) + conn = _get_connection('testdb') + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db('testdb') + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest2') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/fields.py b/tests/fields.py index 536a9f1a..f5f38fc2 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,7 +693,7 @@ class FieldTest(unittest.TestCase): testfile.name = "Hello, World!" testfile.file.put('Hello, World!') testfile.save() - + # Second instance testfiledupe = TestFile() data = testfiledupe.file.read() # Should be None From 2c8f00410301d3a184e03cf012a0737533478be4 Mon Sep 17 00:00:00 2001 From: sib Date: Thu, 30 Sep 2010 02:53:44 -0300 Subject: [PATCH 66/98] added update operator for addToSet --- mongoengine/queryset.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab113..4d0f113c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -662,7 +662,7 @@ class QuerySet(object): """Transform an update spec from Django-style format to Mongo format. """ operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + 'pull_all', 'add_to_set'] mongo_update = {} for key, value in update.items(): @@ -680,7 +680,9 @@ class QuerySet(object): op = 'inc' if value > 0: value = -value - + elif op == 'add_to_set': + op = op.replace('_to_set', 'ToSet') + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) @@ -688,7 +690,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'push', 'pull', 'addToSet'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From 72c7a010ff6d782d850c245deee1857f87b9a1f4 Mon Sep 17 00:00:00 2001 From: sib Date: Thu, 30 Sep 2010 03:05:15 -0300 Subject: [PATCH 67/98] added unit test for addToSet --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 51f92993..10825d07 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -664,6 +664,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + BlogPost.objects.update_one(add_to_set__tags='unique') + BlogPost.objects.update_one(add_to_set__tags='unique') + post.reload() + self.assertEqual(post.tags.count('unique'), 1) + BlogPost.drop_collection() def test_update_pull(self): From 159923fae237f1c292b1eae901ea2fea6b653763 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 01:48:42 +0100 Subject: [PATCH 68/98] Made lists of recursive reference fields possible --- mongoengine/fields.py | 23 ++++++++++++++++++++++- tests/fields.py | 12 +++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 87d52fd6..65b397da 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,6 +20,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', RECURSIVE_REFERENCE_CONSTANT = 'self' + class StringField(BaseField): """A unicode string field. """ @@ -105,6 +106,7 @@ class URLField(StringField): message = 'This URL appears to be a broken link: %s' % e raise ValidationError(message) + class EmailField(StringField): """A field that validates input as an E-Mail-Address. """ @@ -119,6 +121,7 @@ class EmailField(StringField): if not EmailField.EMAIL_REGEX.match(value): raise ValidationError('Invalid Mail-address: %s' % value) + class IntField(BaseField): """An integer field. """ @@ -142,6 +145,7 @@ class IntField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Integer value is too large') + class FloatField(BaseField): """An floating point number field. """ @@ -197,6 +201,7 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Decimal value is too large') + class BooleanField(BaseField): """A boolean field type. @@ -209,6 +214,7 @@ class BooleanField(BaseField): def validate(self, value): assert isinstance(value, bool) + class DateTimeField(BaseField): """A datetime field. """ @@ -216,6 +222,7 @@ class DateTimeField(BaseField): def validate(self, value): assert isinstance(value, datetime.datetime) + class EmbeddedDocumentField(BaseField): """An embedded document field. Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. @@ -331,6 +338,16 @@ class ListField(BaseField): def lookup_member(self, member_name): return self.field.lookup_member(member_name) + def _set_owner_document(self, owner_document): + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) + + class SortedListField(ListField): """A ListField that sorts the contents of its list before writing to the database in order to ensure that a sorted list is always @@ -346,9 +363,11 @@ class SortedListField(ListField): def to_mongo(self, value): if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value], + key=itemgetter(self._ordering)) return sorted([self.field.to_mongo(item) for item in value]) + class DictField(BaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -442,6 +461,7 @@ class ReferenceField(BaseField): def lookup_member(self, member_name): return self.document_type._fields.get(member_name) + class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). @@ -640,6 +660,7 @@ class FileField(BaseField): assert isinstance(value, GridFSProxy) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + class GeoPointField(BaseField): """A list storing a latitude and longitude. """ diff --git a/tests/fields.py b/tests/fields.py index 8c727196..622016c0 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -394,14 +394,24 @@ class FieldTest(unittest.TestCase): class Employee(Document): name = StringField() boss = ReferenceField('self') + friends = ListField(ReferenceField('self')) bill = Employee(name='Bill Lumbergh') bill.save() - peter = Employee(name='Peter Gibbons', boss=bill) + + michael = Employee(name='Michael Bolton') + michael.save() + + samir = Employee(name='Samir Nagheenanajar') + samir.save() + + friends = [michael, samir] + peter = Employee(name='Peter Gibbons', boss=bill, friends=friends) peter.save() peter = Employee.objects.with_id(peter.id) self.assertEqual(peter.boss, bill) + self.assertEqual(peter.friends, friends) def test_undefined_reference(self): """Ensure that ReferenceFields may reference undefined Documents. From 4012722a8d15bb3adc6d8581f092882b474b2ced Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 15:01:45 +0100 Subject: [PATCH 69/98] QuerySet.item_frequencies works with non-list fields --- mongoengine/queryset.py | 28 ++++++++++++++++++++-------- tests/queryset.py | 9 ++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 73a45299..016430d1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -916,20 +916,27 @@ class QuerySet(object): """ return self.exec_js(average_func, field) - def item_frequencies(self, list_field, normalize=False): - """Returns a dictionary of all items present in a list field across + def item_frequencies(self, field, normalize=False): + """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. - :param list_field: the list field to use + If the field is a :class:`~mongoengine.ListField`, the items within + each list will be counted individually. + + :param field: the field to use :param normalize: normalize the results so they add to 1.0 """ freq_func = """ - function(listField) { + function(field) { if (options.normalize) { var total = 0.0; db[collection].find(query).forEach(function(doc) { - total += doc[listField].length; + if (doc[field].constructor == Array) { + total += doc[field].length; + } else { + total++; + } }); } @@ -939,14 +946,19 @@ class QuerySet(object): inc /= total; } db[collection].find(query).forEach(function(doc) { - doc[listField].forEach(function(item) { + if (doc[field].constructor == Array) { + doc[field].forEach(function(item) { + frequencies[item] = inc + (frequencies[item] || 0); + }); + } else { + var item = doc[field]; frequencies[item] = inc + (frequencies[item] || 0); - }); + } }); return frequencies; } """ - return self.exec_js(freq_func, list_field, normalize=normalize) + return self.exec_js(freq_func, field, normalize=normalize) def __repr__(self): limit = REPR_OUTPUT_SIZE + 1 diff --git a/tests/queryset.py b/tests/queryset.py index ab28ff3d..62cf4954 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -973,7 +973,7 @@ class QuerySetTest(unittest.TestCase): BlogPost(hits=1, tags=['music', 'film', 'actors']).save() BlogPost(hits=2, tags=['music']).save() - BlogPost(hits=3, tags=['music', 'actors']).save() + BlogPost(hits=2, tags=['music', 'actors']).save() f = BlogPost.objects.item_frequencies('tags') f = dict((key, int(val)) for key, val in f.items()) @@ -995,6 +995,13 @@ class QuerySetTest(unittest.TestCase): self.assertAlmostEqual(f['actors'], 2.0/6.0) self.assertAlmostEqual(f['film'], 1.0/6.0) + # Check item_frequencies works for non-list fields + f = BlogPost.objects.item_frequencies('hits') + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['1', '2']), set(f.keys())) + self.assertEqual(f['1'], 1) + self.assertEqual(f['2'], 2) + BlogPost.drop_collection() def test_average(self): From 556eed0151ea845bef1cb761f84cc07cadf4329c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 15:22:47 +0100 Subject: [PATCH 70/98] QuerySet.distinct respects query. Closes #64. --- mongoengine/queryset.py | 2 +- tests/queryset.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 016430d1..48936e68 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -649,7 +649,7 @@ class QuerySet(object): .. versionadded:: 0.4 """ - return self._collection.distinct(field) + return self._cursor.distinct(field) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index 62cf4954..2271c366 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1038,9 +1038,13 @@ class QuerySetTest(unittest.TestCase): self.Person(name='Mr Orange', age=20).save() self.Person(name='Mr White', age=20).save() self.Person(name='Mr Orange', age=30).save() - self.assertEqual(self.Person.objects.distinct('name'), - ['Mr Orange', 'Mr White']) - self.assertEqual(self.Person.objects.distinct('age'), [20, 30]) + self.Person(name='Mr Pink', age=30).save() + self.assertEqual(set(self.Person.objects.distinct('name')), + set(['Mr Orange', 'Mr White', 'Mr Pink'])) + self.assertEqual(set(self.Person.objects.distinct('age')), + set([20, 30])) + self.assertEqual(set(self.Person.objects(age=30).distinct('name')), + set(['Mr Orange', 'Mr Pink'])) def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. From 9c9903664a5fe39d6f40728bc877027ec658cb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 3 Oct 2010 18:50:35 +0200 Subject: [PATCH 71/98] add support for pk property in documents and filters --- docs/guide/document-instances.rst | 7 +++++++ mongoengine/base.py | 25 +++++++++++++++++++------ mongoengine/queryset.py | 3 +++ tests/document.py | 26 ++++++++++++++++++++++++++ tests/queryset.py | 24 +++++++++++++++++++++++- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index b5a1f029..7b5d165b 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -59,6 +59,13 @@ you may still use :attr:`id` to access the primary key if you want:: >>> bob.id == bob.email == 'bob@example.com' True +You can also access the document's "primary key" using the :attr:`pk` field; in +is an alias to :attr:`id`:: + + >>> page = Page(title="Another Test Page") + >>> page.save() + >>> page.id == page.pk + .. note:: If you define your own primary key field, the field implicitly becomes required, so a :class:`ValidationError` will be thrown if you don't provide diff --git a/mongoengine/base.py b/mongoengine/base.py index 0cbd707d..addcd6bf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -330,14 +330,17 @@ class BaseDocument(object): def __init__(self, **values): self._data = {} + # Assign default values to instance + for attr_name in self._fields.keys(): + # Use default value if present + value = getattr(self, attr_name, None) + setattr(self, attr_name, value) # Assign initial values to instance - for attr_name, attr_value in self._fields.items(): - if attr_name in values: + for attr_name in values.keys(): + try: setattr(self, attr_name, values.pop(attr_name)) - else: - # Use default value if present - value = getattr(self, attr_name, None) - setattr(self, attr_name, value) + except AttributeError: + pass def validate(self): """Ensure that all fields' values are valid and that required fields @@ -373,6 +376,16 @@ class BaseDocument(object): all_subclasses.update(subclass._get_subclasses()) return all_subclasses + @apply + def pk(): + """Primary key alias + """ + def fget(self): + return getattr(self, self._meta['id_field']) + def fset(self, value): + return setattr(self, self._meta['id_field'], value) + return property(fget, fset) + def __iter__(self): return iter(self._fields) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e68..69a110fe 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -312,6 +312,9 @@ class QuerySet(object): for field_name in parts: if field is None: # Look up first field from the document + if field_name == 'pk': + # Deal with "primary key" alias + field_name = document._meta['id_field'] field = document._fields[field_name] else: # Look up subfield on the previous field diff --git a/tests/document.py b/tests/document.py index c2e0d6f9..81f492c5 100644 --- a/tests/document.py +++ b/tests/document.py @@ -355,12 +355,26 @@ class DocumentTest(unittest.TestCase): user_obj = User.objects.first() self.assertEqual(user_obj.id, 'test') + self.assertEqual(user_obj.pk, 'test') user_son = User.objects._collection.find_one() self.assertEqual(user_son['_id'], 'test') self.assertTrue('username' not in user_son['_id']) User.drop_collection() + + user = User(pk='mongo', name='mongo user') + user.save() + + user_obj = User.objects.first() + self.assertEqual(user_obj.id, 'mongo') + self.assertEqual(user_obj.pk, 'mongo') + + user_son = User.objects._collection.find_one() + self.assertEqual(user_son['_id'], 'mongo') + self.assertTrue('username' not in user_son['_id']) + + User.drop_collection() def test_creation(self): """Ensure that document may be created using keyword arguments. @@ -479,6 +493,18 @@ class DocumentTest(unittest.TestCase): collection = self.db[self.Person._meta['collection']] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') + + def test_save_custom_pk(self): + """Ensure that a document may be saved with a custom _id using pk alias. + """ + # Create person object and save it to the database + person = self.Person(name='Test User', age=30, + pk='497ce96f395f2f052a494fd4') + person.save() + # Ensure that the object is in the database with the correct _id + collection = self.db[self.Person._meta['collection']] + person_obj = collection.find_one({'name': 'Test User'}) + self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') def test_save_list(self): """Ensure that a list field may be properly saved. diff --git a/tests/queryset.py b/tests/queryset.py index 2271c366..92d3e4c5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1094,7 +1094,8 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() data = {'title': 'Post 1', 'comments': [Comment(content='test')]} - BlogPost(**data).save() + post = BlogPost(**data) + post.save() self.assertTrue('postTitle' in BlogPost.objects(title=data['title'])._query) @@ -1102,12 +1103,33 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects(title=data['title'])._query) self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) + self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + self.assertTrue('postComments.commentContent' in BlogPost.objects(comments__content='test')._query) self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) BlogPost.drop_collection() + def test_query_pk_field_name(self): + """Ensure that the correct "primary key" field name is used when querying + """ + class BlogPost(Document): + title = StringField(primary_key=True, db_field='postTitle') + + BlogPost.drop_collection() + + data = { 'title':'Post 1' } + post = BlogPost(**data) + post.save() + + self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) + self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + + BlogPost.drop_collection() + def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. """ From 62388cb740deb5be6845225f2294bb94abac30dc Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 21:08:28 +0100 Subject: [PATCH 72/98] Started work on new Q-object implementation --- mongoengine/queryset.py | 112 ++++++++++++++++++++++++++++++++++++++++ tests/queryset.py | 25 +++++++-- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e68..ad3c2de1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -15,6 +15,7 @@ REPR_OUTPUT_SIZE = 20 class DoesNotExist(Exception): pass + class MultipleObjectsReturned(Exception): pass @@ -26,12 +27,123 @@ class InvalidQueryError(Exception): class OperationError(Exception): pass + class InvalidCollectionError(Exception): pass + RE_TYPE = type(re.compile('')) +class QNodeVisitor(object): + + def visit_combination(self, combination): + return combination + + def visit_query(self, query): + return query + + +class SimplificationVisitor(QNodeVisitor): + + def visit_combination(self, combination): + if combination.operation != combination.AND: + return combination + + if any(not isinstance(node, NewQ) for node in combination.children): + return combination + + query_ops = set() + query = {} + for node in combination.children: + ops = set(node.query.keys()) + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + query.update(copy.deepcopy(node.query)) + return NewQ(**query) + + +class QueryCompilerVisitor(QNodeVisitor): + + def __init__(self, document): + self.document = document + + def visit_combination(self, combination): + if combination.operation == combination.OR: + return combination + return combination + + def visit_query(self, query): + return QuerySet._transform_query(self.document, **query.query) + + +class QNode(object): + + AND = 0 + OR = 1 + + def to_query(self, document): + query = self.accept(SimplificationVisitor()) + query = query.accept(QueryCompilerVisitor(document)) + return query + + def accept(self, visitor): + raise NotImplementedError + + def _combine(self, other, operation): + if other.empty: + return self + + if self.empty: + return other + + return QCombination(operation, [self, other]) + + @property + def empty(self): + return False + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + +class QCombination(QNode): + + def __init__(self, operation, children): + self.operation = operation + self.children = children + + def accept(self, visitor): + for i in range(len(self.children)): + self.children[i] = self.children[i].accept(visitor) + + return visitor.visit_combination(self) + + @property + def empty(self): + return not bool(self.query) + + +class NewQ(QNode): + + def __init__(self, **query): + self.query = query + + def accept(self, visitor): + return visitor.visit_query(self) + + @property + def empty(self): + return not bool(self.query) + + class Q(object): OR = '||' diff --git a/tests/queryset.py b/tests/queryset.py index 2271c366..60952513 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -6,7 +6,7 @@ import pymongo from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist) + DoesNotExist, NewQ) from mongoengine import * @@ -53,9 +53,6 @@ class QuerySetTest(unittest.TestCase): person2 = self.Person(name="User B", age=30) person2.save() - q1 = Q(name='test') - q2 = Q(age__gte=18) - # Find all people in the collection people = self.Person.objects self.assertEqual(len(people), 2) @@ -1415,5 +1412,25 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) +class NewQTest(unittest.TestCase): + + def test_and_combination(self): + class TestDoc(Document): + x = IntField() + + # Check than an error is raised when conflicting queries are anded + def invalid_combination(): + query = NewQ(x__lt=7) & NewQ(x__lt=3) + query.to_query(TestDoc) + self.assertRaises(InvalidQueryError, invalid_combination) + + # Check normal cases work without an error + query = NewQ(x__lt=7) & NewQ(x__gt=3) + + q1 = NewQ(x__lt=7) + q2 = NewQ(x__gt=3) + query = (q1 & q2).to_query(TestDoc) + self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + if __name__ == '__main__': unittest.main() From a3c46fec0778dd6dba5aa9d693eb9cecac530f0c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 21:26:26 +0100 Subject: [PATCH 73/98] Compilation of combinations - simple $or now works --- mongoengine/queryset.py | 32 +++++++++++++++++++------------- tests/queryset.py | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ad3c2de1..b3fe29f5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -43,6 +43,20 @@ class QNodeVisitor(object): def visit_query(self, query): return query + def _query_conjunction(self, queries): + query_ops = set() + combined_query = {} + for query in queries: + ops = set(query.keys()) + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + combined_query.update(copy.deepcopy(query)) + return combined_query + class SimplificationVisitor(QNodeVisitor): @@ -53,18 +67,8 @@ class SimplificationVisitor(QNodeVisitor): if any(not isinstance(node, NewQ) for node in combination.children): return combination - query_ops = set() - query = {} - for node in combination.children: - ops = set(node.query.keys()) - intersection = ops.intersection(query_ops) - if intersection: - msg = 'Duplicate query contitions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - query_ops.update(ops) - query.update(copy.deepcopy(node.query)) - return NewQ(**query) + queries = [node.query for node in combination.children] + return NewQ(**self._query_conjunction(queries)) class QueryCompilerVisitor(QNodeVisitor): @@ -74,7 +78,9 @@ class QueryCompilerVisitor(QNodeVisitor): def visit_combination(self, combination): if combination.operation == combination.OR: - return combination + return {'$or': combination.children} + elif combination.operation == combination.AND: + return self._query_conjunction(combination.children) return combination def visit_query(self, query): diff --git a/tests/queryset.py b/tests/queryset.py index 60952513..6d3114e5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1432,5 +1432,19 @@ class NewQTest(unittest.TestCase): query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + def test_or_combination(self): + class TestDoc(Document): + x = IntField() + + q1 = NewQ(x__lt=3) + q2 = NewQ(x__gt=7) + query = (q1 | q2).to_query(TestDoc) + self.assertEqual(query, { + '$or': [ + {'x': {'$lt': 3}}, + {'x': {'$gt': 7}}, + ] + }) + if __name__ == '__main__': unittest.main() From db2f64c290c5469c0e82952cb3c9ef0c5457b0f8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 23:01:44 +0100 Subject: [PATCH 74/98] Made query-tree code a bit clearer --- mongoengine/queryset.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b3fe29f5..21bd44d0 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -36,18 +36,28 @@ RE_TYPE = type(re.compile('')) class QNodeVisitor(object): + """Base visitor class for visiting Q-object nodes in a query tree. + """ def visit_combination(self, combination): + """Called by QCombination objects. + """ return combination def visit_query(self, query): + """Called by (New)Q objects. + """ return query def _query_conjunction(self, queries): + """Merges two query dicts - effectively &ing them together. + """ query_ops = set() combined_query = {} for query in queries: ops = set(query.keys()) + # Make sure that the same operation isn't applied more than once + # to a single field intersection = ops.intersection(query_ops) if intersection: msg = 'Duplicate query contitions: ' @@ -59,11 +69,15 @@ class QNodeVisitor(object): class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ def visit_combination(self, combination): if combination.operation != combination.AND: return combination + # The simplification only applies to 'simple' queries if any(not isinstance(node, NewQ) for node in combination.children): return combination @@ -72,6 +86,9 @@ class SimplificationVisitor(QNodeVisitor): class QueryCompilerVisitor(QNodeVisitor): + """Compiles the nodes in a query tree to a PyMongo-compatible query + dictionary. + """ def __init__(self, document): self.document = document @@ -88,6 +105,8 @@ class QueryCompilerVisitor(QNodeVisitor): class QNode(object): + """Base class for nodes in query trees. + """ AND = 0 OR = 1 @@ -101,6 +120,8 @@ class QNode(object): raise NotImplementedError def _combine(self, other, operation): + """Combine this node with another node into a QCombination object. + """ if other.empty: return self @@ -121,6 +142,9 @@ class QNode(object): class QCombination(QNode): + """Represents the combination of several conditions by a given logical + operator. + """ def __init__(self, operation, children): self.operation = operation @@ -138,6 +162,9 @@ class QCombination(QNode): class NewQ(QNode): + """A simple query object, used in a query tree to build up more complex + query structures. + """ def __init__(self, **query): self.query = query From c0f7c4ca2ddabb33a5d7dcfd87dfe9f8059844bc Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 23:22:36 +0100 Subject: [PATCH 75/98] Fixed error in empty property on QCombination --- mongoengine/queryset.py | 2 +- tests/queryset.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 21bd44d0..2c822c75 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -158,7 +158,7 @@ class QCombination(QNode): @property def empty(self): - return not bool(self.query) + return not bool(self.children) class NewQ(QNode): diff --git a/tests/queryset.py b/tests/queryset.py index 6d3114e5..6a337640 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1417,6 +1417,7 @@ class NewQTest(unittest.TestCase): def test_and_combination(self): class TestDoc(Document): x = IntField() + y = StringField() # Check than an error is raised when conflicting queries are anded def invalid_combination(): @@ -1432,6 +1433,15 @@ class NewQTest(unittest.TestCase): query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + # More complex nested example + query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString') + query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100) + mongo_query = { + 'x': {'$lt': 100, '$gt': -100}, + 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, + } + self.assertEqual(query.to_query(TestDoc), mongo_query) + def test_or_combination(self): class TestDoc(Document): x = IntField() From 8e651542015ba7de526477d17a5cf661aca949e2 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 00:06:42 +0100 Subject: [PATCH 76/98] Added a tree transformer, got complex ANDs working --- mongoengine/queryset.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2c822c75..9650fe1f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -85,6 +85,44 @@ class SimplificationVisitor(QNodeVisitor): return NewQ(**self._query_conjunction(queries)) +class QueryTreeTransformerVisitor(QNodeVisitor): + """Transforms the query tree in to a form that may be used with MongoDB. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # MongoDB doesn't allow us to have too many $or operations in our + # queries, so the aim is to move the ORs up the tree to one + # 'master' $or. Firstly, we must find all the necessary parts (part + # of an AND combination or just standard Q object), and store them + # separately from the OR parts. + or_parts = [] + and_parts = [] + for node in combination.children: + if isinstance(node, QCombination): + if node.operation == node.OR: + # Any of the children in an $or component may cause + # the query to succeed + or_parts += node.children + elif node.operation == node.AND: + and_parts.append(node) + elif isinstance(node, NewQ): + and_parts.append(node) + + # Now we combine the parts into a usable query. AND together all of + # the necessary parts. Then for each $or part, create a new query + # that ANDs the necessary part with the $or part. + clauses = [] + for or_part in or_parts: + q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) + clauses.append(q_object & or_part) + + # Finally, $or the generated clauses in to one query. Each of the + # clauses is sufficient for the query to succeed. + return reduce(lambda a, b: a | b, clauses, NewQ()) + return combination + + class QueryCompilerVisitor(QNodeVisitor): """Compiles the nodes in a query tree to a PyMongo-compatible query dictionary. @@ -113,6 +151,7 @@ class QNode(object): def to_query(self, document): query = self.accept(SimplificationVisitor()) + query = query.accept(QueryTreeTransformerVisitor()) query = query.accept(QueryCompilerVisitor(document)) return query diff --git a/tests/queryset.py b/tests/queryset.py index 6a337640..f83a8d43 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1456,5 +1456,19 @@ class NewQTest(unittest.TestCase): ] }) + def test_and_or_combination(self): + class TestDoc(Document): + x = IntField() + + query = NewQ(x__gt=0) | NewQ(x__exists=False) + query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000]) + print query.to_query(TestDoc) +# self.assertEqual(query.to_query(TestDoc, { +# '$or': [ +# {'x': {'$lt': 3}}, +# {'x': {'$gt': 7}}, +# ] +# }) + if __name__ == '__main__': unittest.main() From 3fcc0e97895d1f2d2a1bea150f15d1f8aa35b7e9 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 02:10:37 +0100 Subject: [PATCH 77/98] Combining OR nodes works, fixed other Q-object bugs --- mongoengine/queryset.py | 97 +++++++++++++++++++++++++++++++---------- tests/queryset.py | 62 ++++++++++++++++++++++---- 2 files changed, 126 insertions(+), 33 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9650fe1f..06b67ab5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,6 +4,7 @@ import pprint import pymongo import re import copy +import itertools __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'InvalidCollectionError'] @@ -49,8 +50,22 @@ class QNodeVisitor(object): """ return query + +class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # The simplification only applies to 'simple' queries + if all(isinstance(node, NewQ) for node in combination.children): + queries = [node.query for node in combination.children] + return NewQ(**self._query_conjunction(queries)) + return combination + def _query_conjunction(self, queries): - """Merges two query dicts - effectively &ing them together. + """Merges query dicts - effectively &ing them together. """ query_ops = set() combined_query = {} @@ -68,23 +83,6 @@ class QNodeVisitor(object): return combined_query -class SimplificationVisitor(QNodeVisitor): - """Simplifies query trees by combinging unnecessary 'and' connection nodes - into a single Q-object. - """ - - def visit_combination(self, combination): - if combination.operation != combination.AND: - return combination - - # The simplification only applies to 'simple' queries - if any(not isinstance(node, NewQ) for node in combination.children): - return combination - - queries = [node.query for node in combination.children] - return NewQ(**self._query_conjunction(queries)) - - class QueryTreeTransformerVisitor(QNodeVisitor): """Transforms the query tree in to a form that may be used with MongoDB. """ @@ -96,14 +94,14 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # 'master' $or. Firstly, we must find all the necessary parts (part # of an AND combination or just standard Q object), and store them # separately from the OR parts. - or_parts = [] + or_groups = [] and_parts = [] for node in combination.children: if isinstance(node, QCombination): if node.operation == node.OR: # Any of the children in an $or component may cause # the query to succeed - or_parts += node.children + or_groups.append(node.children) elif node.operation == node.AND: and_parts.append(node) elif isinstance(node, NewQ): @@ -113,13 +111,27 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # the necessary parts. Then for each $or part, create a new query # that ANDs the necessary part with the $or part. clauses = [] - for or_part in or_parts: + for or_group in itertools.product(*or_groups): q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) - clauses.append(q_object & or_part) + q_object = reduce(lambda a, b: a & b, or_group, q_object) + clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. return reduce(lambda a, b: a | b, clauses, NewQ()) + + if combination.operation == combination.OR: + children = [] + # Crush any nested ORs in to this combination as MongoDB doesn't + # support nested $or operations + for node in combination.children: + if (isinstance(node, QCombination) and + node.operation == combination.OR): + children += node.children + else: + children.append(node) + combination.children = children + return combination @@ -135,12 +147,42 @@ class QueryCompilerVisitor(QNodeVisitor): if combination.operation == combination.OR: return {'$or': combination.children} elif combination.operation == combination.AND: - return self._query_conjunction(combination.children) + return self._mongo_query_conjunction(combination.children) return combination def visit_query(self, query): return QuerySet._transform_query(self.document, **query.query) + def _mongo_query_conjunction(self, queries): + """Merges Mongo query dicts - effectively &ing them together. + """ + combined_query = {} + for query in queries: + for field, ops in query.items(): + if field not in combined_query: + combined_query[field] = ops + else: + # The field is already present in the query the only way + # we can merge is if both the existing value and the new + # value are operation dicts, reject anything else + if (not isinstance(combined_query[field], dict) or + not isinstance(ops, dict)): + message = 'Conflicting values for ' + field + raise InvalidQueryError(message) + + current_ops = set(combined_query[field].keys()) + new_ops = set(ops.keys()) + # Make sure that the same operation isn't applied more than + # once to a single field + intersection = current_ops.intersection(new_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + # Right! We've got two non-overlapping dicts of operations! + combined_query[field].update(copy.deepcopy(ops)) + return combined_query + class QNode(object): """Base class for nodes in query trees. @@ -187,7 +229,14 @@ class QCombination(QNode): def __init__(self, operation, children): self.operation = operation - self.children = children + self.children = [] + for node in children: + # If the child is a combination of the same type, we can merge its + # children directly into this combinations children + if isinstance(node, QCombination) and node.operation == operation: + self.children += node.children + else: + self.children.append(node) def accept(self, visitor): for i in range(len(self.children)): diff --git a/tests/queryset.py b/tests/queryset.py index f83a8d43..0f8c9a92 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1415,6 +1415,8 @@ class QTest(unittest.TestCase): class NewQTest(unittest.TestCase): def test_and_combination(self): + """Ensure that Q-objects correctly AND together. + """ class TestDoc(Document): x = IntField() y = StringField() @@ -1443,6 +1445,8 @@ class NewQTest(unittest.TestCase): self.assertEqual(query.to_query(TestDoc), mongo_query) def test_or_combination(self): + """Ensure that Q-objects correctly OR together. + """ class TestDoc(Document): x = IntField() @@ -1457,18 +1461,58 @@ class NewQTest(unittest.TestCase): }) def test_and_or_combination(self): + """Ensure that Q-objects handle ANDing ORed components. + """ class TestDoc(Document): x = IntField() + y = BooleanField() + + query = (NewQ(x__gt=0) | NewQ(x__exists=False)) + query &= NewQ(x__lt=100) + self.assertEqual(query.to_query(TestDoc), { + '$or': [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + ] + }) + + q1 = (NewQ(x__gt=0) | NewQ(x__exists=False)) + q2 = (NewQ(x__lt=100) | NewQ(y=True)) + query = (q1 & q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$exists': False}, 'y': True}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_or_and_or_combination(self): + """Ensure that Q-objects handle ORing ANDed ORed components. :) + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False))) + q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False))) + query = (q1 | q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$gt': 0}, 'y': {'$exists': False}}, + {'x': {'$lt': 100}, 'y':False}, + {'x': {'$lt': 100}, 'y': {'$exists': False}}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) - query = NewQ(x__gt=0) | NewQ(x__exists=False) - query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000]) - print query.to_query(TestDoc) -# self.assertEqual(query.to_query(TestDoc, { -# '$or': [ -# {'x': {'$lt': 3}}, -# {'x': {'$gt': 7}}, -# ] -# }) if __name__ == '__main__': unittest.main() From 76cb851c40c00b11700689b72b022541de3b422a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 11:41:07 +0100 Subject: [PATCH 78/98] Replaced old Q-object with new, revamped Q-object --- mongoengine/queryset.py | 154 +++++++--------------------------------- tests/queryset.py | 86 ++++++++-------------- 2 files changed, 52 insertions(+), 188 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 06b67ab5..2eabfb0e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -59,9 +59,9 @@ class SimplificationVisitor(QNodeVisitor): def visit_combination(self, combination): if combination.operation == combination.AND: # The simplification only applies to 'simple' queries - if all(isinstance(node, NewQ) for node in combination.children): + if all(isinstance(node, Q) for node in combination.children): queries = [node.query for node in combination.children] - return NewQ(**self._query_conjunction(queries)) + return Q(**self._query_conjunction(queries)) return combination def _query_conjunction(self, queries): @@ -104,7 +104,7 @@ class QueryTreeTransformerVisitor(QNodeVisitor): or_groups.append(node.children) elif node.operation == node.AND: and_parts.append(node) - elif isinstance(node, NewQ): + elif isinstance(node, Q): and_parts.append(node) # Now we combine the parts into a usable query. AND together all of @@ -112,13 +112,13 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # that ANDs the necessary part with the $or part. clauses = [] for or_group in itertools.product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) + q_object = reduce(lambda a, b: a & b, and_parts, Q()) q_object = reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, NewQ()) + return reduce(lambda a, b: a | b, clauses, Q()) if combination.operation == combination.OR: children = [] @@ -249,7 +249,7 @@ class QCombination(QNode): return not bool(self.children) -class NewQ(QNode): +class Q(QNode): """A simple query object, used in a query tree to build up more complex query structures. """ @@ -265,124 +265,6 @@ class NewQ(QNode): return not bool(self.query) -class Q(object): - - OR = '||' - AND = '&&' - OPERATORS = { - 'eq': ('((this.%(field)s instanceof Array) && ' - ' this.%(field)s.indexOf(%(value)s) != -1) ||' - ' this.%(field)s == %(value)s'), - 'ne': 'this.%(field)s != %(value)s', - 'gt': 'this.%(field)s > %(value)s', - 'gte': 'this.%(field)s >= %(value)s', - 'lt': 'this.%(field)s < %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'in': '%(value)s.indexOf(this.%(field)s) != -1', - 'nin': '%(value)s.indexOf(this.%(field)s) == -1', - 'mod': '%(field)s %% %(value)s', - 'all': ('%(value)s.every(function(a){' - 'return this.%(field)s.indexOf(a) != -1 })'), - 'size': 'this.%(field)s.length == %(value)s', - 'exists': 'this.%(field)s != null', - 'regex_eq': '%(value)s.test(this.%(field)s)', - 'regex_ne': '!%(value)s.test(this.%(field)s)', - } - - def __init__(self, **query): - self.query = [query] - - def _combine(self, other, op): - obj = Q() - if not other.query[0]: - return self - if self.query[0]: - obj.query = (['('] + copy.deepcopy(self.query) + [op] + - copy.deepcopy(other.query) + [')']) - else: - obj.query = copy.deepcopy(other.query) - return obj - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - def as_js(self, document): - js = [] - js_scope = {} - for i, item in enumerate(self.query): - if isinstance(item, dict): - item_query = QuerySet._transform_query(document, **item) - # item_query will values will either be a value or a dict - js.append(self._item_query_as_js(item_query, js_scope, i)) - else: - js.append(item) - return pymongo.code.Code(' '.join(js), js_scope) - - def _item_query_as_js(self, item_query, js_scope, item_num): - # item_query will be in one of the following forms - # {'age': 25, 'name': 'Test'} - # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} - # {'age': {'$lt': 25, '$gt': 18}} - js = [] - for i, (key, value) in enumerate(item_query.items()): - op = 'eq' - # Construct a variable name for the value in the JS - value_name = 'i%sf%s' % (item_num, i) - if isinstance(value, dict): - # Multiple operators for this field - for j, (op, value) in enumerate(value.items()): - # Create a custom variable name for this operator - op_value_name = '%so%s' % (value_name, j) - # Construct the JS that uses this op - value, operation_js = self._build_op_js(op, key, value, - op_value_name) - # Update the js scope with the value for this op - js_scope[op_value_name] = value - js.append(operation_js) - else: - # Construct the JS for this field - value, field_js = self._build_op_js(op, key, value, value_name) - js_scope[value_name] = value - js.append(field_js) - return ' && '.join(js) - - def _build_op_js(self, op, key, value, value_name): - """Substitute the values in to the correct chunk of Javascript. - """ - if isinstance(value, RE_TYPE): - # Regexes are handled specially - if op.strip('$') == 'ne': - op_js = Q.OPERATORS['regex_ne'] - else: - op_js = Q.OPERATORS['regex_eq'] - else: - op_js = Q.OPERATORS[op.strip('$')] - - # Comparing two ObjectIds in Javascript doesn't work.. - if isinstance(value, pymongo.objectid.ObjectId): - value = unicode(value) - - # Handle DBRef - if isinstance(value, pymongo.dbref.DBRef): - op_js = '(this.%(field)s.$id == "%(id)s" &&'\ - ' this.%(field)s.$ref == "%(ref)s")' % { - 'field': key, - 'id': unicode(value.id), - 'ref': unicode(value.collection) - } - value = None - - # Perform the substitution - operation_js = op_js % { - 'field': key, - 'value': value_name - } - return value, operation_js - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -392,7 +274,9 @@ class QuerySet(object): self._document = document self._collection_obj = collection self._accessed_collection = False - self._query = {} + self._mongo_query = None + self._query_obj = Q() + self._initial_query = {} self._where_clause = None self._loaded_fields = [] self._ordering = [] @@ -400,11 +284,18 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance'): - self._query = {'_types': self._document._class_name} + self._initial_query = {'_types': self._document._class_name} self._cursor_obj = None self._limit = None self._skip = None + @property + def _query(self): + if self._mongo_query is None: + self._mongo_query = self._query_obj.to_query(self._document) + self._mongo_query.update(self._initial_query) + return self._mongo_query + def ensure_index(self, key_or_list, drop_dups=False, background=False, **kwargs): """Ensure that the given indexes are in place. @@ -463,10 +354,13 @@ class QuerySet(object): objects, only the last one will be used :param query: Django-style query keyword arguments """ + #if q_obj: + #self._where_clause = q_obj.as_js(self._document) + query = Q(**query) if q_obj: - self._where_clause = q_obj.as_js(self._document) - query = QuerySet._transform_query(_doc_cls=self._document, **query) - self._query.update(query) + query &= q_obj + self._query_obj &= query + self._mongo_query = None return self def filter(self, *q_objs, **query): @@ -616,7 +510,7 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + for i, part in indices: parts.insert(i, part) key = '.'.join(parts) diff --git a/tests/queryset.py b/tests/queryset.py index 0f8c9a92..c941ecd5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -6,7 +6,7 @@ import pymongo from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist, NewQ) + DoesNotExist) from mongoengine import * @@ -153,7 +153,8 @@ class QuerySetTest(unittest.TestCase): # Retrieve the first person from the database self.assertRaises(MultipleObjectsReturned, self.Person.objects.get) - self.assertRaises(self.Person.MultipleObjectsReturned, self.Person.objects.get) + self.assertRaises(self.Person.MultipleObjectsReturned, + self.Person.objects.get) # Use a query to filter the people found to just person2 person = self.Person.objects.get(age=30) @@ -231,7 +232,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(created, False) # Try retrieving when no objects exists - new doc should be created - person, created = self.Person.objects.get_or_create(age=50, defaults={'name': 'User C'}) + kwargs = dict(age=50, defaults={'name': 'User C'}) + person, created = self.Person.objects.get_or_create(**kwargs) self.assertEqual(created, True) person = self.Person.objects.get(age=50) @@ -545,6 +547,7 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first() self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() self.assertEqual(obj, None) @@ -1343,43 +1346,6 @@ class QuerySetTest(unittest.TestCase): class QTest(unittest.TestCase): - def test_or_and(self): - """Ensure that Q objects may be combined correctly. - """ - q1 = Q(name='test') - q2 = Q(age__gte=18) - - query = ['(', {'name': 'test'}, '||', {'age__gte': 18}, ')'] - self.assertEqual((q1 | q2).query, query) - - query = ['(', {'name': 'test'}, '&&', {'age__gte': 18}, ')'] - self.assertEqual((q1 & q2).query, query) - - query = ['(', '(', {'name': 'test'}, '&&', {'age__gte': 18}, ')', '||', - {'name': 'example'}, ')'] - self.assertEqual((q1 & q2 | Q(name='example')).query, query) - - def test_item_query_as_js(self): - """Ensure that the _item_query_as_js utilitiy method works properly. - """ - q = Q() - examples = [ - - ({'name': 'test'}, ('((this.name instanceof Array) && ' - 'this.name.indexOf(i0f0) != -1) || this.name == i0f0'), - {'i0f0': 'test'}), - ({'age': {'$gt': 18}}, 'this.age > i0f0o0', {'i0f0o0': 18}), - ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}}, - ('this.age <= i0f0o0 && this.age > i0f0o1 && ' - '((this.name instanceof Array) && ' - 'this.name.indexOf(i0f1) != -1) || this.name == i0f1'), - {'i0f0o0': 65, 'i0f0o1': 18, 'i0f1': 'test'}), - ] - for item, js, scope in examples: - test_scope = {} - self.assertEqual(q._item_query_as_js(item, test_scope, 0), js) - self.assertEqual(scope, test_scope) - def test_empty_q(self): """Ensure that empty Q objects won't hurt. """ @@ -1389,11 +1355,15 @@ class QTest(unittest.TestCase): q4 = Q(name='test') q5 = Q() - query = ['(', {'age__gte': 18}, '||', {'name': 'test'}, ')'] - self.assertEqual((q1 | q2 | q3 | q4 | q5).query, query) + class Person(Document): + name = StringField() + age = IntField() - query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] - self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) + query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} + self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) + + query = {'age': {'$gte': 18}, 'name': 'test'} + self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" @@ -1423,21 +1393,21 @@ class NewQTest(unittest.TestCase): # Check than an error is raised when conflicting queries are anded def invalid_combination(): - query = NewQ(x__lt=7) & NewQ(x__lt=3) + query = Q(x__lt=7) & Q(x__lt=3) query.to_query(TestDoc) self.assertRaises(InvalidQueryError, invalid_combination) # Check normal cases work without an error - query = NewQ(x__lt=7) & NewQ(x__gt=3) + query = Q(x__lt=7) & Q(x__gt=3) - q1 = NewQ(x__lt=7) - q2 = NewQ(x__gt=3) + q1 = Q(x__lt=7) + q2 = Q(x__gt=3) query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) # More complex nested example - query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString') - query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100) + query = Q(x__lt=100) & Q(y__ne='NotMyString') + query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) mongo_query = { 'x': {'$lt': 100, '$gt': -100}, 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, @@ -1450,8 +1420,8 @@ class NewQTest(unittest.TestCase): class TestDoc(Document): x = IntField() - q1 = NewQ(x__lt=3) - q2 = NewQ(x__gt=7) + q1 = Q(x__lt=3) + q2 = Q(x__gt=7) query = (q1 | q2).to_query(TestDoc) self.assertEqual(query, { '$or': [ @@ -1467,8 +1437,8 @@ class NewQTest(unittest.TestCase): x = IntField() y = BooleanField() - query = (NewQ(x__gt=0) | NewQ(x__exists=False)) - query &= NewQ(x__lt=100) + query = (Q(x__gt=0) | Q(x__exists=False)) + query &= Q(x__lt=100) self.assertEqual(query.to_query(TestDoc), { '$or': [ {'x': {'$lt': 100, '$gt': 0}}, @@ -1476,8 +1446,8 @@ class NewQTest(unittest.TestCase): ] }) - q1 = (NewQ(x__gt=0) | NewQ(x__exists=False)) - q2 = (NewQ(x__lt=100) | NewQ(y=True)) + q1 = (Q(x__gt=0) | Q(x__exists=False)) + q2 = (Q(x__lt=100) | Q(y=True)) query = (q1 & q2).to_query(TestDoc) self.assertEqual(['$or'], query.keys()) @@ -1498,8 +1468,8 @@ class NewQTest(unittest.TestCase): x = IntField() y = BooleanField() - q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False))) - q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False))) + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) + q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) query = (q1 | q2).to_query(TestDoc) self.assertEqual(['$or'], query.keys()) From b4c54b1b6257a2b5a262eb0c794aa1371f7aa1fb Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 11:41:49 +0100 Subject: [PATCH 79/98] Added support for the $not operator --- mongoengine/queryset.py | 130 +++++++++++++++++++++++++++++++++++++++- tests/queryset.py | 16 ++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2eabfb0e..e275bb31 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -265,6 +265,124 @@ class Q(QNode): return not bool(self.query) +class OldQ(object): + + OR = '||' + AND = '&&' + OPERATORS = { + 'eq': ('((this.%(field)s instanceof Array) && ' + ' this.%(field)s.indexOf(%(value)s) != -1) ||' + ' this.%(field)s == %(value)s'), + 'ne': 'this.%(field)s != %(value)s', + 'gt': 'this.%(field)s > %(value)s', + 'gte': 'this.%(field)s >= %(value)s', + 'lt': 'this.%(field)s < %(value)s', + 'lte': 'this.%(field)s <= %(value)s', + 'lte': 'this.%(field)s <= %(value)s', + 'in': '%(value)s.indexOf(this.%(field)s) != -1', + 'nin': '%(value)s.indexOf(this.%(field)s) == -1', + 'mod': '%(field)s %% %(value)s', + 'all': ('%(value)s.every(function(a){' + 'return this.%(field)s.indexOf(a) != -1 })'), + 'size': 'this.%(field)s.length == %(value)s', + 'exists': 'this.%(field)s != null', + 'regex_eq': '%(value)s.test(this.%(field)s)', + 'regex_ne': '!%(value)s.test(this.%(field)s)', + } + + def __init__(self, **query): + self.query = [query] + + def _combine(self, other, op): + obj = Q() + if not other.query[0]: + return self + if self.query[0]: + obj.query = (['('] + copy.deepcopy(self.query) + [op] + + copy.deepcopy(other.query) + [')']) + else: + obj.query = copy.deepcopy(other.query) + return obj + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + def as_js(self, document): + js = [] + js_scope = {} + for i, item in enumerate(self.query): + if isinstance(item, dict): + item_query = QuerySet._transform_query(document, **item) + # item_query will values will either be a value or a dict + js.append(self._item_query_as_js(item_query, js_scope, i)) + else: + js.append(item) + return pymongo.code.Code(' '.join(js), js_scope) + + def _item_query_as_js(self, item_query, js_scope, item_num): + # item_query will be in one of the following forms + # {'age': 25, 'name': 'Test'} + # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} + # {'age': {'$lt': 25, '$gt': 18}} + js = [] + for i, (key, value) in enumerate(item_query.items()): + op = 'eq' + # Construct a variable name for the value in the JS + value_name = 'i%sf%s' % (item_num, i) + if isinstance(value, dict): + # Multiple operators for this field + for j, (op, value) in enumerate(value.items()): + # Create a custom variable name for this operator + op_value_name = '%so%s' % (value_name, j) + # Construct the JS that uses this op + value, operation_js = self._build_op_js(op, key, value, + op_value_name) + # Update the js scope with the value for this op + js_scope[op_value_name] = value + js.append(operation_js) + else: + # Construct the JS for this field + value, field_js = self._build_op_js(op, key, value, value_name) + js_scope[value_name] = value + js.append(field_js) + return ' && '.join(js) + + def _build_op_js(self, op, key, value, value_name): + """Substitute the values in to the correct chunk of Javascript. + """ + if isinstance(value, RE_TYPE): + # Regexes are handled specially + if op.strip('$') == 'ne': + op_js = Q.OPERATORS['regex_ne'] + else: + op_js = Q.OPERATORS['regex_eq'] + else: + op_js = Q.OPERATORS[op.strip('$')] + + # Comparing two ObjectIds in Javascript doesn't work.. + if isinstance(value, pymongo.objectid.ObjectId): + value = unicode(value) + + # Handle DBRef + if isinstance(value, pymongo.dbref.DBRef): + op_js = '(this.%(field)s.$id == "%(id)s" &&'\ + ' this.%(field)s.$ref == "%(ref)s")' % { + 'field': key, + 'id': unicode(value.id), + 'ref': unicode(value.collection) + } + value = None + + # Perform the substitution + operation_js = op_js % { + 'field': key, + 'value': value_name + } + return value, operation_js + class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -462,7 +580,7 @@ class QuerySet(object): """Transform a query from Django-style format to Mongo format. """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists'] + 'all', 'size', 'exists', 'not'] geo_operators = ['within_distance', 'within_box', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', @@ -478,6 +596,11 @@ class QuerySet(object): if parts[-1] in operators + match_operators + geo_operators: op = parts.pop() + negate = False + if parts[-1] == 'not': + parts.pop() + negate = True + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) @@ -485,7 +608,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += match_operators if op in singular_ops: value = field.prepare_query_value(op, value) @@ -511,6 +634,9 @@ class QuerySet(object): elif op not in match_operators: value = {'$' + op: value} + if negate: + value = {'$not': value} + for i, part in indices: parts.insert(i, part) key = '.'.join(parts) diff --git a/tests/queryset.py b/tests/queryset.py index c941ecd5..32d1902e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -336,6 +336,18 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() self.assertEqual(obj, person) + def test_not(self): + """Ensure that the __not operator works as expected. + """ + alice = self.Person(name='Alice', age=25) + alice.save() + + obj = self.Person.objects(name__iexact='alice').first() + self.assertEqual(obj, alice) + + obj = self.Person.objects(name__not__iexact='alice').first() + self.assertEqual(obj, None) + def test_filter_chaining(self): """Ensure filters can be chained together. """ @@ -545,10 +557,10 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first() + obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() + obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() self.assertEqual(obj, None) def test_q_lists(self): From 4742328b908e57b3d87e2c0752ed8d4b51763e55 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 12:10:29 +0100 Subject: [PATCH 80/98] Delete stale cursor when query is filtered. Closes #62. --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e275bb31..6d4ad55e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -479,6 +479,7 @@ class QuerySet(object): query &= q_obj self._query_obj &= query self._mongo_query = None + self._cursor_obj = None return self def filter(self, *q_objs, **query): From 3acfd907202afa4f2712f6e7fce9c34c2e67c8a0 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 14:58:00 +0100 Subject: [PATCH 81/98] Added some imports for PyMongo 1.9 compatibility. --- mongoengine/base.py | 1 + mongoengine/fields.py | 3 +++ mongoengine/queryset.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0cbd707d..1f7ba1fe 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -3,6 +3,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned import sys import pymongo +import pymongo.objectid _document_registry = {} diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 65b397da..8fcb1d62 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -5,6 +5,9 @@ from operator import itemgetter import re import pymongo +import pymongo.dbref +import pymongo.son +import pymongo.binary import datetime import decimal import gridfs diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e68..99417850 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -2,6 +2,9 @@ from connection import _get_db import pprint import pymongo +import pymongo.code +import pymongo.dbref +import pymongo.objectid import re import copy From 92471445ec5aede3d6ffb14e6b63026156c6d623 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 5 Oct 2010 00:46:13 +0100 Subject: [PATCH 82/98] Fix changing databases Conflicts: mongoengine/connection.py mongoengine/queryset.py --- mongoengine/connection.py | 22 ++++++++++++---------- mongoengine/queryset.py | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 94cc6ea1..814fde13 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -4,11 +4,12 @@ import multiprocessing __all__ = ['ConnectionError', 'connect'] -_connection_settings = { +_connection_defaults = { 'host': 'localhost', 'port': 27017, } _connection = {} +_connection_settings = _connection_defaults.copy() _db_name = None _db_username = None @@ -20,25 +21,25 @@ class ConnectionError(Exception): pass -def _get_connection(): +def _get_connection(reconnect=False): global _connection identity = get_identity() # Connect to the database if not already connected - if _connection.get(identity) is None: + if _connection.get(identity) is None or reconnect: try: _connection[identity] = Connection(**_connection_settings) except: raise ConnectionError('Cannot connect to the database') return _connection[identity] -def _get_db(): +def _get_db(reconnect=False): global _db, _connection identity = get_identity() # Connect if not already connected - if _connection.get(identity) is None: - _connection[identity] = _get_connection() + if _connection.get(identity) is None or reconnect: + _connection[identity] = _get_connection(reconnect=reconnect) - if _db.get(identity) is None: + if _db.get(identity) is None or reconnect: # _db_name will be None if the user hasn't called connect() if _db_name is None: raise ConnectionError('Not connected to the database') @@ -61,9 +62,10 @@ def connect(db, username=None, password=None, **kwargs): the default port on localhost. If authentication is needed, provide username and password arguments as well. """ - global _connection_settings, _db_name, _db_username, _db_password - _connection_settings.update(kwargs) + global _connection_settings, _db_name, _db_username, _db_password, _db + _connection_settings = dict(_connection_defaults, **kwargs) _db_name = db _db_username = username _db_password = password - return _get_db() \ No newline at end of file + return _get_db(reconnect=True) + diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 99417850..fae2aabf 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -977,7 +977,7 @@ class QuerySetManager(object): def __init__(self, manager_func=None): self._manager_func = manager_func - self._collection = None + self._collections = {} def __get__(self, instance, owner): """Descriptor for instantiating a new QuerySet object when @@ -987,8 +987,8 @@ class QuerySetManager(object): # Document class being used rather than a document object return self - if self._collection is None: - db = _get_db() + db = _get_db() + if db not in self._collections: collection = owner._meta['collection'] # Create collection as a capped collection if specified @@ -998,10 +998,10 @@ class QuerySetManager(object): max_documents = owner._meta['max_documents'] if collection in db.collection_names(): - self._collection = db[collection] + self._collections[db] = db[collection] # The collection already exists, check if its capped # options match the specified capped options - options = self._collection.options() + options = self._collections[db].options() if options.get('max') != max_documents or \ options.get('size') != max_size: msg = ('Cannot create collection "%s" as a capped ' @@ -1012,13 +1012,15 @@ class QuerySetManager(object): opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collection = db.create_collection(collection, **opts) + self._collections[db] = db.create_collection( + collection, **opts + ) else: - self._collection = db[collection] + self._collections[db] = db[collection] # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet - queryset = queryset_class(owner, self._collection) + queryset = queryset_class(owner, self._collections[db]) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) From 833fa3d94dcb35fd10131894f1d45f57000e0fbe Mon Sep 17 00:00:00 2001 From: Jaime Date: Wed, 6 Oct 2010 19:56:12 +0100 Subject: [PATCH 83/98] Added note about the use of default parameters --- docs/guide/defining-documents.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..a8e9924f 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -66,6 +66,25 @@ arguments can be set on all fields: :attr:`default` (Default: None) A value to use when no value is set for this field. + The definion of default parameters follow `the general rules on Python + `__, + which means that some care should be taken when dealing with default mutable objects + (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`):: + + class ExampleFirst(Document): + # Default an empty list + values = ListField(IntField(), default=list) + + class ExampleSecond(Document): + # Default a set of values + values = ListField(IntField(), default=lambda: [1,2,3]) + + class ExampleDangerous(Document): + # This can make an .append call to add values to the default (and all the following objects), + # instead to just an object + values = ListField(IntField(), default=[1,2,3]) + + :attr:`unique` (Default: False) When True, no documents in the collection will have the same value for this field. From f6661419816157bec89db3f97b4f8d4a95c67897 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 13:23:11 +0100 Subject: [PATCH 84/98] Added test for list of referencefields --- tests/fields.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index 622016c0..e30f843e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -189,6 +189,9 @@ class FieldTest(unittest.TestCase): def test_list_validation(self): """Ensure that a list field only accepts lists with valid elements. """ + class User(Document): + pass + class Comment(EmbeddedDocument): content = StringField() @@ -196,6 +199,7 @@ class FieldTest(unittest.TestCase): content = StringField() comments = ListField(EmbeddedDocumentField(Comment)) tags = ListField(StringField()) + authors = ListField(ReferenceField(User)) post = BlogPost(content='Went for a walk today...') post.validate() @@ -210,15 +214,21 @@ class FieldTest(unittest.TestCase): post.tags = ('fun', 'leisure') post.validate() - comments = [Comment(content='Good for you'), Comment(content='Yay.')] - post.comments = comments - post.validate() - post.comments = ['a'] self.assertRaises(ValidationError, post.validate) post.comments = 'yay' self.assertRaises(ValidationError, post.validate) + comments = [Comment(content='Good for you'), Comment(content='Yay.')] + post.comments = comments + post.validate() + + post.authors = [Comment()] + self.assertRaises(ValidationError, post.validate) + + post.authors = [User()] + post.validate() + def test_sorted_list_sorting(self): """Ensure that a sorted list field properly sorts values. """ From 3591593ac73dbd0a2275e5335b0ceaace232048d Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 13:55:48 +0100 Subject: [PATCH 85/98] Fixed GenericReferenceField query issue --- mongoengine/fields.py | 2 +- mongoengine/queryset.py | 3 --- tests/fields.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f65ca15d..9fc50d68 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -509,7 +509,7 @@ class GenericReferenceField(BaseField): return {'_cls': document.__class__.__name__, '_ref': ref} def prepare_query_value(self, op, value): - return self.to_mongo(value)['_ref'] + return self.to_mongo(value) class BinaryField(BaseField): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a3d4c544..06524dcd 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -624,9 +624,6 @@ class QuerySet(object): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] - if field.__class__.__name__ == 'GenericReferenceField': - parts.append('_ref') - # if op and op not in match_operators: if op: if op in geo_operators: diff --git a/tests/fields.py b/tests/fields.py index 1df484bf..e30f843e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -545,7 +545,6 @@ class FieldTest(unittest.TestCase): user.save() user = User.objects(bookmarks__all=[post_1, link_1]).first() - print User.objects(bookmarks__all=[post_1, link_1]).explain() self.assertEqual(user.bookmarks[0], post_1) self.assertEqual(user.bookmarks[1], link_1) From 26723992e30d1db03c40250c7635374d0cc502d8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:14:05 +0100 Subject: [PATCH 86/98] Combined Q-object tests --- tests/queryset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 32d1902e..38e1cb33 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1393,9 +1393,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - -class NewQTest(unittest.TestCase): - def test_and_combination(self): """Ensure that Q-objects correctly AND together. """ From 012352cf24e55558c9638ff29ad36010f98e34a6 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:21:55 +0100 Subject: [PATCH 87/98] Added snapshot and timeout methods to QuerySet --- mongoengine/queryset.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 06524dcd..ead659d0 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -401,6 +401,8 @@ class QuerySet(object): self._where_clause = None self._loaded_fields = [] self._ordering = [] + self._snapshot = False + self._timeout = True # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -534,9 +536,12 @@ class QuerySet(object): @property def _cursor(self): if self._cursor_obj is None: - cursor_args = {} + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout, + } if self._loaded_fields: - cursor_args = {'fields': self._loaded_fields} + cursor_args['fields'] = self._loaded_fields self._cursor_obj = self._collection.find(self._query, **cursor_args) # Apply where clauses to cursor @@ -967,6 +972,20 @@ class QuerySet(object): plan = pprint.pformat(plan) return plan + def snapshot(self, enabled): + """Enable or disable snapshot mode when querying. + + :param enabled: whether or not snapshot mode is enabled + """ + self._snapshot = enabled + + def timeout(self, enabled): + """Enable or disable the default mongod timeout when querying. + + :param enabled: whether or not the timeout is used + """ + self._timeout = enabled + def delete(self, safe=False): """Delete the documents matched by the query. From 36993029ad4aa5b3a929e68b626eec083078738b Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:22:45 +0100 Subject: [PATCH 88/98] Removed old Q-object implementation --- mongoengine/queryset.py | 118 ---------------------------------------- 1 file changed, 118 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ead659d0..f6296dd2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -268,124 +268,6 @@ class Q(QNode): return not bool(self.query) -class OldQ(object): - - OR = '||' - AND = '&&' - OPERATORS = { - 'eq': ('((this.%(field)s instanceof Array) && ' - ' this.%(field)s.indexOf(%(value)s) != -1) ||' - ' this.%(field)s == %(value)s'), - 'ne': 'this.%(field)s != %(value)s', - 'gt': 'this.%(field)s > %(value)s', - 'gte': 'this.%(field)s >= %(value)s', - 'lt': 'this.%(field)s < %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'in': '%(value)s.indexOf(this.%(field)s) != -1', - 'nin': '%(value)s.indexOf(this.%(field)s) == -1', - 'mod': '%(field)s %% %(value)s', - 'all': ('%(value)s.every(function(a){' - 'return this.%(field)s.indexOf(a) != -1 })'), - 'size': 'this.%(field)s.length == %(value)s', - 'exists': 'this.%(field)s != null', - 'regex_eq': '%(value)s.test(this.%(field)s)', - 'regex_ne': '!%(value)s.test(this.%(field)s)', - } - - def __init__(self, **query): - self.query = [query] - - def _combine(self, other, op): - obj = Q() - if not other.query[0]: - return self - if self.query[0]: - obj.query = (['('] + copy.deepcopy(self.query) + [op] + - copy.deepcopy(other.query) + [')']) - else: - obj.query = copy.deepcopy(other.query) - return obj - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - def as_js(self, document): - js = [] - js_scope = {} - for i, item in enumerate(self.query): - if isinstance(item, dict): - item_query = QuerySet._transform_query(document, **item) - # item_query will values will either be a value or a dict - js.append(self._item_query_as_js(item_query, js_scope, i)) - else: - js.append(item) - return pymongo.code.Code(' '.join(js), js_scope) - - def _item_query_as_js(self, item_query, js_scope, item_num): - # item_query will be in one of the following forms - # {'age': 25, 'name': 'Test'} - # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} - # {'age': {'$lt': 25, '$gt': 18}} - js = [] - for i, (key, value) in enumerate(item_query.items()): - op = 'eq' - # Construct a variable name for the value in the JS - value_name = 'i%sf%s' % (item_num, i) - if isinstance(value, dict): - # Multiple operators for this field - for j, (op, value) in enumerate(value.items()): - # Create a custom variable name for this operator - op_value_name = '%so%s' % (value_name, j) - # Construct the JS that uses this op - value, operation_js = self._build_op_js(op, key, value, - op_value_name) - # Update the js scope with the value for this op - js_scope[op_value_name] = value - js.append(operation_js) - else: - # Construct the JS for this field - value, field_js = self._build_op_js(op, key, value, value_name) - js_scope[value_name] = value - js.append(field_js) - return ' && '.join(js) - - def _build_op_js(self, op, key, value, value_name): - """Substitute the values in to the correct chunk of Javascript. - """ - if isinstance(value, RE_TYPE): - # Regexes are handled specially - if op.strip('$') == 'ne': - op_js = Q.OPERATORS['regex_ne'] - else: - op_js = Q.OPERATORS['regex_eq'] - else: - op_js = Q.OPERATORS[op.strip('$')] - - # Comparing two ObjectIds in Javascript doesn't work.. - if isinstance(value, pymongo.objectid.ObjectId): - value = unicode(value) - - # Handle DBRef - if isinstance(value, pymongo.dbref.DBRef): - op_js = '(this.%(field)s.$id == "%(id)s" &&'\ - ' this.%(field)s.$ref == "%(ref)s")' % { - 'field': key, - 'id': unicode(value.id), - 'ref': unicode(value.collection) - } - value = None - - # Perform the substitution - operation_js = op_js % { - 'field': key, - 'value': value_name - } - return value, operation_js - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. From 6817f3b7ba1b6f25554937179cd112a636140e01 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 15:40:49 +0100 Subject: [PATCH 89/98] Updated docs for v0.4 --- docs/apireference.rst | 2 ++ docs/changelog.rst | 11 ++++++- docs/guide/defining-documents.rst | 6 +++- docs/guide/querying.rst | 51 +++++++++++++++++++++++++++++++ docs/index.rst | 2 +- mongoengine/fields.py | 6 ++++ mongoengine/queryset.py | 1 + 7 files changed, 76 insertions(+), 3 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 4fff317a..34d4536d 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -66,3 +66,5 @@ Fields .. autoclass:: mongoengine.GenericReferenceField .. autoclass:: mongoengine.FileField + +.. autoclass:: mongoengine.GeoPointField diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dd5b00d..64bdf90a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,16 +4,25 @@ Changelog Changes in v0.4 =============== +- New Q-object implementation, which is no longer based on Javascript - Added ``SortedListField`` - Added ``EmailField`` - Added ``GeoPointField`` - Added ``exact`` and ``iexact`` match operators to ``QuerySet`` - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts -- Fixed bug in Q-objects +- Added new query operators for Geo queries +- Added ``not`` query operator +- Added new update operators: ``pop`` and ``add_to_set`` +- Added ``__raw__`` query parameter - Fixed document inheritance primary key issue +- Added support for querying by array element position - Base class can now be defined for ``DictField`` - Fixed MRO error that occured on document inheritance +- Added ``QuerySet.distinct``, ``QuerySet.create``, ``QuerySet.snapshot``, + ``QuerySet.timeout`` and ``QuerySet.all`` +- Subsequent calls to ``connect()`` now work - Introduced ``min_length`` for ``StringField`` +- Fixed multi-process connection issue - Other minor fixes Changes in v0.3 diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index dc136f27..106d4ec8 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -47,11 +47,11 @@ are as follows: * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` * :class:`~mongoengine.BooleanField` -* :class:`~mongoengine.GeoLocationField` * :class:`~mongoengine.FileField` * :class:`~mongoengine.EmailField` * :class:`~mongoengine.SortedListField` * :class:`~mongoengine.BinaryField` +* :class:`~mongoengine.GeoPointField` Field arguments --------------- @@ -298,6 +298,10 @@ or a **-** sign. Note that direction only matters on multi-field indexes. :: meta = { 'indexes': ['title', ('title', '-rating')] } + +.. note:: + Geospatial indexes will be automatically created for all + :class:`~mongoengine.GeoPointField`\ s Ordering ======== diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index bef19bc5..58bf9f63 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -53,6 +53,16 @@ lists that contain that item will be matched:: # 'tags' list Page.objects(tags='coding') +Raw queries +----------- +It is possible to provide a raw PyMongo query as a query parameter, which will +be integrated directly into the query. This is done using the ``__raw__`` +keyword argument:: + + Page.objects(__raw__={'tags': 'coding'}) + +.. versionadded:: 0.4 + Query operators =============== Operators other than equality may also be used in queries; just attach the @@ -68,6 +78,8 @@ Available operators are as follows: * ``lte`` -- less than or equal to * ``gt`` -- greater than * ``gte`` -- greater than or equal to +* ``not`` -- negate a standard check, may be used before other operators (e.g. + ``Q(age__not__mod=5)``) * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values @@ -89,6 +101,27 @@ expressions: .. versionadded:: 0.3 +There are a few special operators for performing geographical queries, that +may used with :class:`~mongoengine.GeoPointField`\ s: + +* ``within_distance`` -- provide a list containing a point and a maximum + distance (e.g. [(41.342, -87.653), 5]) +* ``within_box`` -- filter documents to those within a given bounding box (e.g. + [(35.0, -125.0), (40.0, -100.0)]) +* ``near`` -- order the documents by how close they are to a given point + +.. versionadded:: 0.4 + +Querying by position +==================== +It is possible to query by position in a list by using a numerical value as a +query operator. So if you wanted to find all pages whose first tag was ``db``, +you could use the following query:: + + BlogPost.objects(tags__0='db') + +.. versionadded:: 0.4 + Limiting and skipping results ============================= Just as with traditional ORMs, you may limit the number of results returned, or @@ -181,6 +214,22 @@ custom manager methods as you like:: assert len(BlogPost.objects) == 2 assert len(BlogPost.live_posts) == 1 +Custom QuerySets +================ +Should you want to add custom methods for interacting with or filtering +documents, extending the :class:`~mongoengine.queryset.QuerySet` class may be +the way to go. To use a custom :class:`~mongoengine.queryset.QuerySet` class on +a document, set ``queryset_class`` to the custom class in a +:class:`~mongoengine.Document`\ s ``meta`` dictionary:: + + class AwesomerQuerySet(QuerySet): + pass + + class Page(Document): + meta = {'queryset_class': AwesomerQuerySet} + +.. versionadded:: 0.4 + Aggregation =========== MongoDB provides some aggregation methods out of the box, but there are not as @@ -402,8 +451,10 @@ that you may use with these methods: * ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list +* ``pop`` -- remove the first or last element of a list * ``pull`` -- remove a value from a list * ``pull_all`` -- remove several values from a list +* ``add_to_set`` -- add value to a list only if its not in the list already The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: diff --git a/docs/index.rst b/docs/index.rst index a28b344c..ccb7fbe2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ MongoDB. To install it, simply run .. code-block:: console - # easy_install -U mongoengine + # pip install -U mongoengine The source is available on `GitHub `_. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9fc50d68..62d2ef2f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -112,6 +112,8 @@ class URLField(StringField): class EmailField(StringField): """A field that validates input as an E-Mail-Address. + + .. versionadded:: 0.4 """ EMAIL_REGEX = re.compile( @@ -355,6 +357,8 @@ class SortedListField(ListField): """A ListField that sorts the contents of its list before writing to the database in order to ensure that a sorted list is always retrieved. + + .. versionadded:: 0.4 """ _ordering = None @@ -666,6 +670,8 @@ class FileField(BaseField): class GeoPointField(BaseField): """A list storing a latitude and longitude. + + .. versionadded:: 0.4 """ _geo_index = True diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f6296dd2..86f8b1d5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -590,6 +590,7 @@ class QuerySet(object): def create(self, **kwargs): """Create new object. Returns the saved object instance. + .. versionadded:: 0.4 """ doc = self._document(**kwargs) From 007f116bfaec40205626075ef9050ab35b1753df Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 15:42:31 +0100 Subject: [PATCH 90/98] Increment version to 0.4 --- docs/changelog.rst | 1 + mongoengine/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64bdf90a..277d8e96 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,6 +14,7 @@ Changes in v0.4 - Added ``not`` query operator - Added new update operators: ``pop`` and ``add_to_set`` - Added ``__raw__`` query parameter +- Added support for custom querysets - Fixed document inheritance primary key issue - Added support for querying by array element position - Base class can now be defined for ``DictField`` diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index e01d31ae..6d18ffe7 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 3, 0) +VERSION = (0, 4, 0) def get_version(): version = '%s.%s' % (VERSION[0], VERSION[1]) From dcec61e9b2738f9fc3bd0f09a2cb1bc631438c66 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 16:36:22 +0100 Subject: [PATCH 91/98] Raise AttributeError when necessary on QuerySet[] --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 86f8b1d5..a8d739d9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -787,6 +787,7 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) + raise AttributeError def distinct(self, field): """Return a list of distinct values for a given field. From e93c4c87d8953db1c4da24415c0da3c463523ab9 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 17:41:20 +0100 Subject: [PATCH 92/98] Fixed inheritance collection issue --- mongoengine/queryset.py | 15 +++++++-------- tests/document.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a8d739d9..7422bb8f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1146,9 +1146,8 @@ class QuerySetManager(object): return self db = _get_db() - if db not in self._collections: - collection = owner._meta['collection'] - + collection = owner._meta['collection'] + if (db, collection) not in self._collections: # Create collection as a capped collection if specified if owner._meta['max_size'] or owner._meta['max_documents']: # Get max document limit and max byte size from meta @@ -1156,10 +1155,10 @@ class QuerySetManager(object): max_documents = owner._meta['max_documents'] if collection in db.collection_names(): - self._collections[db] = db[collection] + self._collections[(db, collection)] = db[collection] # The collection already exists, check if its capped # options match the specified capped options - options = self._collections[db].options() + options = self._collections[(db, collection)].options() if options.get('max') != max_documents or \ options.get('size') != max_size: msg = ('Cannot create collection "%s" as a capped ' @@ -1170,15 +1169,15 @@ class QuerySetManager(object): opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collections[db] = db.create_collection( + self._collections[(db, collection)] = db.create_collection( collection, **opts ) else: - self._collections[db] = db[collection] + self._collections[(db, collection)] = db[collection] # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet - queryset = queryset_class(owner, self._collections[db]) + queryset = queryset_class(owner, self._collections[(db, collection)]) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/document.py b/tests/document.py index c2e0d6f9..aa901813 100644 --- a/tests/document.py +++ b/tests/document.py @@ -200,6 +200,37 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() self.assertFalse(collection in self.db.collection_names()) + def test_inherited_collections(self): + """Ensure that subclassed documents don't override parents' collections. + """ + class Drink(Document): + name = StringField() + + class AlcoholicDrink(Drink): + meta = {'collection': 'booze'} + + class Drinker(Document): + drink = GenericReferenceField() + + Drink.drop_collection() + AlcoholicDrink.drop_collection() + Drinker.drop_collection() + + red_bull = Drink(name='Red Bull') + red_bull.save() + + programmer = Drinker(drink=red_bull) + programmer.save() + + beer = AlcoholicDrink(name='Beer') + beer.save() + + real_person = Drinker(drink=beer) + real_person.save() + + self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) + self.assertEqual(Drinker.objects[1].drink.name, beer.name) + def test_capped_collection(self): """Ensure that capped collections work properly. """ From dc7181a3fdef5e5a1724683677fd5bc21298db8f Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Sun, 17 Oct 2010 23:43:58 +0100 Subject: [PATCH 93/98] Begun GridFS documentation --- docs/guide/gridfs.rst | 26 ++++++++++++++++++++++++++ docs/guide/index.rst | 1 + mongoengine/django/auth.py | 7 +++---- mongoengine/fields.py | 1 + 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 docs/guide/gridfs.rst diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst new file mode 100644 index 00000000..7c6e8acd --- /dev/null +++ b/docs/guide/gridfs.rst @@ -0,0 +1,26 @@ +====== +GridFS +====== +GridFS support comes in the form of the :class:`~mongoengine.FileField` field +object. This field acts as a file-like object and provides a couple of +different ways of inserting and retrieving data. Metadata such as content-type +can also be stored alongside the stored files. In the following example, an +document is created to store details about animals, including a photo: + + class Animal(Document): + genus = StringField() + family = StringField() + photo = FileField() + + marmot = Animal('Marmota', 'Sciuridae') + + marmot_photo = open('marmot.jpg') # Retrieve a photo from disk + marmot.photo = marmot_photo # Store the photo in the document + + marmot.save() + +So adding file data to a document is as easy as adding data to any other + +.. versionadded:: 0.4 + + diff --git a/docs/guide/index.rst b/docs/guide/index.rst index 7fdfe932..aac72469 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -10,3 +10,4 @@ User Guide defining-documents document-instances querying + gridfs diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index da0005c8..595852ef 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -75,10 +75,9 @@ class User(Document): email address. """ now = datetime.datetime.now() - + # Normalize the address by lowercasing the domain part of the email # address. - # Not sure why we'r allowing null email when its not allowed in django if email is not None: try: email_name, domain_part = email.strip().split('@', 1) @@ -86,12 +85,12 @@ class User(Document): pass else: email = '@'.join([email_name, domain_part.lower()]) - + user = User(username=username, email=email, date_joined=now) user.set_password(password) user.save() return user - + def get_and_delete_messages(self): return [] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 86da2e3d..23a88dee 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,6 +20,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', RECURSIVE_REFERENCE_CONSTANT = 'self' + class StringField(BaseField): """A unicode string field. """ From 0902b957641e09861f0864e09501b174624577f0 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 18 Oct 2010 00:27:40 +0100 Subject: [PATCH 94/98] Added support for recursive embedded documents --- mongoengine/base.py | 7 +++---- mongoengine/fields.py | 33 +++++++++++++++++++++------------ tests/fields.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 1f7ba1fe..2253e4a2 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -204,6 +204,9 @@ class DocumentMetaclass(type): exc = subclass_exception('MultipleObjectsReturned', base_excs, module) new_class.add_to_class('MultipleObjectsReturned', exc) + global _document_registry + _document_registry[name] = new_class + return new_class def add_to_class(self, name, value): @@ -216,8 +219,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): """ def __new__(cls, name, bases, attrs): - global _document_registry - super_new = super(TopLevelDocumentMetaclass, cls).__new__ # Classes defined in this package are abstract and should not have # their own metadata with DB collection, etc. @@ -322,8 +323,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._fields['id'] = ObjectIdField(db_field='_id') new_class.id = new_class._fields['id'] - _document_registry[name] = new_class - return new_class diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 62d2ef2f..63107f23 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -233,33 +233,43 @@ class EmbeddedDocumentField(BaseField): :class:`~mongoengine.EmbeddedDocument`. """ - def __init__(self, document, **kwargs): - if not issubclass(document, EmbeddedDocument): - raise ValidationError('Invalid embedded document class provided ' - 'to an EmbeddedDocumentField') - self.document = document + def __init__(self, document_type, **kwargs): + if not isinstance(document_type, basestring): + if not issubclass(document_type, EmbeddedDocument): + raise ValidationError('Invalid embedded document class ' + 'provided to an EmbeddedDocumentField') + self.document_type_obj = document_type super(EmbeddedDocumentField, self).__init__(**kwargs) + @property + def document_type(self): + if isinstance(self.document_type_obj, basestring): + if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: + self.document_type_obj = self.owner_document + else: + self.document_type_obj = get_document(self.document_type_obj) + return self.document_type_obj + def to_python(self, value): - if not isinstance(value, self.document): - return self.document._from_son(value) + if not isinstance(value, self.document_type): + return self.document_type._from_son(value) return value def to_mongo(self, value): - return self.document.to_mongo(value) + return self.document_type.to_mongo(value) def validate(self, value): """Make sure that the document instance is an instance of the EmbeddedDocument subclass provided when the document was defined. """ # Using isinstance also works for subclasses of self.document - if not isinstance(value, self.document): + if not isinstance(value, self.document_type): raise ValidationError('Invalid embedded document instance ' 'provided to an EmbeddedDocumentField') - self.document.validate(value) + self.document_type.validate(value) def lookup_member(self, member_name): - return self.document._fields.get(member_name) + return self.document_type._fields.get(member_name) def prepare_query_value(self, op, value): return self.to_mongo(value) @@ -413,7 +423,6 @@ class ReferenceField(BaseField): raise ValidationError('Argument to ReferenceField constructor ' 'must be a document class or a string') self.document_type_obj = document_type - self.document_obj = None super(ReferenceField, self).__init__(**kwargs) @property diff --git a/tests/fields.py b/tests/fields.py index e30f843e..208b4643 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -423,6 +423,36 @@ class FieldTest(unittest.TestCase): self.assertEqual(peter.boss, bill) self.assertEqual(peter.friends, friends) + def test_recursive_embedding(self): + """Ensure that EmbeddedDocumentFields can contain their own documents. + """ + class Tree(Document): + name = StringField() + children = ListField(EmbeddedDocumentField('TreeNode')) + + class TreeNode(EmbeddedDocument): + name = StringField() + children = ListField(EmbeddedDocumentField('self')) + + tree = Tree(name="Tree") + + first_child = TreeNode(name="Child 1") + tree.children.append(first_child) + + second_child = TreeNode(name="Child 2") + first_child.children.append(second_child) + + third_child = TreeNode(name="Child 3") + first_child.children.append(third_child) + + tree.save() + + tree_obj = Tree.objects.first() + self.assertEqual(len(tree.children), 1) + self.assertEqual(tree.children[0].name, first_child.name) + self.assertEqual(tree.children[0].children[0].name, second_child.name) + self.assertEqual(tree.children[0].children[1].name, third_child.name) + def test_undefined_reference(self): """Ensure that ReferenceFields may reference undefined Documents. """ From 67736c849dd65d6dd96cef6e7bcc41cdb8503c22 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 00:55:44 +0100 Subject: [PATCH 95/98] Finished GridFS Documentation * Also made GridFS replace test pass --- docs/django.rst | 32 ++++++++++--------- docs/guide/gridfs.rst | 71 +++++++++++++++++++++++++++++++++++++---- docs/guide/querying.rst | 8 ++--- mongoengine/fields.py | 5 +-- 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index 2cce3f02..8a490571 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -47,9 +47,10 @@ into you settings module:: Storage ======= -With MongoEngine's support for GridFS via the FileField, it is useful to have a -Django file storage backend that wraps this. The new storage module is called -GridFSStorage. Using it is very similar to using the default FileSystemStorage.:: +With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, +it is useful to have a Django file storage backend that wraps this. The new +storage module is called :class:`~mongoengine.django.GridFSStorage`. Using it +is very similar to using the default FileSystemStorage.:: fs = mongoengine.django.GridFSStorage() @@ -57,29 +58,30 @@ GridFSStorage. Using it is very similar to using the default FileSystemStorage.: All of the `Django Storage API methods `_ have been -implemented except ``path()``. If the filename provided already exists, an +implemented except :func:`path`. If the filename provided already exists, an underscore and a number (before # the file extension, if one exists) will be appended to the filename until the generated filename doesn't exist. The -``save()`` method will return the new filename.:: +:func:`save` method will return the new filename.:: - > fs.exists('hello.txt') + >>> fs.exists('hello.txt') True - > fs.open('hello.txt').read() + >>> fs.open('hello.txt').read() 'Hello, World!' - > fs.size('hello.txt') + >>> fs.size('hello.txt') 13 - > fs.url('hello.txt') + >>> fs.url('hello.txt') 'http://your_media_url/hello.txt' - > fs.open('hello.txt').name + >>> fs.open('hello.txt').name 'hello.txt' - > fs.listdir() + >>> fs.listdir() ([], [u'hello.txt']) -All files will be saved and retrieved in GridFS via the ``FileDocument`` document, -allowing easy access to the files without the GridFSStorage backend.:: +All files will be saved and retrieved in GridFS via the :class::`FileDocument` +document, allowing easy access to the files without the GridFSStorage +backend.:: - > from mongoengine.django.storage import FileDocument - > FileDocument.objects() + >>> from mongoengine.django.storage import FileDocument + >>> FileDocument.objects() [] .. versionadded:: 0.4 diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 7c6e8acd..503610c9 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -1,11 +1,17 @@ ====== GridFS ====== + +.. versionadded:: 0.4 + +Writing +------- + GridFS support comes in the form of the :class:`~mongoengine.FileField` field object. This field acts as a file-like object and provides a couple of -different ways of inserting and retrieving data. Metadata such as content-type -can also be stored alongside the stored files. In the following example, an -document is created to store details about animals, including a photo: +different ways of inserting and retrieving data. Arbitrary metadata such as +content type can also be stored alongside the files. In the following example, +a document is created to store details about animals, including a photo:: class Animal(Document): genus = StringField() @@ -14,13 +20,64 @@ document is created to store details about animals, including a photo: marmot = Animal('Marmota', 'Sciuridae') - marmot_photo = open('marmot.jpg') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store the photo in the document + marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk + marmot.photo = marmot_photo # Store the photo in the document + marmot.photo.content_type = 'image/jpeg' # Store metadata marmot.save() -So adding file data to a document is as easy as adding data to any other +Another way of writing to a :class:`~mongoengine.FileField` is to use the +:func:`put` method. This allows for metadata to be stored in the same call as +the file:: -.. versionadded:: 0.4 + marmot.photo.put(marmot_photo, content_type='image/jpeg') + + marmot.save() + +Retrieval +--------- + +So using the :class:`~mongoengine.FileField` is just like using any other +field. The file can also be retrieved just as easily:: + + marmot = Animal.objects('Marmota').first() + photo = marmot.photo.read() + content_type = marmot.photo.content_type + +Streaming +--------- + +Streaming data into a :class:`~mongoengine.FileField` is achieved in a +slightly different manner. First, a new file must be created by calling the +:func:`new_file` method. Data can then be written using :func:`write`:: + + marmot.photo.new_file() + marmot.photo.write('some_image_data') + marmot.photo.write('some_more_image_data') + marmot.photo.close() + + marmot.photo.save() + +Deletion +-------- + +Deleting stored files is achieved with the :func:`delete` method:: + + marmot.photo.delete() + +.. note:: + The FileField in a Document actually only stores the ID of a file in a + separate GridFS collection. This means that deleting a document + with a defined FileField does not actually delete the file. You must be + careful to delete any files in a Document as above before deleting the + Document itself. +Replacing files +--------------- + +Files can be replaced with the :func:`replace` method. This works just like +the :func:`put` method so even metadata can (and should) be replaced:: + + another_marmot = open('another_marmot.png', 'r') + marmot.photo.replace(another_marmot, content_type='image/png') diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 58bf9f63..832fed50 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -34,7 +34,7 @@ arguments. The keys in the keyword arguments correspond to fields on the Fields on embedded documents may also be referred to using field lookup syntax by using a double-underscore in place of the dot in object attribute access syntax:: - + # This will return a QuerySet that will only iterate over pages that have # been written by a user whose 'country' field is set to 'uk' uk_pages = Page.objects(author__country='uk') @@ -67,7 +67,7 @@ Query operators =============== Operators other than equality may also be used in queries; just attach the operator name to a key with a double-underscore:: - + # Only find users whose age is 18 or less young_users = Users.objects(age__lte=18) @@ -144,7 +144,7 @@ You may also index the query to retrieve a single result. If an item at that index does not exists, an :class:`IndexError` will be raised. A shortcut for retrieving the first result and returning :attr:`None` if no result exists is provided (:meth:`~mongoengine.queryset.QuerySet.first`):: - + >>> # Make sure there are no users >>> User.drop_collection() >>> User.objects[0] @@ -458,7 +458,7 @@ that you may use with these methods: The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: - + >>> post = BlogPost(title='Test', page_views=0, tags=['database']) >>> post.save() >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 30a508d1..ef9540fc 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -586,14 +586,14 @@ class GridFSProxy(object): def put(self, file, **kwargs): if self.grid_id: - raise GridFSError('This document alreay has a file. Either delete ' + raise GridFSError('This document already has a file. Either delete ' 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): if self.grid_id: if not self.newfile: - raise GridFSError('This document alreay has a file. Either ' + raise GridFSError('This document already has a file. Either ' 'delete it or call replace to overwrite it') else: self.new_file() @@ -622,6 +622,7 @@ class GridFSProxy(object): def replace(self, file, **kwargs): self.delete() + self.grid_id = None self.put(file, **kwargs) def close(self): From 5580b003b5aca4dc272c1e046f0100657faf1a25 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 01:30:32 +0100 Subject: [PATCH 96/98] Added self to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 93ecfa8d..93fe819e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Harry Marr Matt Dennewitz Deepak Thukral Florian Schlachter +Steve Challis From d6cb5b9abe64309b4e57dd36888bac4c0f34e6d8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 18 Oct 2010 10:21:23 +0100 Subject: [PATCH 97/98] Removed invalid connection tests --- tests/connnection.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 tests/connnection.py diff --git a/tests/connnection.py b/tests/connnection.py deleted file mode 100644 index 1903a5f4..00000000 --- a/tests/connnection.py +++ /dev/null @@ -1,44 +0,0 @@ -import unittest -import datetime -import pymongo - -import mongoengine.connection -from mongoengine import * -from mongoengine.connection import _get_db, _get_connection - - -class ConnectionTest(unittest.TestCase): - - def tearDown(self): - mongoengine.connection._connection_settings = {} - mongoengine.connection._connections = {} - mongoengine.connection._dbs = {} - - def test_connect(self): - """Ensure that the connect() method works properly. - """ - connect('mongoenginetest') - - conn = _get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) - - db = _get_db() - self.assertTrue(isinstance(db, pymongo.database.Database)) - self.assertEqual(db.name, 'mongoenginetest') - - def test_register_connection(self): - """Ensure that connections with different aliases may be registered. - """ - register_connection('testdb', 'mongoenginetest2') - - self.assertRaises(ConnectionError, _get_connection) - conn = _get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) - - db = _get_db('testdb') - self.assertTrue(isinstance(db, pymongo.database.Database)) - self.assertEqual(db.name, 'mongoenginetest2') - - -if __name__ == '__main__': - unittest.main() From d7c42861fbcf32754d53559fbc0863ad78c8cabd Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 10:25:06 +0100 Subject: [PATCH 98/98] Minor GridFS corrections --- docs/guide/gridfs.rst | 4 ++-- mongoengine/fields.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 503610c9..0cd06539 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -21,7 +21,7 @@ a document is created to store details about animals, including a photo:: marmot = Animal('Marmota', 'Sciuridae') marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store the photo in the document + marmot.photo = marmot_photo # Store photo in the document marmot.photo.content_type = 'image/jpeg' # Store metadata marmot.save() @@ -40,7 +40,7 @@ Retrieval So using the :class:`~mongoengine.FileField` is just like using any other field. The file can also be retrieved just as easily:: - marmot = Animal.objects('Marmota').first() + marmot = Animal.objects(genus='Marmota').first() photo = marmot.photo.read() content_type = marmot.photo.content_type diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ef9540fc..e95fd65e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -614,15 +614,10 @@ class GridFSProxy(object): def delete(self): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) - - #self.grid_id = None - # Doesn't make a difference because will be put back in when - # reinstantiated We should delete all the metadata stored with the - # file too + self.grid_id = None def replace(self, file, **kwargs): self.delete() - self.grid_id = None self.put(file, **kwargs) def close(self):