From a4d2f22fd2016e47987eb8c48f541db99eafb067 Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:14:01 -0500 Subject: [PATCH 01/49] added 'geo_indexes' to TopLevelDocumentMetaclass; added GeoPointField, a glorified [lat float, lng float] container; added geo lookup operators to QuerySet; added initial geo tests --- mongoengine/base.py | 1 + mongoengine/fields.py | 17 ++++++++++++- mongoengine/queryset.py | 26 ++++++++++++++++---- tests/queryset.py | 54 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0a904467..e4b45f26 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -206,6 +206,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'max_size': None, 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime + 'geo_indexes': [], 'id_field': id_field, } diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9a9f4e0e..bb61f1da 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField'] + 'BinaryField', 'GeoPointField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -443,6 +443,7 @@ class GenericReferenceField(BaseField): def prepare_query_value(self, op, value): return self.to_mongo(value)['_ref'] + class BinaryField(BaseField): """A binary data field. """ @@ -462,3 +463,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + + +class GeoPointField(BaseField): + """A list storing a latitude and longitude. + """ + + def validate(self, value): + assert isinstance(value, (list, tuple)) + + if not len(value) == 2: + raise ValidationError('Value must be a two-dimensional point.') + if not isinstance(value[0], (float, int)) and \ + not isinstance(value[1], (float, int)): + raise ValidationError('Both values in point must be float or int.') diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11dc2bcb..f03f97a5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -232,12 +232,17 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - #self.ensure_index(key_or_list) self._collection.ensure_index(key_or_list) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True) + + if self._document._meta['geo_indexes'] and \ + pymongo.version >= "1.5.1": + from pymongo import GEO2D + for index in self._document._meta['geo_indexes']: + self._collection.ensure_index([(index, GEO2D)]) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: @@ -298,6 +303,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists'] + geo_operators = ['within_distance', 'within_box', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith'] @@ -306,7 +312,7 @@ class QuerySet(object): parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in operators + match_operators: + if parts[-1] in operators + match_operators + geo_operators: op = parts.pop() if _doc_cls: @@ -320,15 +326,25 @@ class QuerySet(object): singular_ops += match_operators if op in singular_ops: value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all'): + elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] if field.__class__.__name__ == 'GenericReferenceField': parts.append('_ref') - if op and op not in match_operators: - value = {'$' + op: value} + # if op and op not in match_operators: + if op: + if op in geo_operators: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "near": + value = {'$near': value} + else: + raise NotImplmenetedError, \ + "Geo method has been implemented" + elif op not in match_operators: + value = {'$' + op: value} key = '.'.join(parts) if op is None or key not in mongo_query: diff --git a/tests/queryset.py b/tests/queryset.py index c0bd8fa9..004c4160 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,6 +1070,60 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() + def test_near(self): + """Ensure that "near" queries work with and without radii. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + def __unicode__(self): + return self.title + + meta = {'geo_indexes': ["location"]} + + Event.drop_collection() + + event1 = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + location=[37.7749295, -122.4194155]) + event3 = Event(title="Coltrane Motion @ Empty Bottle", + location=[41.900474, -87.686638]) + + event1.save() + event2.save() + event3.save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(location__near=[41.9120459, -87.67892]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + # find events within 5 miles of pitchfork office, chicago + point_and_distance = [[41.9120459, -87.67892], 5] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 2) + events = list(events) + self.assertTrue(event2 not in events) + self.assertTrue(event1 in events) + self.assertTrue(event3 in events) + + # find events around san francisco + point_and_distance = [[37.7566023, -122.415579], 10] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 1 mile of greenpoint, broolyn, nyc, ny + point_and_distance = [[40.7237134, -73.9509714], 1] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 0) + + Event.drop_collection() + class QTest(unittest.TestCase): From 600ca3bcf9d9f87129f8504286f7e2108154b3cd Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:57:26 -0500 Subject: [PATCH 02/49] renamed 'test_near' to 'test_geospatial_operators', updated added ordering checks to test --- tests/queryset.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 004c4160..c8db29a2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,11 +1070,12 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() - def test_near(self): - """Ensure that "near" queries work with and without radii. + def test_geospatial_operators(self): + """Ensure that geospatial queries are working. """ class Event(Document): title = StringField() + date = DateTimeField() location = GeoPointField() def __unicode__(self): @@ -1085,10 +1086,13 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", + date=datetime.now() - timedelta(days=1), location=[41.909889, -87.677137]) event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + date=datetime.now() - timedelta(days=10), location=[37.7749295, -122.4194155]) event3 = Event(title="Coltrane Motion @ Empty Bottle", + date=datetime.now(), location=[41.900474, -87.686638]) event1.save() @@ -1111,6 +1115,12 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(event1 in events) self.assertTrue(event3 in events) + # ensure ordering is respected by "near" + events = Event.objects(location__near=[41.9120459, -87.67892]) + events = events.order_by("-date") + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event3, event1, event2]) + # find events around san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -1122,6 +1132,13 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 0) + # ensure ordering is respected by "within_distance" + point_and_distance = [[41.9120459, -87.67892], 10] + events = Event.objects(location__within_distance=point_and_distance) + events = events.order_by("-date") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0], event3) + Event.drop_collection() From 86e2797c577d007b1883164867eaca7548703ea9 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 9 Jun 2010 22:28:30 -0300 Subject: [PATCH 03/49] added a TestCase for tests that uses mongoDB --- mongoengine/django/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 mongoengine/django/tests.py diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py new file mode 100644 index 00000000..a8d7c7ff --- /dev/null +++ b/mongoengine/django/tests.py @@ -0,0 +1,21 @@ +#coding: utf-8 +from django.test import TestCase +from django.conf import settings + +from mongoengine import connect + +class MongoTestCase(TestCase): + """ + TestCase class that clear the collection between the tests + """ + db_name = 'test_%s' % settings.MONGO_DATABASE_NAME + def __init__(self, methodName='runtest'): + self.db = connect(self.db_name) + super(MongoTestCase, self).__init__(methodName) + + def _post_teardown(self): + super(MongoTestCase, self)._post_teardown() + for collection in self.db.collection_names(): + if collection == 'system.indexes': + continue + self.db.drop_collection(collection) From f5e39c0064e52c03194e09a72c204c409a9ab5b2 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 10:25:31 +0200 Subject: [PATCH 04/49] Allowed _id to be missing when converting to mongo --- mongoengine/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..fe942b10 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -407,11 +407,15 @@ class BaseDocument(object): value = getattr(self, field_name, None) if value is not None: data[field.db_field] = field.to_mongo(value) + else: + data[field.db_field] = None # Only add _cls and _types if allow_inheritance is not False if not (hasattr(self, '_meta') and self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] + if hasattr(self, '_id') and not data['_id']: + del data['_id'] return data @classmethod From 3179c4e4aca20cdaa3daafb7cbc1204d847e2e2f Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 11:25:49 +0200 Subject: [PATCH 05/49] Now only removes _id if none, for real --- mongoengine/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index fe942b10..253c758a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -414,8 +414,11 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if hasattr(self, '_id') and not data['_id']: - del data['_id'] + try: + if not data['_id']: + del data['_id'] + except KeyError: + pass return data @classmethod From b89d71bfa5ffea3d3aff6c23ab0be4bcfefc930e Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 14:17:30 +0200 Subject: [PATCH 06/49] Do not convert None objects --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 253c758a..8137eddf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -450,7 +450,8 @@ class BaseDocument(object): for field_name, field in cls._fields.items(): if field.db_field in data: - data[field_name] = field.to_python(data[field.db_field]) + value = data[field.db_field] + data[field_name] = value if value is None else field.to_python(value) obj = cls(**data) obj._present_fields = present_fields From 71689fcf234ec3cc9349850de4e4afd7af574650 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:00:46 +0100 Subject: [PATCH 07/49] Got within_box working for Geo fields --- docs/changelog.rst | 14 ++++++++++++++ mongoengine/queryset.py | 2 ++ tests/queryset.py | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 479ea21c..6f2d6f1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog ========= +Changes in v0.4 +=============== +- Added ``SortedListField`` +- Added ``EmailField`` +- Added ``GeoLocationField`` +- Added ``exact`` and ``iexact`` match operators to ``QuerySet`` +- Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts +- Fixed bug in Q-objects +- Fixed document inheritance primary key issue +- Base class can now be defined for ``DictField`` +- Fixed MRO error that occured on document inheritance +- Introduced ``min_length`` for ``StringField`` +- Other minor fixes + Changes in v0.3 =============== - Added MapReduce support diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a837c4f..4840b537 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -351,6 +351,8 @@ class QuerySet(object): value = {'$within': {'$center': value}} elif op == "near": value = {'$near': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) diff --git a/tests/queryset.py b/tests/queryset.py index a7719b88..a84c8c50 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1232,6 +1232,12 @@ class QuerySetTest(unittest.TestCase): events = events.order_by("-date") self.assertEqual(events.count(), 2) self.assertEqual(events[0], event3) + + # check that within_box works + box = [(35.0, -125.0), (40.0, -100.0)] + events = Event.objects(location__within_box=box) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event2.id) Event.drop_collection() From c2163ecee5674e4379d6db7ad6b46254e59e2ee4 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:12:14 +0100 Subject: [PATCH 08/49] Added test for Geo indexes --- docs/changelog.rst | 2 +- tests/fields.py | 18 ++++++++++++++++++ tests/queryset.py | 2 -- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f2d6f1a..8dd5b00d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changes in v0.4 =============== - Added ``SortedListField`` - Added ``EmailField`` -- Added ``GeoLocationField`` +- Added ``GeoPointField`` - Added ``exact`` and ``iexact`` match operators to ``QuerySet`` - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts - Fixed bug in Q-objects diff --git a/tests/fields.py b/tests/fields.py index 4050e264..80ce3b67 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -607,6 +607,24 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_geo_indexes(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + Event.drop_collection() + event = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event.save() + + info = Event.objects._collection.index_information() + self.assertTrue(u'location_2d' in info) + self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + + Event.drop_collection() + if __name__ == '__main__': diff --git a/tests/queryset.py b/tests/queryset.py index a84c8c50..4187d550 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1175,8 +1175,6 @@ class QuerySetTest(unittest.TestCase): def __unicode__(self): return self.title - meta = {'geo_indexes': ["location"]} - Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", From acbc741037c00736ce1dbdcd39bd77ddf6f663b3 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Thu, 15 Jul 2010 18:20:29 +0200 Subject: [PATCH 09/49] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..6bfcff9f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -261,6 +261,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 0512dd4c25e0f8395c317725fa149a64025faf3e Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 03:53:39 +0800 Subject: [PATCH 10/49] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22aa..4fff317a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029f..7d9c47f4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e264..e22e6ddd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 6bfd6c322b166d55eb151de33e9eaee7ca214b8a Mon Sep 17 00:00:00 2001 From: martin Date: Fri, 18 Jun 2010 10:41:23 +0800 Subject: [PATCH 11/49] Fixed bug with GeoLocationField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f4..5893049d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -388,7 +388,7 @@ class GeoLocationField(DictField): return {'x': value[0], 'y': value[1]} def to_python(self, value): - return value.keys() + return (value['x'], value['y']) class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on From 47bfeec115a297c1a859543e842797751ab9c14a Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 15:27:21 +0800 Subject: [PATCH 12/49] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c276869..7b8dcd5b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..b6d5a63b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5893049d..ebefbb75 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6ddd..8dddcb3e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 9596a25bb92212806e98fd77b6ced1b79c2ed1bf Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 00:56:16 +0200 Subject: [PATCH 13/49] Fixed documentation bug. --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..1fd2ed57 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -174,7 +174,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset(published=True).filter(published=True) + return queryset.filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From f9057e1a288dea4b99ac1936a75696071476870d Mon Sep 17 00:00:00 2001 From: martin Date: Thu, 24 Jun 2010 00:56:51 +0800 Subject: [PATCH 14/49] Fixed bug in FileField, proxy was not getting the grid_id set --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ebefbb75..d2b41ab9 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -542,6 +542,7 @@ class GridFSProxy(object): return self def get(self, id=None): + if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: return None # File has been deleted @@ -611,7 +612,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() + return self.gridfs.get(id=value) def validate(self, value): assert isinstance(value, GridFSProxy) From ec519f20fa8e1c769ffb6ae766dd00bb36f9eabc Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 01:32:28 +0200 Subject: [PATCH 15/49] Makes the tests compatible to pymongo 1.7+. Not backwards compatible! --- tests/document.py | 14 ++++++++------ tests/fields.py | 2 +- tests/queryset.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/document.py b/tests/document.py index 8bc907c5..1160b353 100644 --- a/tests/document.py +++ b/tests/document.py @@ -264,11 +264,12 @@ class DocumentTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info.values()) + self.assertTrue([('tags', 1)] in info) class ExtendedBlogPost(BlogPost): title = StringField() @@ -278,10 +279,11 @@ class DocumentTest(unittest.TestCase): list(ExtendedBlogPost.objects) info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) - self.assertTrue([('_types', 1), ('title', 1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) + self.assertTrue([('_types', 1), ('title', 1)] in info) BlogPost.drop_collection() diff --git a/tests/fields.py b/tests/fields.py index d95f4d3f..136437b8 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -689,7 +689,7 @@ class FieldTest(unittest.TestCase): info = Event.objects._collection.index_information() self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) Event.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index 4187d550..0424d323 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1087,8 +1087,9 @@ class QuerySetTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() - self.assertTrue([('_types', 1)] in info.values()) - self.assertTrue([('_types', 1), ('date', -1)] in info.values()) + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1)] in info) + self.assertTrue([('_types', 1), ('date', -1)] in info) BlogPost.drop_collection() From 6093e88eebf932d04741364f53de77878198c1f1 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Fri, 16 Jul 2010 00:20:29 +0800 Subject: [PATCH 16/49] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 24c5b569..8e8a97c8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,6 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 03c0fd9ada5eb1c90a9213fe4579e64d9a6a111c Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 19:01:53 +0200 Subject: [PATCH 17/49] Make default value of DictField an empty dict instead of None. --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8e8a97c8..759697ac 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault("default", []) + kwargs.setdefault('default', []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,6 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) + kwargs.setdefault('default', {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From aa00feb6a55eba7f720969828242436103817e64 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 20 Jul 2010 22:46:00 +0200 Subject: [PATCH 18/49] FileField's values are now optional. When no value is applied, no File object is created and referenced. --- mongoengine/base.py | 4 ++-- mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 806d83bb..086c7874 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,8 +339,8 @@ class BaseDocument(object): try: field._validate(value) except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field of type "' + - field.__class__.__name__ + '"') + raise ValidationError('Invalid value for field of type "%s": %s' + % (field.__class__.__name__, value)) elif field.required: raise ValidationError('Field "%s" is required' % field.name) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 759697ac..f84f751b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -591,15 +591,20 @@ class FileField(BaseField): def to_mongo(self, value): # Store the GridFS file id in MongoDB - return self.gridfs.grid_id + if self.gridfs.grid_id is not None: + return self.gridfs.grid_id + return None def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get(id=value) + if self.gridfs.grid_id is not None: + return self.gridfs.get(id=value) + return None def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + if value.grid_id is not None: + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) class GeoPointField(BaseField): """A list storing a latitude and longitude. From be651caa688c31b434d641c270e1bf073b3c576a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:02:37 +0100 Subject: [PATCH 19/49] Removed a couple of sneaky print statements --- mongoengine/queryset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4840b537..f81adb5c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -114,13 +114,11 @@ class Q(object): value, field_js = self._build_op_js(op, key, value, value_name) js_scope[value_name] = value js.append(field_js) - print ' && '.join(js) return ' && '.join(js) def _build_op_js(self, op, key, value, value_name): """Substitute the values in to the correct chunk of Javascript. """ - print op, key, value, value_name if isinstance(value, RE_TYPE): # Regexes are handled specially if op.strip('$') == 'ne': From 9f98025b8c13d9e845f5de68e29e825f5517b245 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:29:02 +0100 Subject: [PATCH 20/49] Added QuerySet.distinct. Closes #44. --- mongoengine/queryset.py | 9 +++++++++ tests/queryset.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f81adb5c..1e42cd1d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -604,6 +604,15 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) + + def distinct(self, field): + """Return a list of distinct values for a given field. + + :param field: the field to select distinct values from + + .. versionadded:: 0.4 + """ + return self._collection.distinct(field) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index 0424d323..3691d89e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -970,6 +970,16 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + def test_distinct(self): + """Ensure that the QuerySet.distinct method works. + """ + self.Person(name='Mr Orange', age=20).save() + self.Person(name='Mr White', age=20).save() + self.Person(name='Mr Orange', age=30).save() + self.assertEqual(self.Person.objects.distinct('name'), + ['Mr Orange', 'Mr White']) + self.assertEqual(self.Person.objects.distinct('age'), [20, 30]) + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From 13316e5380f9886b0d8c65458096311b8382233a Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 17:35:09 +0200 Subject: [PATCH 21/49] Introduced new Document.objects.create, like django has. It creates a new object, saves it and returns the new object instance. --- mongoengine/queryset.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1e42cd1d..e176c54c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,5 +1,6 @@ from connection import _get_db +import pprint import pymongo import re import copy @@ -414,6 +415,14 @@ class QuerySet(object): message = u'%d items returned, instead of 1' % count raise self._document.MultipleObjectsReturned(message) + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + .. versionadded:: 0.4 + """ + doc = self._document(**kwargs) + doc.save() + return doc + def first(self): """Retrieve the first object matching the query. """ @@ -667,7 +676,6 @@ class QuerySet(object): plan = self._cursor.explain() if format: - import pprint plan = pprint.pformat(plan) return plan From 327452622e3081c81b05916c61454a9fd97d992d Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:22:26 +0200 Subject: [PATCH 22/49] Handle DBRefs correctly within Q objects. Closes #55 --- mongoengine/queryset.py | 11 +++++++++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e176c54c..00a7f7a2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -133,6 +133,17 @@ class Q(object): if isinstance(value, pymongo.objectid.ObjectId): value = unicode(value) + # Handle DBRef + if isinstance(value, pymongo.dbref.DBRef): + # this.created_user.$id == "4c4c56f8cc1831418c000000" + op_js = '(this.%(field)s.$id == "%(id)s" &&'\ + ' this.%(field)s.$ref == "%(ref)s")' % { + 'field': key, + 'id': unicode(value.id), + 'ref': unicode(value.collection) + } + value = None + # Perform the substitution operation_js = op_js % { 'field': key, diff --git a/tests/queryset.py b/tests/queryset.py index 3691d89e..8cbd9a40 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1304,6 +1304,20 @@ class QTest(unittest.TestCase): query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) + + def test_q_with_dbref(self): + """Ensure Q objects handle DBRefs correctly""" + class User(Document): + pass + + class Post(Document): + created_user = ReferenceField(User) + + user = User.objects.create() + Post.objects.create(created_user=user) + + self.assertEqual(Post.objects.filter(created_user=user).count(), 1) + self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) if __name__ == '__main__': unittest.main() From 51065e7a4dda96943dadfcbdfcca6111dfa1f9f5 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:33:33 +0200 Subject: [PATCH 23/49] Closes #46 by instantiating a new default instance for every field by request. --- mongoengine/base.py | 5 ++++- mongoengine/fields.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 086c7874..10ff1219 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,7 +52,10 @@ class BaseField(object): # Get value from document instance if available, if not use default value = instance._data.get(self.name) if value is None: - value = self.default + if callable(self.default): # fixes #46 + value = self.default() + else: + value = self.default # Allow callable default values if callable(value): value = value() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..670e3cd3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From 9d82911f6338747411788291175739f436d95709 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:38:24 +0200 Subject: [PATCH 24/49] Added tests for #46. --- tests/fields.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 386c48b116191a754e3cec550ccdbd5246efdc97 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:43:11 +0200 Subject: [PATCH 25/49] Typo. --- tests/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields.py b/tests/fields.py index ef776d4a..6e42ea4d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -694,7 +694,7 @@ class FieldTest(unittest.TestCase): Event.drop_collection() def test_ensure_unique_default_instances(self): - """Ensure that every document has it's own unique default instance.""" + """Ensure that every field has it's own unique default instance.""" class D(Document): data = DictField() data2 = DictField(default=lambda: {}) From 9411b38508ac261f166dac1f8f8adc7f01084334 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:45:49 +0200 Subject: [PATCH 26/49] Removed unnecessary comment. --- mongoengine/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 00a7f7a2..be29ae83 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -135,7 +135,6 @@ class Q(object): # Handle DBRef if isinstance(value, pymongo.dbref.DBRef): - # this.created_user.$id == "4c4c56f8cc1831418c000000" op_js = '(this.%(field)s.$id == "%(id)s" &&'\ ' this.%(field)s.$ref == "%(ref)s")' % { 'field': key, From 2f991ac6f18172008ed71f1d97ab6ee71ea360ba Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 19:02:15 +0200 Subject: [PATCH 27/49] Added all() method to get all document instances from a document. Extended the FileField's tests with testing on empty filefield. --- mongoengine/queryset.py | 4 ++++ tests/fields.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index be29ae83..ffbe5255 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -238,6 +238,10 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) + + def all(self): + """Returns all documents.""" + return self.__call__() @property def _collection(self): diff --git a/tests/fields.py b/tests/fields.py index 6e42ea4d..1e53d23d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -674,7 +674,12 @@ class FieldTest(unittest.TestCase): PutFile.drop_collection() StreamFile.drop_collection() SetFile.drop_collection() - + + # Make sure FileField is optional and not required + class DemoFile(Document): + file = FileField() + d = DemoFile.objects.create() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 7ab2e21c106fc89875b1ae4ae79ecd7e0988ded3 Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:42:10 +0200 Subject: [PATCH 28/49] Handle unsafe expressions when using startswith/endswith/contains with unsafe expressions. Closes #58 --- mongoengine/fields.py | 3 +++ tests/queryset.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 670e3cd3..bd81d3a8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,6 +66,9 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' + + # escape unsafe characters which could lead to a re.error + value = re.escape(value) value = re.compile(regex % value, flags) return value diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..1efd034c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -288,6 +288,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() self.assertEqual(obj, None) + + # Test unsafe expressions + person = self.Person(name='Guido van Rossum [.\'Geek\']') + person.save() + + obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. From 6791f205af3d15a0f4ccaf357692ba868b6ebfff Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:50:09 +0200 Subject: [PATCH 29/49] Style fix. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bd81d3a8..30a11f20 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,7 +66,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - + # escape unsafe characters which could lead to a re.error value = re.escape(value) value = re.compile(regex % value, flags) From 21d267cb112e9030a0f17271fecd052dbe35b50a Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 17:28:59 +0200 Subject: [PATCH 30/49] Now order_by() works like queries for referencing deeper fields (replacing . with __). old: order_by('mydoc.myattr') / new: order_by('mydoc__myattr'). Closes #45 --- mongoengine/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ffbe5255..0b9218af 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -670,11 +670,13 @@ class QuerySet(object): """ key_list = [] for key in keys: + if not key: continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING if key[0] in ('-', '+'): key = key[1:] + key = key.replace('__', '.') key_list.append((key, direction)) self._ordering = key_list From 1147ac43506283a37554d09242894e9088f2133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 23 May 2010 00:19:50 +0800 Subject: [PATCH 31/49] ignore virtualenv directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42dcc6e6..57cf1b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ docs/.build docs/_build build/ dist/ -mongoengine.egg-info/ \ No newline at end of file +mongoengine.egg-info/ +env/ \ No newline at end of file From b96e27a7e4675c0564cff05bcba66f7662b6099d Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Fri, 30 Jul 2010 22:09:00 +1000 Subject: [PATCH 32/49] Allow documents to override the 'objects' QuerySetManager --- mongoengine/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b4..ed8214af 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - new_class.objects = QuerySetManager() + + # Provide a default queryset unless one has been manually provided + if not 'objects' in dir(new_class): + new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes From 17addbefe2e7befe45edcc764add45c397d4202b Mon Sep 17 00:00:00 2001 From: sp Date: Wed, 18 Aug 2010 16:50:52 -0400 Subject: [PATCH 33/49] made it more like Django's user model --- mongoengine/django/auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d4b0ff0b..da0005c8 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -32,6 +32,9 @@ class User(Document): last_login = DateTimeField(default=datetime.datetime.now) date_joined = DateTimeField(default=datetime.datetime.now) + def __unicode__(self): + return self.username + def get_full_name(self): """Returns the users first and last names, separated by a space. """ From 7de9adc6b1677b5e9c2b7b2b6d5b670effdd5ead Mon Sep 17 00:00:00 2001 From: Richard Henry Date: Sat, 28 Aug 2010 09:16:02 +0100 Subject: [PATCH 34/49] Adding support for pop operations to QuerySet.update and QuerySet.update_one --- docs/guide/querying.rst | 1 + mongoengine/queryset.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee431..7de6c5c8 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -399,6 +399,7 @@ that you may use with these methods: * ``unset`` -- delete a particular value (since MongoDB v1.3+) * ``inc`` -- increment a value by a given amount * ``dec`` -- decrement a value by a given amount +* ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list * ``pull`` -- remove a value from a list diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab113..182dfd52 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -661,8 +661,8 @@ class QuerySet(object): def _transform_update(cls, _doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ - operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', + 'pull', 'pull_all'] mongo_update = {} for key, value in update.items(): @@ -688,7 +688,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'pop', 'push', 'pull'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From d99c5973c3cb6da18670c6592ff396d18bb0e946 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 12:52:24 +0100 Subject: [PATCH 35/49] Fixed Q object DBRef test bug --- tests/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a40..b1f16576 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,8 @@ class QTest(unittest.TestCase): def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + class User(Document): pass From 3b62cf80cde0b83715662ce9bb01053dc13a7060 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:00:34 +0100 Subject: [PATCH 36/49] Fixed {Dict,List}Field default issue. Closes #46. --- mongoengine/fields.py | 6 +++--- tests/fields.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751b..d8d310dd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): @@ -623,4 +623,4 @@ class GeoPointField(BaseField): raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int.') \ No newline at end of file + raise ValidationError('Both values in point must be float or int.') diff --git a/tests/fields.py b/tests/fields.py index 136437b8..ef776d4a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 4fb6fcabefa04f9ab4b410ce1cf3b6293e0ec52c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:54:20 +0100 Subject: [PATCH 37/49] Added test for overriding objects --- tests/queryset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/queryset.py b/tests/queryset.py index b339d613..0c7ba05a 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -992,10 +992,15 @@ class QuerySetTest(unittest.TestCase): """ class BlogPost(Document): tags = ListField(StringField()) + deleted = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(deleted=False) @queryset_manager def music_posts(doc_cls, queryset): - return queryset(tags='music') + return queryset(tags='music', deleted=False) BlogPost.drop_collection() @@ -1005,6 +1010,8 @@ class QuerySetTest(unittest.TestCase): post2.save() post3 = BlogPost(tags=['film', 'actors']) post3.save() + post4 = BlogPost(tags=['film', 'actors'], deleted=True) + post4.save() self.assertEqual([p.id for p in BlogPost.objects], [post1.id, post2.id, post3.id]) From 95efa39b5229081deef179ae4b08299a50b18ccb Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:56:18 +0200 Subject: [PATCH 38/49] Added *.egg to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57cf1b7b..51a9ca1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*.egg docs/.build docs/_build build/ From e0911a5fe00d77bb74b0b8b2d8e5d6d963afc687 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:58:58 +0200 Subject: [PATCH 39/49] Replaced slow exception handling with has_key. --- mongoengine/base.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4523a268..e0e71d31 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -415,11 +415,8 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - try: - if not data['_id']: - del data['_id'] - except KeyError: - pass + if data.has_key('_id') and not data['_id']: + del data['_id'] return data @classmethod From 1ed9a36d0a15eef4038568ccc974d14bfb425c42 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 14:02:02 +0100 Subject: [PATCH 40/49] Added test for pop operation --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 0c7ba05a..2f86953d 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -671,6 +671,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + tags = post.tags[:-1] + BlogPost.objects.update(pop__tags=1) + post.reload() + self.assertEqual(post.tags, tags) + BlogPost.drop_collection() def test_update_pull(self): From 5b230b90b984b20d09322d17dd6b43b93db050f7 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 15:34:29 +0100 Subject: [PATCH 41/49] Doc fix (all operator) --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 86563082..bef19bc5 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -71,7 +71,7 @@ Available operators are as follows: * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values -* ``all`` -- every item in array is in list of values provided +* ``all`` -- every item in list of values provided is in array * ``size`` -- the size of the array is * ``exists`` -- value for field exists From 185e7a6a7e27884ac4896bf5c6a6881db8d419a0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 18:38:41 +0200 Subject: [PATCH 42/49] Better way of checking if new_class has an 'objects' attribute. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3172e9a3..9505d416 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -257,7 +257,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) # Provide a default queryset unless one has been manually provided - if not 'objects' in dir(new_class): + if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) From dcc8d22cec6d59392c8470a9325d0d8fedc6d4fa Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 16:22:56 -0700 Subject: [PATCH 43/49] Proper unique index name when using db_field. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9505d416..92a450de 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -269,7 +269,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True - unique_fields = [field_name] + unique_fields = [field.db_field] # Add any unique_with fields to the back of the index spec if field.unique_with: From 266f33adc4730859c272d9a8233769b0d9e54205 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 19:16:54 -0700 Subject: [PATCH 44/49] Bug fix for gridfs FileField. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index dc5dbe0e..49e9d056 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -600,7 +600,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - if self.gridfs.grid_id is not None: + if value is not None: return self.gridfs.get(id=value) return None From 3e30d71263356082ef5d391a2ea614d74790bb32 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:25:07 -0700 Subject: [PATCH 45/49] Support for background and drop_dups indexing options. --- mongoengine/queryset.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dd7fcef6..7969c196 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -172,7 +172,8 @@ class QuerySet(object): self._limit = None self._skip = None - def ensure_index(self, key_or_list): + def ensure_index(self, key_or_list, drop_dups=False, background=False, + **kwargs): """Ensure that the given indexes are in place. :param key_or_list: a single index key or a list of index keys (to @@ -180,7 +181,8 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_list = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index(index_list) + self._collection.ensure_index(index_list, drop_dups=drop_dups, + background=background) return self @classmethod @@ -250,25 +252,33 @@ class QuerySet(object): """ if not self._accessed_collection: self._accessed_collection = True + + background = self._document._meta.get('index_background', False) + drop_dups = self._document._meta.get('index_drop_dups', False) + index_opts = self._document._meta.get('index_options', {}) # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - self._collection.ensure_index(key_or_list) + self._collection.ensure_index(key_or_list, + background=background, **index_opts) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - self._collection.ensure_index(index, unique=True) + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: - self._collection.ensure_index('_types') + self._collection.ensure_index('_types', + background=background, **index_opts) # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec) + self._collection.ensure_index(index_spec, + background=background, **index_opts) return self._collection_obj From f1aec68f239d77edff296e05764c419c4187b2d0 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:46:44 -0700 Subject: [PATCH 46/49] Inherit index options. --- mongoengine/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92a450de..836817da 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -230,12 +230,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_field = None base_indexes = [] + base_meta = {} # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta') and 'collection' in base._meta: collection = base._meta['collection'] + # Propagate index options. + for key in ('index_background', 'index_drop_dups', 'index_opts'): + if key in base._meta: + base_meta[key] = base._meta[key] + id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) @@ -246,7 +252,11 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime 'id_field': id_field, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': {}, } + meta.update(base_meta) # Apply document-defined meta options meta.update(attrs.get('meta', {})) From 17642c8a8cc2e37741a1d10be04d704770679193 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:48:17 +0100 Subject: [PATCH 47/49] Fixed QuerySet.average issue that ignored 0 --- mongoengine/queryset.py | 2 +- tests/queryset.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7969c196..662fa8c3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -892,7 +892,7 @@ class QuerySet(object): var total = 0.0; var num = 0; db[collection].find(query).forEach(function(doc) { - if (doc[averageField]) { + if (doc[averageField] !== undefined) { total += doc[averageField]; num += 1; } diff --git a/tests/queryset.py b/tests/queryset.py index 2f86953d..0c6c3ca4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -960,11 +960,14 @@ class QuerySetTest(unittest.TestCase): def test_average(self): """Ensure that field can be averaged correctly. """ + self.Person(name='person', age=0).save() + self.assertEqual(int(self.Person.objects.average('age')), 0) + ages = [23, 54, 12, 94, 27] for i, age in enumerate(ages): self.Person(name='test%s' % i, age=age).save() - avg = float(sum(ages)) / len(ages) + avg = float(sum(ages)) / (len(ages) + 1) # take into account the 0 self.assertAlmostEqual(int(self.Person.objects.average('age')), avg) self.Person(name='ageless person').save() @@ -1340,5 +1343,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + if __name__ == '__main__': unittest.main() From 69012e8ad11a7e4312450ff71fc80445e007270f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:59:49 +0100 Subject: [PATCH 48/49] Fixed incorrect $pull test --- tests/queryset.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 0c6c3ca4..e3912246 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -681,23 +681,17 @@ class QuerySetTest(unittest.TestCase): def test_update_pull(self): """Ensure that the 'pull' update operation works correctly. """ - class Comment(EmbeddedDocument): - content = StringField() - class BlogPost(Document): slug = StringField() - comments = ListField(EmbeddedDocumentField(Comment)) + tags = ListField(StringField()) - comment1 = Comment(content="test1") - comment2 = Comment(content="test2") - - post = BlogPost(slug="test", comments=[comment1, comment2]) + post = BlogPost(slug="test", tags=['code', 'mongodb', 'code']) post.save() - self.assertTrue(comment2 in post.comments) - BlogPost.objects(slug="test").update(pull__comments__content="test2") + BlogPost.objects(slug="test").update(pull__tags="code") post.reload() - self.assertTrue(comment2 not in post.comments) + self.assertTrue('code' not in post.tags) + self.assertEqual(len(post.tags), 1) def test_order_by(self): """Ensure that QuerySets may be ordered. From 32e66b29f44f3015be099851201241caee92054f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 22:12:05 +0100 Subject: [PATCH 49/49] Fixed FileField problem caused by shared objects --- mongoengine/fields.py | 50 ++++++++++++++++++++++++++++++------------- tests/fields.py | 22 +++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 49e9d056..ffcfb53d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -510,14 +510,15 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ - def __init__(self): + def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file + self.grid_id = grid_id # Store GridFS id for file def __getattr__(self, name): obj = self.get() @@ -528,9 +529,12 @@ class GridFSProxy(object): return self def get(self, id=None): - if id: self.grid_id = id - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted + if id: + self.grid_id = id + try: + return self.fs.get(id or self.grid_id) + except: + return None # File has been deleted def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -552,8 +556,10 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - try: return self.get().read() - except: return None + try: + return self.get().read() + except: + return None def delete(self): # Delete file from GridFS, FileField still remains @@ -571,38 +577,52 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) + class FileField(BaseField): """A GridFS storage field. """ def __init__(self, **kwargs): - self.gridfs = GridFSProxy() super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self - return self.gridfs + # Check if a file already exists for this model + grid_file = instance._data.get(self.name) + if grid_file: + return grid_file + return GridFSProxy() def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation - self.gridfs.put(value) + grid_file = instance._data.get(self.name) + # If a file already exists, delete it + if grid_file: + try: + grid_file.delete() + except: + pass + # Create a new file with the new data + grid_file.put(value) + else: + # Create a new proxy object as we don't already have one + instance._data[self.name] = GridFSProxy() + instance._data[self.name].put(value) else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB - if self.gridfs.grid_id is not None: - return self.gridfs.grid_id + if isinstance(value, GridFSProxy) and value.grid_id is not None: + return value.grid_id return None def to_python(self, value): - # Use stored value (id) to lookup file in GridFS if value is not None: - return self.gridfs.get(id=value) - return None + return GridFSProxy(value) def validate(self, value): if value.grid_id is not None: diff --git a/tests/fields.py b/tests/fields.py index 1e53d23d..8c727196 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -680,6 +680,28 @@ class FieldTest(unittest.TestCase): file = FileField() d = DemoFile.objects.create() + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + file = FileField() + + # First instance + testfile = TestFile() + testfile.name = "Hello, World!" + testfile.file.put('Hello, World!') + testfile.save() + + # Second instance + testfiledupe = TestFile() + data = testfiledupe.file.read() # Should be None + + self.assertTrue(testfile.name != testfiledupe.name) + self.assertTrue(testfile.file.read() != data) + + TestFile.drop_collection() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """