From 5e94637adc2d0fcb89b5569cba5ffec13d511147 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 15:39:57 +0000 Subject: [PATCH] DecimalField now stores as float not string (#289) --- docs/apireference.rst | 60 ++++++++++++------------ docs/changelog.rst | 1 + docs/conf.py | 7 ++- docs/upgrade.rst | 29 ++++++++++++ mongoengine/fields.py | 54 ++++++++++++++++----- mongoengine/queryset/transform.py | 6 +-- tests/fields/fields.py | 45 ++++++++++++++++-- tests/migration/__init__.py | 1 + tests/migration/decimalfield_as_float.py | 50 ++++++++++++++++++++ tests/queryset/queryset.py | 4 +- 10 files changed, 204 insertions(+), 53 deletions(-) create mode 100644 tests/migration/decimalfield_as_float.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 0040f45e..3a156299 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,33 +54,33 @@ Querying Fields ====== -.. autoclass:: mongoengine.StringField -.. autoclass:: mongoengine.URLField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.LongField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.DateTimeField -.. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.DynamicField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.SortedListField -.. autoclass:: mongoengine.DictField -.. autoclass:: mongoengine.MapField -.. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.ImageField -.. autoclass:: mongoengine.GeoPointField -.. autoclass:: mongoengine.SequenceField -.. autoclass:: mongoengine.ObjectIdField -.. autoclass:: mongoengine.UUIDField -.. autoclass:: mongoengine.GridFSError -.. autoclass:: mongoengine.GridFSProxy -.. autoclass:: mongoengine.ImageGridFsProxy -.. autoclass:: mongoengine.ImproperlyConfigured +.. autoclass:: mongoengine.fields.StringField +.. autoclass:: mongoengine.fields.URLField +.. autoclass:: mongoengine.fields.EmailField +.. autoclass:: mongoengine.fields.IntField +.. autoclass:: mongoengine.fields.LongField +.. autoclass:: mongoengine.fields.FloatField +.. autoclass:: mongoengine.fields.DecimalField +.. autoclass:: mongoengine.fields.BooleanField +.. autoclass:: mongoengine.fields.DateTimeField +.. autoclass:: mongoengine.fields.ComplexDateTimeField +.. autoclass:: mongoengine.fields.EmbeddedDocumentField +.. autoclass:: mongoengine.fields.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.fields.DynamicField +.. autoclass:: mongoengine.fields.ListField +.. autoclass:: mongoengine.fields.SortedListField +.. autoclass:: mongoengine.fields.DictField +.. autoclass:: mongoengine.fields.MapField +.. autoclass:: mongoengine.fields.ReferenceField +.. autoclass:: mongoengine.fields.GenericReferenceField +.. autoclass:: mongoengine.fields.BinaryField +.. autoclass:: mongoengine.fields.FileField +.. autoclass:: mongoengine.fields.ImageField +.. autoclass:: mongoengine.fields.GeoPointField +.. autoclass:: mongoengine.fields.SequenceField +.. autoclass:: mongoengine.fields.ObjectIdField +.. autoclass:: mongoengine.fields.UUIDField +.. autoclass:: mongoengine.fields.GridFSError +.. autoclass:: mongoengine.fields.GridFSProxy +.. autoclass:: mongoengine.fields.ImageGridFsProxy +.. autoclass:: mongoengine.fields.ImproperlyConfigured diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ea4ad54..d0167c51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) diff --git a/docs/conf.py b/docs/conf.py index 3cfcef5d..8bcb9ec9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -173,8 +173,8 @@ latex_paper_size = 'a4' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MongoEngine.tex', u'MongoEngine Documentation', - u'Harry Marr', 'manual'), + ('index', 'MongoEngine.tex', 'MongoEngine Documentation', + 'Ross Lawley', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -193,3 +193,6 @@ latex_documents = [ # If false, no module index is generated. #latex_use_modindex = True + +autoclass_content = 'both' + diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bcdc1101..dddce912 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -145,6 +145,35 @@ eg:: a._mark_as_dirty('uuid') a.save() +DecimalField +------------ + +DecimalField now store floats - previous it was storing strings and that +made it impossible to do comparisons when querying correctly.:: + + # Old code + class Person(Document): + balance = DecimalField() + + # New code + class Person(Document): + balance = DecimalField(force_string=True) + +To migrate all the uuid's you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Person(Document): + balance = DecimalField() + + # Mark all ReferenceFields as dirty and save + for p in Person.objects: + p._mark_as_dirty('balance') + p.save() + +.. note:: DecimalField's have also been improved with the addition of precision + and rounding. See :class:`~mongoengine.DecimalField` for more information. + Cascading Saves --------------- To improve performance document saves will no longer automatically cascade. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bea827c4..2e149330 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -260,30 +260,57 @@ class FloatField(BaseField): class DecimalField(BaseField): """A fixed-point decimal number field. + .. versionchanged:: 0.8 .. versionadded:: 0.3 """ - def __init__(self, min_value=None, max_value=None, **kwargs): - self.min_value, self.max_value = min_value, max_value + def __init__(self, min_value=None, max_value=None, force_string=False, + precision=2, rounding=decimal.ROUND_HALF_UP, **kwargs): + """ + :param min_value: Validation rule for the minimum acceptable value. + :param max_value: Validation rule for the maximum acceptable value. + :param force_string: Store as a string. + :param precision: Number of decimal places to store. + :param rounding: The rounding rule from the python decimal libary: + + - decimial.ROUND_CEILING (towards Infinity) + - decimial.ROUND_DOWN (towards zero) + - decimial.ROUND_FLOOR (towards -Infinity) + - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero) + - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer) + - decimial.ROUND_HALF_UP (to nearest with ties going away from zero) + - decimial.ROUND_UP (away from zero) + - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero) + + Defaults to: ``decimal.ROUND_HALF_UP`` + + """ + self.min_value = min_value + self.max_value = max_value + self.force_string = force_string + self.precision = decimal.Decimal(".%s" % ("0" * precision)) + self.rounding = rounding + super(DecimalField, self).__init__(**kwargs) def to_python(self, value): - original_value = value - if not isinstance(value, basestring): - value = unicode(value) - try: - value = decimal.Decimal(value) - except ValueError: - return original_value - return value + if value is None: + return value + + return decimal.Decimal(value).quantize(self.precision, + rounding=self.rounding) def to_mongo(self, value): - return unicode(value) + if value is None: + return value + if self.force_string: + return unicode(value) + return float(self.to_python(value)) def validate(self, value): if not isinstance(value, decimal.Decimal): if not isinstance(value, basestring): - value = str(value) + value = unicode(value) try: value = decimal.Decimal(value) except Exception, exc: @@ -295,6 +322,9 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Decimal value is too large') + def prepare_query_value(self, op, value): + return self.to_mongo(value) + class BooleanField(BaseField): """A boolean field type. diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 71f12e37..3da26935 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -9,7 +9,7 @@ __all__ = ('query', 'update') COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not') + 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere', 'max_distance') @@ -74,7 +74,7 @@ def query(_doc_cls=None, _field_operation=False, **query): if op in singular_ops: if isinstance(field, basestring): if (op in STRING_OPERATORS and - isinstance(value, basestring)): + isinstance(value, basestring)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: @@ -144,7 +144,7 @@ def query(_doc_cls=None, _field_operation=False, **query): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): - value = [{k:val} for val in v] + value = [{k: val} for val in v] if '$and' in mongo_query.keys(): mongo_query['$and'].append(value) else: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7eae3f43..4fa6989c 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -272,10 +272,8 @@ class FieldTest(unittest.TestCase): Person.drop_collection() - person = Person() - person.height = Decimal('1.89') - person.save() - person.reload() + Person(height=Decimal('1.89')).save() + person = Person.objects.first() self.assertEqual(person.height, Decimal('1.89')) person.height = '2.0' @@ -289,6 +287,45 @@ class FieldTest(unittest.TestCase): Person.drop_collection() + def test_decimal_comparison(self): + + class Person(Document): + money = DecimalField() + + Person.drop_collection() + + Person(money=6).save() + Person(money=8).save() + Person(money=10).save() + + self.assertEqual(2, Person.objects(money__gt=Decimal("7")).count()) + self.assertEqual(2, Person.objects(money__gt=7).count()) + self.assertEqual(2, Person.objects(money__gt="7").count()) + + def test_decimal_storage(self): + class Person(Document): + btc = DecimalField(precision=4) + + Person.drop_collection() + Person(btc=10).save() + Person(btc=10.1).save() + Person(btc=10.11).save() + Person(btc="10.111").save() + Person(btc=Decimal("10.1111")).save() + Person(btc=Decimal("10.11111")).save() + + # How its stored + expected = [{'btc': 10.0}, {'btc': 10.1}, {'btc': 10.11}, + {'btc': 10.111}, {'btc': 10.1111}, {'btc': 10.1111}] + actual = list(Person.objects.exclude('id').as_pymongo()) + self.assertEqual(expected, actual) + + # How it comes out locally + expected = [Decimal('10.0000'), Decimal('10.1000'), Decimal('10.1100'), + Decimal('10.1110'), Decimal('10.1111'), Decimal('10.1111')] + actual = list(Person.objects().scalar('btc')) + self.assertEqual(expected, actual) + def test_boolean_validation(self): """Ensure that invalid values cannot be assigned to boolean fields. """ diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index bff50c36..6fc83e02 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,4 +1,5 @@ from convert_to_new_inheritance_model import * +from decimalfield_as_float import * from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * from uuidfield_to_binary import * diff --git a/tests/migration/decimalfield_as_float.py b/tests/migration/decimalfield_as_float.py new file mode 100644 index 00000000..3903c913 --- /dev/null +++ b/tests/migration/decimalfield_as_float.py @@ -0,0 +1,50 @@ + # -*- coding: utf-8 -*- +import unittest +import decimal +from decimal import Decimal + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, DecimalField, ListField + +__all__ = ('ConvertDecimalField', ) + + +class ConvertDecimalField(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_decimal_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + money = DecimalField(force_string=True) + monies = ListField(DecimalField(force_string=True)) + + Person.drop_collection() + Person(name="Wilson Jr", money=Decimal("2.50"), + monies=[Decimal("2.10"), Decimal("5.00")]).save() + + # 2. Start the migration by changing the schema + # Change DecimalField - add precision and rounding settings + class Person(Document): + name = StringField() + money = DecimalField(precision=2, rounding=decimal.ROUND_HALF_UP) + monies = ListField(DecimalField(precision=2, + rounding=decimal.ROUND_HALF_UP)) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('money') + p._mark_as_changed('monies') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertTrue(isinstance(wilson['money'], float)) + self.assertTrue(all([isinstance(m, float) for m in wilson['monies']])) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c7c4c7ce..5e403c4e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3262,9 +3262,9 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(results[0], dict)) self.assertTrue(isinstance(results[1], dict)) self.assertEqual(results[0]['name'], 'Bob Dole') - self.assertEqual(results[0]['price'], '1.11') + self.assertEqual(results[0]['price'], 1.11) self.assertEqual(results[1]['name'], 'Barack Obama') - self.assertEqual(results[1]['price'], '2.22') + self.assertEqual(results[1]['price'], 2.22) # Test coerce_types users = User.objects.only('name', 'price').as_pymongo(coerce_types=True)