From 7cc1a4eba04d33039e2422b39865ea72c30dee54 Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:34:22 +0100 Subject: [PATCH 1/7] Fix long fields stored as int32 in Python 3. issue #1253 --- AUTHORS | 1 + mongoengine/fields.py | 5 ++++- tests/fields/fields.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index ad070b30..758c52ea 100644 --- a/AUTHORS +++ b/AUTHORS @@ -235,3 +235,4 @@ that much better: * Steven Rossiter (https://github.com/BeardedSteve) * Luo Peng (https://github.com/RussellLuo) * Bryan Bennett (https://github.com/bbenne10) + * Gilb's Gilb's (https://github.com/gilbsgilbs) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 6f69465c..35d5f35a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -19,7 +19,7 @@ else: import pymongo import gridfs -from bson import Binary, DBRef, SON, ObjectId +from bson import Binary, DBRef, SON, ObjectId, Int64 from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, @@ -227,6 +227,9 @@ class LongField(BaseField): pass return value + def to_mongo(self, value, **kwargs): + return Int64(value) + def validate(self, value): try: value = long(value) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 1c1f49ad..2695e849 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -21,6 +21,7 @@ except ImportError: from decimal import Decimal from bson import Binary, DBRef, ObjectId +from bson.int64 import Int64 from mongoengine import * from mongoengine.connection import get_db @@ -4114,6 +4115,19 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): self.assertTrue(hasattr(CustomData.c_field, 'custom_data')) self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a']) + def test_long_field_is_stored_as_long(self): + """ + Tests that long fields are stored as long in mongo, even if long value + is small enough to be an int. + """ + class TestDocument(Document): + some_long = LongField() + + doc = TestDocument(some_long=42).save() + db = get_db() + self.assertTrue(isinstance(get_db().test_document.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(doc.some_long, (int, long,))) + if __name__ == '__main__': unittest.main() From cef4e313e11d594735520799a6e10bffbcafe4fe Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:36:31 +0100 Subject: [PATCH 2/7] Update changelog for #1253 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 69555b95..c34094a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Changes in 0.10.7 - DEV - Add `signal_kwargs` argument to `Document.save`, `Document.delete` and `BaseQuerySet.insert` to be passed to signals calls #1206 - Raise `OperationError` when trying to do a `drop_collection` on document with no collection set. - count on ListField of EmbeddedDocumentField fails. #1187 +- Fixed long fields stored as int32 in Python 3. #1253 Changes in 0.10.6 ================= From 87a2358a65fbf701506fb661005fa55d492aa5a2 Mon Sep 17 00:00:00 2001 From: Gilbert Gilb's Date: Tue, 15 Mar 2016 14:42:23 +0100 Subject: [PATCH 3/7] Fix unused variable. issue #1253 --- tests/fields/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 2695e849..bd0af17a 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -4125,7 +4125,7 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): doc = TestDocument(some_long=42).save() db = get_db() - self.assertTrue(isinstance(get_db().test_document.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(db.test_document.find()[0]['some_long'], Int64)) self.assertTrue(isinstance(doc.some_long, (int, long,))) From d651d0d472205c629f0b3d3de792b8609c2a6917 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Tue, 15 Mar 2016 20:50:34 +0100 Subject: [PATCH 4/7] Fix tests and imports. issue #1253 --- mongoengine/fields.py | 7 ++++++- tests/fields/fields.py | 31 +++++++++++++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 35d5f35a..206edcff 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -19,7 +19,12 @@ else: import pymongo import gridfs -from bson import Binary, DBRef, SON, ObjectId, Int64 +from bson import Binary, DBRef, SON, ObjectId + +try: + from bson.int64 import Int64 +except ImportError: + Int64 = long from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, diff --git a/tests/fields/fields.py b/tests/fields/fields.py index bd0af17a..c09cbf29 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -21,7 +21,10 @@ except ImportError: from decimal import Decimal from bson import Binary, DBRef, ObjectId -from bson.int64 import Int64 +try: + from bson.int64 import Int64 +except ImportError: + Int64 = long from mongoengine import * from mongoengine.connection import get_db @@ -3606,6 +3609,19 @@ class FieldTest(unittest.TestCase): self.assertRaises(FieldDoesNotExist, test) + def test_long_field_is_considered_as_int64(self): + """ + Tests that long fields are stored as long in mongo, even if long value + is small enough to be an int. + """ + class TestLongFieldConsideredAsInt64(Document): + some_long = LongField() + + doc = TestLongFieldConsideredAsInt64(some_long=42).save() + db = get_db() + self.assertTrue(isinstance(db.test_long_field_considered_as_int64.find()[0]['some_long'], Int64)) + self.assertTrue(isinstance(doc.some_long, (int, long,))) + class EmbeddedDocumentListFieldTestCase(unittest.TestCase): @@ -4115,19 +4131,6 @@ class EmbeddedDocumentListFieldTestCase(unittest.TestCase): self.assertTrue(hasattr(CustomData.c_field, 'custom_data')) self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a']) - def test_long_field_is_stored_as_long(self): - """ - Tests that long fields are stored as long in mongo, even if long value - is small enough to be an int. - """ - class TestDocument(Document): - some_long = LongField() - - doc = TestDocument(some_long=42).save() - db = get_db() - self.assertTrue(isinstance(db.test_document.find()[0]['some_long'], Int64)) - self.assertTrue(isinstance(doc.some_long, (int, long,))) - if __name__ == '__main__': unittest.main() From 39eec59c9025887d118a0027e7f338a872f94e64 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Tue, 15 Mar 2016 22:28:31 +0100 Subject: [PATCH 5/7] Fix test failing randomly because of concurrency. --- tests/document/indexes.py | 53 +++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 7ee017a6..1b4c4d96 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -11,6 +11,7 @@ from datetime import datetime from mongoengine import * from mongoengine.connection import get_db, get_connection +from mongoengine.context_managers import switch_db __all__ = ("IndexesTest", ) @@ -822,33 +823,29 @@ class IndexesTest(unittest.TestCase): name = StringField(required=True) term = StringField(required=True) - class Report(Document): + class ReportEmbedded(Document): key = EmbeddedDocumentField(CompoundKey, primary_key=True) text = StringField() - Report.drop_collection() - my_key = CompoundKey(name="n", term="ok") - report = Report(text="OK", key=my_key).save() + report = ReportEmbedded(text="OK", key=my_key).save() self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, report.to_mongo()) - self.assertEqual(report, Report.objects.get(pk=my_key)) + self.assertEqual(report, ReportEmbedded.objects.get(pk=my_key)) def test_compound_key_dictfield(self): - class Report(Document): + class ReportDictField(Document): key = DictField(primary_key=True) text = StringField() - Report.drop_collection() - my_key = {"name": "n", "term": "ok"} - report = Report(text="OK", key=my_key).save() + report = ReportDictField(text="OK", key=my_key).save() self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, report.to_mongo()) - self.assertEqual(report, Report.objects.get(pk=my_key)) + self.assertEqual(report, ReportDictField.objects.get(pk=my_key)) def test_string_indexes(self): @@ -909,26 +906,38 @@ class IndexesTest(unittest.TestCase): Issue #812 """ + # Use a new connection and database since dropping the database could + # cause concurrent tests to fail. + connection = connect(db='tempdatabase', + alias='test_indexes_after_database_drop') + class BlogPost(Document): title = StringField() slug = StringField(unique=True) - BlogPost.drop_collection() + meta = {'db_alias': 'test_indexes_after_database_drop'} - # Create Post #1 - post1 = BlogPost(title='test1', slug='test') - post1.save() + try: + BlogPost.drop_collection() - # Drop the Database - self.connection.drop_database(BlogPost._get_db().name) + # Create Post #1 + post1 = BlogPost(title='test1', slug='test') + post1.save() - # Re-create Post #1 - post1 = BlogPost(title='test1', slug='test') - post1.save() + # Drop the Database + connection.drop_database('tempdatabase') + + # Re-create Post #1 + post1 = BlogPost(title='test1', slug='test') + post1.save() + + # Create Post #2 + post2 = BlogPost(title='test2', slug='test') + self.assertRaises(NotUniqueError, post2.save) + finally: + # Drop the temporary database at the end + connection.drop_database('tempdatabase') - # Create Post #2 - post2 = BlogPost(title='test2', slug='test') - self.assertRaises(NotUniqueError, post2.save) def test_index_dont_send_cls_option(self): """ From d9b3a9fb60ecff7b250a5db3cfecdc406bfc15d5 Mon Sep 17 00:00:00 2001 From: Gilb's Date: Fri, 18 Mar 2016 19:51:09 +0100 Subject: [PATCH 6/7] Use six integer types instead of explicit types, since six is now a dependency of the project. --- mongoengine/fields.py | 1 - tests/fields/fields.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 206edcff..c807809a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,7 +20,6 @@ else: import pymongo import gridfs from bson import Binary, DBRef, SON, ObjectId - try: from bson.int64 import Int64 except ImportError: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index c09cbf29..9af87f7f 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -12,6 +12,7 @@ import uuid import math import itertools import re +import six try: import dateutil @@ -3620,7 +3621,7 @@ class FieldTest(unittest.TestCase): doc = TestLongFieldConsideredAsInt64(some_long=42).save() db = get_db() self.assertTrue(isinstance(db.test_long_field_considered_as_int64.find()[0]['some_long'], Int64)) - self.assertTrue(isinstance(doc.some_long, (int, long,))) + self.assertTrue(isinstance(doc.some_long, six.integer_types)) class EmbeddedDocumentListFieldTestCase(unittest.TestCase): From e34100bab4678b41b38568c933a318b50c7a280d Mon Sep 17 00:00:00 2001 From: Gilb's Date: Fri, 18 Mar 2016 23:43:23 +0100 Subject: [PATCH 7/7] Another attempt to fix random fails of test `test_compound_key_dictfield`. --- tests/document/indexes.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 1b4c4d96..e13d8b84 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -5,21 +5,23 @@ import sys sys.path[0:0] = [""] import pymongo +from random import randint from nose.plugins.skip import SkipTest from datetime import datetime from mongoengine import * from mongoengine.connection import get_db, get_connection -from mongoengine.context_managers import switch_db __all__ = ("IndexesTest", ) class IndexesTest(unittest.TestCase): + _MAX_RAND = 10 ** 10 def setUp(self): - self.connection = connect(db='mongoenginetest') + self.db_name = 'mongoenginetest_IndexesTest_' + str(randint(0, self._MAX_RAND)) + self.connection = connect(db=self.db_name) self.db = get_db() class Person(Document): @@ -33,10 +35,7 @@ class IndexesTest(unittest.TestCase): self.Person = Person def tearDown(self): - for collection in self.db.collection_names(): - if 'system.' in collection: - continue - self.db.drop_collection(collection) + self.connection.drop_database(self.db) def test_indexes_document(self): """Ensure that indexes are used when meta[indexes] is specified for