diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1cd6be11..aa5aa805 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -37,6 +37,7 @@ from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError from mongoengine.python_support import StringIO from mongoengine.queryset import DO_NOTHING from mongoengine.queryset.base import BaseQuerySet +from mongoengine.queryset.transform import STRING_OPERATORS try: from PIL import Image, ImageOps @@ -106,11 +107,11 @@ class StringField(BaseField): if not isinstance(op, six.string_types): return value - if op.lstrip('i') in ('startswith', 'endswith', 'contains', 'exact'): - flags = 0 - if op.startswith('i'): - flags = re.IGNORECASE - op = op.lstrip('i') + if op in STRING_OPERATORS: + case_insensitive = op.startswith('i') + op = op.lstrip('i') + + flags = re.IGNORECASE if case_insensitive else 0 regex = r'%s' if op == 'startswith': @@ -497,15 +498,18 @@ class DateTimeField(BaseField): if not isinstance(value, six.string_types): return None + return self._parse_datetime(value) + + def _parse_datetime(self, value): + # Attempt to parse a datetime from a string value = value.strip() if not value: return None - # Attempt to parse a datetime: if dateutil: try: return dateutil.parser.parse(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): return None # split usecs, because they are not recognized by strptime. diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 8234a616..b20ebc1e 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -6,6 +6,7 @@ from mongoengine.connection import get_connection # Constant that can be used to compare the version retrieved with # get_mongodb_version() +MONGODB_34 = (3, 4) MONGODB_36 = (3, 6) diff --git a/tests/document/instance.py b/tests/document/instance.py index f4527ad8..0f2f0c0f 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -12,6 +12,7 @@ from bson import DBRef, ObjectId from pymongo.errors import DuplicateKeyError from six import iteritems +from mongoengine.mongodb_support import get_mongodb_version, MONGODB_36, MONGODB_34 from mongoengine.pymongo_support import list_collection_names from tests import fixtures from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest, @@ -466,7 +467,16 @@ class InstanceTest(MongoDBTestCase): Animal.drop_collection() doc = Animal(superphylum='Deuterostomia') doc.save() - doc.reload() + + mongo_db = get_mongodb_version() + CMD_QUERY_KEY = 'command' if mongo_db >= MONGODB_36 else 'query' + + with query_counter() as q: + doc.reload() + query_op = q.db.system.profile.find({'ns': 'mongoenginetest.animal'})[0] + self.assertEqual(set(query_op[CMD_QUERY_KEY]['filter'].keys()), set(['_id', 'superphylum'])) + + Animal.drop_collection() def test_reload_sharded_nested(self): class SuperPhylum(EmbeddedDocument): @@ -480,6 +490,34 @@ class InstanceTest(MongoDBTestCase): doc = Animal(superphylum=SuperPhylum(name='Deuterostomia')) doc.save() doc.reload() + Animal.drop_collection() + + def test_update_shard_key_routing(self): + """Ensures updating a doc with a specified shard_key includes it in + the query. + """ + class Animal(Document): + is_mammal = BooleanField() + name = StringField() + meta = {'shard_key': ('is_mammal', 'id')} + + Animal.drop_collection() + doc = Animal(is_mammal=True, name='Dog') + doc.save() + + mongo_db = get_mongodb_version() + + with query_counter() as q: + doc.name = 'Cat' + doc.save() + query_op = q.db.system.profile.find({'ns': 'mongoenginetest.animal'})[0] + self.assertEqual(query_op['op'], 'update') + if mongo_db == MONGODB_34: + self.assertEqual(set(query_op['query'].keys()), set(['_id', 'is_mammal'])) + else: + self.assertEqual(set(query_op['command']['q'].keys()), set(['_id', 'is_mammal'])) + + Animal.drop_collection() def test_reload_with_changed_fields(self): """Ensures reloading will not affect changed fields""" diff --git a/tests/fields/test_datetime_field.py b/tests/fields/test_datetime_field.py index 5af6a011..92f0668a 100644 --- a/tests/fields/test_datetime_field.py +++ b/tests/fields/test_datetime_field.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import datetime +import datetime as dt import six try: @@ -41,13 +41,13 @@ class TestDateTimeField(MongoDBTestCase): a document. """ class Person(Document): - created = DateTimeField(default=datetime.datetime.utcnow) + created = DateTimeField(default=dt.datetime.utcnow) - utcnow = datetime.datetime.utcnow() + utcnow = dt.datetime.utcnow() person = Person() person.validate() person_created_t0 = person.created - self.assertLess(person.created - utcnow, datetime.timedelta(seconds=1)) + self.assertLess(person.created - utcnow, dt.timedelta(seconds=1)) self.assertEqual(person_created_t0, person.created) # make sure it does not change self.assertEqual(person._data['created'], person.created) @@ -65,15 +65,15 @@ class TestDateTimeField(MongoDBTestCase): # Test can save dates log = LogEntry() - log.date = datetime.date.today() + log.date = dt.date.today() log.save() log.reload() - self.assertEqual(log.date.date(), datetime.date.today()) + self.assertEqual(log.date.date(), dt.date.today()) # Post UTC - microseconds are rounded (down) nearest millisecond and # dropped - d1 = datetime.datetime(1970, 1, 1, 0, 0, 1, 999) - d2 = datetime.datetime(1970, 1, 1, 0, 0, 1) + d1 = dt.datetime(1970, 1, 1, 0, 0, 1, 999) + d2 = dt.datetime(1970, 1, 1, 0, 0, 1) log = LogEntry() log.date = d1 log.save() @@ -82,8 +82,8 @@ class TestDateTimeField(MongoDBTestCase): self.assertEqual(log.date, d2) # Post UTC - microseconds are rounded (down) nearest millisecond - d1 = datetime.datetime(1970, 1, 1, 0, 0, 1, 9999) - d2 = datetime.datetime(1970, 1, 1, 0, 0, 1, 9000) + d1 = dt.datetime(1970, 1, 1, 0, 0, 1, 9999) + d2 = dt.datetime(1970, 1, 1, 0, 0, 1, 9000) log.date = d1 log.save() log.reload() @@ -93,8 +93,8 @@ class TestDateTimeField(MongoDBTestCase): if not six.PY3: # Pre UTC dates microseconds below 1000 are dropped # This does not seem to be true in PY3 - d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) - d2 = datetime.datetime(1969, 12, 31, 23, 59, 59) + d1 = dt.datetime(1969, 12, 31, 23, 59, 59, 999) + d2 = dt.datetime(1969, 12, 31, 23, 59, 59) log.date = d1 log.save() log.reload() @@ -108,7 +108,7 @@ class TestDateTimeField(MongoDBTestCase): LogEntry.drop_collection() - d1 = datetime.datetime(1970, 1, 1, 0, 0, 1) + d1 = dt.datetime(1970, 1, 1, 0, 0, 1) log = LogEntry() log.date = d1 log.validate() @@ -124,7 +124,7 @@ class TestDateTimeField(MongoDBTestCase): # create additional 19 log entries for a total of 20 for i in range(1971, 1990): - d = datetime.datetime(i, 1, 1, 0, 0, 1) + d = dt.datetime(i, 1, 1, 0, 0, 1) LogEntry(date=d).save() self.assertEqual(LogEntry.objects.count(), 20) @@ -143,15 +143,15 @@ class TestDateTimeField(MongoDBTestCase): i += 1 # Test searching - logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980, 1, 1)) + logs = LogEntry.objects.filter(date__gte=dt.datetime(1980, 1, 1)) self.assertEqual(logs.count(), 10) - logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980, 1, 1)) + logs = LogEntry.objects.filter(date__lte=dt.datetime(1980, 1, 1)) self.assertEqual(logs.count(), 10) logs = LogEntry.objects.filter( - date__lte=datetime.datetime(1980, 1, 1), - date__gte=datetime.datetime(1975, 1, 1), + date__lte=dt.datetime(1980, 1, 1), + date__gte=dt.datetime(1975, 1, 1), ) self.assertEqual(logs.count(), 5) @@ -163,20 +163,20 @@ class TestDateTimeField(MongoDBTestCase): time = DateTimeField() log = LogEntry() - log.time = datetime.datetime.now() + log.time = dt.datetime.now() log.validate() - log.time = datetime.date.today() + log.time = dt.date.today() log.validate() - log.time = datetime.datetime.now().isoformat(' ') + log.time = dt.datetime.now().isoformat(' ') log.validate() log.time = '2019-05-16 21:42:57.897847' log.validate() if dateutil: - log.time = datetime.datetime.now().isoformat('T') + log.time = dt.datetime.now().isoformat('T') log.validate() log.time = -1 @@ -190,6 +190,25 @@ class TestDateTimeField(MongoDBTestCase): log.time = '2019-05-16 21:42:57.123.456' self.assertRaises(ValidationError, log.validate) + def test_parse_datetime_as_str(self): + class DTDoc(Document): + date = DateTimeField() + + date_str = '2019-03-02 22:26:01' + + # make sure that passing a parsable datetime works + dtd = DTDoc() + dtd.date = date_str + self.assertIsInstance(dtd.date, six.string_types) + dtd.save() + dtd.reload() + + self.assertIsInstance(dtd.date, dt.datetime) + self.assertEqual(str(dtd.date), date_str) + + dtd.date = 'January 1st, 9999999999' + self.assertRaises(ValidationError, dtd.validate) + class TestDateTimeTzAware(MongoDBTestCase): def test_datetime_tz_aware_mark_as_changed(self): @@ -205,8 +224,8 @@ class TestDateTimeTzAware(MongoDBTestCase): LogEntry.drop_collection() - LogEntry(time=datetime.datetime(2013, 1, 1, 0, 0, 0)).save() + LogEntry(time=dt.datetime(2013, 1, 1, 0, 0, 0)).save() log = LogEntry.objects.first() - log.time = datetime.datetime(2013, 1, 1, 0, 0, 0) + log.time = dt.datetime(2013, 1, 1, 0, 0, 0) self.assertEqual(['time'], log._changed_fields)