Compare commits

..

1 Commits

Author SHA1 Message Date
Stefan Wojcik
160379ea85 cleanup BaseQuerySet.__getitem__ 2017-03-05 20:43:38 -05:00
10 changed files with 48 additions and 442 deletions

View File

@@ -5,27 +5,11 @@ Changelog
Development
===========
- (Fill this out as you fix issues and develop your features).
Changes in 0.13.0
=================
- POTENTIAL BREAKING CHANGE: Added Unicode support to the `EmailField`, see
docs/upgrade.rst for details.
Changes in 0.12.0
=================
- Fixed using sets in field choices #1481
- POTENTIAL BREAKING CHANGE: Fixed limit/skip/hint/batch_size chaining #1476
- POTENTIAL BREAKING CHANGE: Changed a public `QuerySet.clone_into` method to a private `QuerySet._clone_into` #1476
- Fixed the way `Document.objects.create` works with duplicate IDs #1485
- Fixed connecting to a replica set with PyMongo 2.x #1436
- Fixed using sets in field choices #1481
- Fixed deleting items from a `ListField` #1318
- Fixed an obscure error message when filtering by `field__in=non_iterable`. #1237
- Fixed behavior of a `dec` update operator #1450
- Added a `rename` update operator #1454
- Added validation for the `db_field` parameter #1448
- Fixed the error message displayed when querying an `EmbeddedDocumentField` by an invalid value #1440
- Fixed the error message displayed when validating unicode URLs #1486
- Raise an error when trying to save an abstract document #1449
Changes in 0.11.0
=================

View File

@@ -6,20 +6,6 @@ Development
***********
(Fill this out whenever you introduce breaking changes to MongoEngine)
0.13.0
******
This release adds Unicode support to the `EmailField` and changes its
structure significantly. Previously, email addresses containing Unicode
characters didn't work at all. Starting with v0.13.0, domains with Unicode
characters are supported out of the box, meaning some emails that previously
didn't pass validation now do. Make sure the rest of your application can
accept such email addresses. Additionally, if you subclassed the `EmailField`
in your application and overrode `EmailField.EMAIL_REGEX`, you will have to
adjust your code to override `EmailField.USER_REGEX`, `EmailField.DOMAIN_REGEX`,
and potentially `EmailField.UTF8_USER_REGEX`.
0.12.0
******
This release includes various fixes for the `BaseQuerySet` methods and how they
are chained together. Since version 0.10.1 applying limit/skip/hint/batch_size
to an already-existing queryset wouldn't modify the underlying PyMongo cursor.

View File

@@ -23,7 +23,7 @@ __all__ = (list(document.__all__) + list(fields.__all__) +
list(signals.__all__) + list(errors.__all__))
VERSION = (0, 13, 0)
VERSION = (0, 11, 0)
def get_version():

View File

@@ -684,13 +684,8 @@ class BaseDocument(object):
# class if unavailable
class_name = son.get('_cls', cls._class_name)
# Convert SON to a data dict, making sure each key is a string and
# corresponds to the right db field.
data = {}
for key, value in son.iteritems():
key = str(key)
key = cls._db_field_map.get(key, key)
data[key] = value
# Convert SON to a dict, making sure each key is a string
data = {str(key): value for key, value in son.iteritems()}
# Return correct subclass for document type
if class_name != cls._class_name:

View File

@@ -1,4 +1,3 @@
from collections import OrderedDict
from bson import DBRef, SON
import six
@@ -202,10 +201,6 @@ class DeReference(object):
as_tuple = isinstance(items, tuple)
iterator = enumerate(items)
data = []
elif isinstance(items, OrderedDict):
is_list = False
iterator = items.iteritems()
data = OrderedDict()
else:
is_list = False
iterator = items.iteritems()

View File

@@ -2,11 +2,9 @@ import datetime
import decimal
import itertools
import re
import socket
import time
import uuid
import warnings
from collections import Mapping
from operator import itemgetter
from bson import Binary, DBRef, ObjectId, SON
@@ -155,105 +153,21 @@ class EmailField(StringField):
.. versionadded:: 0.4
"""
USER_REGEX = re.compile(
# `dot-atom` defined in RFC 5322 Section 3.2.3.
r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"
# `quoted-string` defined in RFC 5322 Section 3.2.4.
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)',
re.IGNORECASE
EMAIL_REGEX = re.compile(
# dot-atom
r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
# quoted-string
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
# domain (max length of an ICAAN TLD is 22 characters)
r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
)
UTF8_USER_REGEX = re.compile(
six.u(
# RFC 6531 Section 3.3 extends `atext` (used by dot-atom) to
# include `UTF8-non-ascii`.
r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+)*\Z"
# `quoted-string`
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)'
), re.IGNORECASE | re.UNICODE
)
DOMAIN_REGEX = re.compile(
r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z',
re.IGNORECASE
)
error_msg = u'Invalid email address: %s'
def __init__(self, domain_whitelist=None, allow_utf8_user=False,
allow_ip_domain=False, *args, **kwargs):
"""Initialize the EmailField.
Args:
domain_whitelist (list) - list of otherwise invalid domain
names which you'd like to support.
allow_utf8_user (bool) - if True, the user part of the email
address can contain UTF8 characters.
False by default.
allow_ip_domain (bool) - if True, the domain part of the email
can be a valid IPv4 or IPv6 address.
"""
self.domain_whitelist = domain_whitelist or []
self.allow_utf8_user = allow_utf8_user
self.allow_ip_domain = allow_ip_domain
super(EmailField, self).__init__(*args, **kwargs)
def validate_user_part(self, user_part):
"""Validate the user part of the email address. Return True if
valid and False otherwise.
"""
if self.allow_utf8_user:
return self.UTF8_USER_REGEX.match(user_part)
return self.USER_REGEX.match(user_part)
def validate_domain_part(self, domain_part):
"""Validate the domain part of the email address. Return True if
valid and False otherwise.
"""
# Skip domain validation if it's in the whitelist.
if domain_part in self.domain_whitelist:
return True
if self.DOMAIN_REGEX.match(domain_part):
return True
# Validate IPv4/IPv6, e.g. user@[192.168.0.1]
if (
self.allow_ip_domain and
domain_part[0] == '[' and
domain_part[-1] == ']'
):
for addr_family in (socket.AF_INET, socket.AF_INET6):
try:
socket.inet_pton(addr_family, domain_part[1:-1])
return True
except (socket.error, UnicodeEncodeError):
pass
return False
def validate(self, value):
if not EmailField.EMAIL_REGEX.match(value):
self.error('Invalid email address: %s' % value)
super(EmailField, self).validate(value)
if '@' not in value:
self.error(self.error_msg % value)
user_part, domain_part = value.rsplit('@', 1)
# Validate the user part.
if not self.validate_user_part(user_part):
self.error(self.error_msg % value)
# Validate the domain and, if invalid, see if it's IDN-encoded.
if not self.validate_domain_part(domain_part):
try:
domain_part = domain_part.encode('idna').decode('ascii')
except UnicodeError:
self.error(self.error_msg % value)
else:
if not self.validate_domain_part(domain_part):
self.error(self.error_msg % value)
class IntField(BaseField):
"""32-bit integer field."""
@@ -705,14 +619,6 @@ class DynamicField(BaseField):
Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data"""
def __init__(self, container_class=dict, *args, **kwargs):
self._container_cls = container_class
if not issubclass(self._container_cls, Mapping):
self.error('The class that is specified in `container_class` parameter '
'must be a subclass of `dict`.')
super(DynamicField, self).__init__(*args, **kwargs)
def to_mongo(self, value, use_db_field=True, fields=None):
"""Convert a Python type to a MongoDB compatible type.
"""
@@ -738,7 +644,7 @@ class DynamicField(BaseField):
is_list = True
value = {k: v for k, v in enumerate(value)}
data = self._container_cls()
data = {}
for k, v in value.iteritems():
data[k] = self.to_mongo(v, use_db_field, fields)
@@ -1092,8 +998,8 @@ class ReferenceField(BaseField):
def validate(self, value):
if not isinstance(value, (self.document_type, DBRef, ObjectId)):
self.error('A ReferenceField only accepts DBRef, ObjectId or documents')
if not isinstance(value, (self.document_type, DBRef)):
self.error('A ReferenceField only accepts DBRef or documents')
if isinstance(value, Document) and value.id is None:
self.error('You can only reference documents once they have been '

View File

@@ -844,7 +844,7 @@ class InstanceTest(unittest.TestCase):
class Recipient(Document):
email = EmailField(required=True)
recipient = Recipient(email='not-an-email')
recipient = Recipient(email='root@localhost')
self.assertRaises(ValidationError, recipient.save)
recipient.save(validate=False)

View File

@@ -5,11 +5,8 @@ import uuid
import math
import itertools
import re
import pymongo
import sys
from nose.plugins.skip import SkipTest
from collections import OrderedDict
import six
try:
@@ -28,12 +25,9 @@ except ImportError:
from mongoengine import *
from mongoengine.connection import get_db
from mongoengine.base import (BaseDict, BaseField, EmbeddedDocumentList,
_document_registry, TopLevelDocumentMetaclass)
_document_registry)
from tests.utils import MongoDBTestCase, MONGO_TEST_DB
from mongoengine.python_support import IS_PYMONGO_3
if IS_PYMONGO_3:
from bson import CodecOptions
from tests.utils import MongoDBTestCase
__all__ = ("FieldTest", "EmbeddedDocumentListFieldTestCase")
@@ -343,6 +337,8 @@ class FieldTest(MongoDBTestCase):
class Link(Document):
url = URLField()
Link.drop_collection()
link = Link()
link.url = 'google'
self.assertRaises(ValidationError, link.validate)
@@ -355,6 +351,8 @@ class FieldTest(MongoDBTestCase):
class Link(Document):
url = URLField()
Link.drop_collection()
link = Link()
link.url = u'http://привет.com'
@@ -1903,51 +1901,6 @@ class FieldTest(MongoDBTestCase):
doc = self.db.test.find_one()
self.assertEqual(doc['x']['i'], 2)
def test_double_embedded_db_field(self):
"""Make sure multiple layers of embedded docs resolve db fields
properly and can be initialized using dicts.
"""
class C(EmbeddedDocument):
txt = StringField()
class B(EmbeddedDocument):
c = EmbeddedDocumentField(C, db_field='fc')
class A(Document):
b = EmbeddedDocumentField(B, db_field='fb')
a = A(
b=B(
c=C(txt='hi')
)
)
a.validate()
a = A(b={'c': {'txt': 'hi'}})
a.validate()
def test_double_embedded_db_field_from_son(self):
"""Make sure multiple layers of embedded docs resolve db fields
from SON properly.
"""
class C(EmbeddedDocument):
txt = StringField()
class B(EmbeddedDocument):
c = EmbeddedDocumentField(C, db_field='fc')
class A(Document):
b = EmbeddedDocumentField(B, db_field='fb')
a = A._from_son(SON([
('fb', SON([
('fc', SON([
('txt', 'hi')
]))
]))
]))
self.assertEqual(a.b.c.txt, 'hi')
def test_embedded_document_validation(self):
"""Ensure that invalid embedded documents cannot be assigned to
embedded document fields.
@@ -2090,12 +2043,6 @@ class FieldTest(MongoDBTestCase):
post1.author = post2
self.assertRaises(ValidationError, post1.validate)
# Ensure ObjectID's are accepted as references
user_object_id = user.pk
post3 = BlogPost(content="Chips and curry sauce taste good.")
post3.author = user_object_id
post3.save()
# Make sure referencing a saved document of the right type works
user.save()
post1.author = user
@@ -2106,20 +2053,6 @@ class FieldTest(MongoDBTestCase):
post1.author = post2
self.assertRaises(ValidationError, post1.validate)
def test_objectid_reference_fields(self):
"""Make sure storing Object ID references works."""
class Person(Document):
name = StringField()
parent = ReferenceField('self')
Person.drop_collection()
p1 = Person(name="John").save()
Person(name="Ross", parent=p1.pk).save()
p = Person.objects.get(name="Ross")
self.assertEqual(p.parent, p1)
def test_dbref_reference_fields(self):
"""Make sure storing references as bson.dbref.DBRef works."""
class Person(Document):
@@ -3453,99 +3386,23 @@ class FieldTest(MongoDBTestCase):
class User(Document):
email = EmailField()
user = User(email='ross@example.com')
user.validate()
user = User(email="ross@example.com")
self.assertTrue(user.validate() is None)
user = User(email='ross@example.co.uk')
user.validate()
user = User(email="ross@example.co.uk")
self.assertTrue(user.validate() is None)
user = User(email=('Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S'
'aJIazqqWkm7.net'))
user.validate()
user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S"
"aJIazqqWkm7.net"))
self.assertTrue(user.validate() is None)
user = User(email='new-tld@example.technology')
user.validate()
user = User(email='ross@example.com.')
self.assertRaises(ValidationError, user.validate)
# unicode domain
user = User(email=u'user@пример.рф')
user.validate()
# invalid unicode domain
user = User(email=u'user@пример')
self.assertRaises(ValidationError, user.validate)
# invalid data type
user = User(email=123)
self.assertRaises(ValidationError, user.validate)
def test_email_field_unicode_user(self):
# Don't run this test on pypy3, which doesn't support unicode regex:
# https://bitbucket.org/pypy/pypy/issues/1821/regular-expression-doesnt-find-unicode
if sys.version_info[:2] == (3, 2):
raise SkipTest('unicode email addresses are not supported on PyPy 3')
class User(Document):
email = EmailField()
# unicode user shouldn't validate by default...
user = User(email=u'Dörte@Sörensen.example.com')
self.assertRaises(ValidationError, user.validate)
# ...but it should be fine with allow_utf8_user set to True
class User(Document):
email = EmailField(allow_utf8_user=True)
user = User(email=u'Dörte@Sörensen.example.com')
user.validate()
def test_email_field_domain_whitelist(self):
class User(Document):
email = EmailField()
# localhost domain shouldn't validate by default...
user = User(email='me@localhost')
self.assertRaises(ValidationError, user.validate)
# ...but it should be fine if it's whitelisted
class User(Document):
email = EmailField(domain_whitelist=['localhost'])
user = User(email="new-tld@example.technology")
self.assertTrue(user.validate() is None)
user = User(email='me@localhost')
user.validate()
def test_email_field_ip_domain(self):
class User(Document):
email = EmailField()
valid_ipv4 = 'email@[127.0.0.1]'
valid_ipv6 = 'email@[2001:dB8::1]'
invalid_ip = 'email@[324.0.0.1]'
# IP address as a domain shouldn't validate by default...
user = User(email=valid_ipv4)
self.assertRaises(ValidationError, user.validate)
user = User(email=valid_ipv6)
self.assertRaises(ValidationError, user.validate)
user = User(email=invalid_ip)
self.assertRaises(ValidationError, user.validate)
# ...but it should be fine with allow_ip_domain set to True
class User(Document):
email = EmailField(allow_ip_domain=True)
user = User(email=valid_ipv4)
user.validate()
user = User(email=valid_ipv6)
user.validate()
# invalid IP should still fail validation
user = User(email=invalid_ip)
user = User(email="ross@example.com.")
self.assertRaises(ValidationError, user.validate)
def test_email_field_honors_regex(self):
@@ -4188,67 +4045,6 @@ class EmbeddedDocumentListFieldTestCase(MongoDBTestCase):
self.assertTrue(hasattr(CustomData.c_field, 'custom_data'))
self.assertEqual(custom_data['a'], CustomData.c_field.custom_data['a'])
def test_dynamicfield_with_container_class(self):
"""
Tests that object can be stored in order by DynamicField class
with container_class parameter.
"""
raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
class Doc(Document):
ordered_data = DynamicField(container_class=OrderedDict)
unordered_data = DynamicField()
Doc.drop_collection()
doc = Doc(ordered_data=OrderedDict(raw_data), unordered_data=dict(raw_data)).save()
# checks that the data is in order
self.assertEqual(type(doc.ordered_data), OrderedDict)
self.assertEqual(type(doc.unordered_data), dict)
self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
# checks that the data is stored to the database in order
pymongo_db = pymongo.MongoClient()[MONGO_TEST_DB]
if IS_PYMONGO_3:
codec_option = CodecOptions(document_class=OrderedDict)
db_doc = pymongo_db.doc.with_options(codec_options=codec_option).find_one()
else:
db_doc = pymongo_db.doc.find_one(as_class=OrderedDict)
self.assertEqual(','.join(doc.ordered_data.keys()), 'd,c,b,a')
def test_dynamicfield_with_wrong_container_class(self):
with self.assertRaises(ValidationError):
class DocWithInvalidField:
data = DynamicField(container_class=list)
def test_dynamicfield_with_wrong_container_class_and_reload_docuemnt(self):
# This is because 'codec_options' is supported on pymongo3 or later
if IS_PYMONGO_3:
class OrderedDocument(Document):
my_metaclass = TopLevelDocumentMetaclass
__metaclass__ = TopLevelDocumentMetaclass
@classmethod
def _get_collection(cls):
collection = super(OrderedDocument, cls)._get_collection()
opts = CodecOptions(document_class=OrderedDict)
return collection.with_options(codec_options=opts)
raw_data = [('d', 1), ('c', 2), ('b', 3), ('a', 4)]
class Doc(OrderedDocument):
data = DynamicField(container_class=OrderedDict)
Doc.drop_collection()
doc = Doc(data=OrderedDict(raw_data)).save()
doc.reload()
self.assertEqual(type(doc.data), OrderedDict)
self.assertEqual(','.join(doc.data.keys()), 'd,c,b,a')
class CachedReferenceFieldTest(MongoDBTestCase):

View File

@@ -4962,6 +4962,20 @@ class QuerySetTest(unittest.TestCase):
for p in Person.objects():
self.assertEqual(p.name, 'a')
def test_last_field_name_like_operator(self):
class EmbeddedItem(EmbeddedDocument):
type = StringField()
class Doc(Document):
item = EmbeddedDocumentField(EmbeddedItem)
Doc.drop_collection()
doc = Doc(item=EmbeddedItem(type="axe"))
doc.save()
self.assertEqual(1, Doc.objects(item__type__="axe").count())
def test_len_during_iteration(self):
"""Tests that calling len on a queyset during iteration doesn't
stop paging.

View File

@@ -2,15 +2,10 @@
import unittest
from bson import DBRef, ObjectId
from collections import OrderedDict
from mongoengine import *
from mongoengine.connection import get_db
from mongoengine.context_managers import query_counter
from mongoengine.python_support import IS_PYMONGO_3
from mongoengine.base import TopLevelDocumentMetaclass
if IS_PYMONGO_3:
from bson import CodecOptions
class FieldTest(unittest.TestCase):
@@ -1292,70 +1287,5 @@ class FieldTest(unittest.TestCase):
self.assertEqual(q, 2)
def test_dynamic_field_dereference(self):
class Merchandise(Document):
name = StringField()
price = IntField()
class Store(Document):
merchandises = DynamicField()
Merchandise.drop_collection()
Store.drop_collection()
merchandises = {
'#1': Merchandise(name='foo', price=100).save(),
'#2': Merchandise(name='bar', price=120).save(),
'#3': Merchandise(name='baz', price=110).save(),
}
Store(merchandises=merchandises).save()
store = Store.objects().first()
for obj in store.merchandises.values():
self.assertFalse(isinstance(obj, Merchandise))
store.select_related()
for obj in store.merchandises.values():
self.assertTrue(isinstance(obj, Merchandise))
def test_dynamic_field_dereference_with_ordering_guarantee_on_pymongo3(self):
# This is because 'codec_options' is supported on pymongo3 or later
if IS_PYMONGO_3:
class OrderedDocument(Document):
my_metaclass = TopLevelDocumentMetaclass
__metaclass__ = TopLevelDocumentMetaclass
@classmethod
def _get_collection(cls):
collection = super(OrderedDocument, cls)._get_collection()
opts = CodecOptions(document_class=OrderedDict)
return collection.with_options(codec_options=opts)
class Merchandise(Document):
name = StringField()
price = IntField()
class Store(OrderedDocument):
merchandises = DynamicField(container_class=OrderedDict)
Merchandise.drop_collection()
Store.drop_collection()
merchandises = OrderedDict()
merchandises['#1'] = Merchandise(name='foo', price=100).save()
merchandises['#2'] = Merchandise(name='bar', price=120).save()
merchandises['#3'] = Merchandise(name='baz', price=110).save()
Store(merchandises=merchandises).save()
store = Store.objects().first()
store.select_related()
# confirms that the load data order is same with the one at storing
self.assertTrue(type(store.merchandises), OrderedDict)
self.assertEqual(','.join(store.merchandises.keys()), '#1,#2,#3')
if __name__ == '__main__':
unittest.main()