Merge remote-tracking branch 'MongoEngine/master'

This commit is contained in:
Breeze.kay
2015-06-19 11:14:51 +08:00
19 changed files with 765 additions and 184 deletions

View File

@@ -184,7 +184,7 @@ class BaseDocument(object):
self__initialised = False
# Check if the user has created a new instance of a class
if (self._is_document and self__initialised
and self__created and name == self._meta['id_field']):
and self__created and name == self._meta.get('id_field')):
super(BaseDocument, self).__setattr__('_created', False)
super(BaseDocument, self).__setattr__(name, value)
@@ -672,7 +672,7 @@ class BaseDocument(object):
@classmethod
def _get_collection_name(cls):
"""Returns the collection name for this class.
"""Returns the collection name for this class. None for abstract class
"""
return cls._meta.get('collection', None)
@@ -782,7 +782,7 @@ class BaseDocument(object):
allow_inheritance = cls._meta.get('allow_inheritance',
ALLOW_INHERITANCE)
include_cls = (allow_inheritance and not spec.get('sparse', False) and
spec.get('cls', True))
spec.get('cls', True) and '_cls' not in spec['fields'])
# 733: don't include cls if index_cls is False unless there is an explicit cls with the index
include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))
@@ -795,16 +795,25 @@ class BaseDocument(object):
# ASCENDING from +
# DESCENDING from -
# GEO2D from *
# TEXT from $
# HASHED from #
# GEOSPHERE from (
# GEOHAYSTACK from )
# GEO2D from *
direction = pymongo.ASCENDING
if key.startswith("-"):
direction = pymongo.DESCENDING
elif key.startswith("*"):
direction = pymongo.GEO2D
elif key.startswith("$"):
direction = pymongo.TEXT
if key.startswith(("+", "-", "*", "$")):
elif key.startswith("#"):
direction = pymongo.HASHED
elif key.startswith("("):
direction = pymongo.GEOSPHERE
elif key.startswith(")"):
direction = pymongo.GEOHAYSTACK
elif key.startswith("*"):
direction = pymongo.GEO2D
if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
key = key[1:]
# Use real field name, do it manually because we need field
@@ -827,7 +836,8 @@ class BaseDocument(object):
index_list.append((key, direction))
# Don't add cls to a geo index
if include_cls and direction is not pymongo.GEO2D:
if include_cls and direction not in (
pymongo.GEO2D, pymongo.GEOHAYSTACK, pymongo.GEOSPHERE):
index_list.insert(0, ('_cls', 1))
if index_list:
@@ -973,8 +983,13 @@ class BaseDocument(object):
if hasattr(getattr(field, 'field', None), 'lookup_member'):
new_field = field.field.lookup_member(field_name)
else:
# Look up subfield on the previous field
new_field = field.lookup_member(field_name)
# Look up subfield on the previous field or raise
try:
new_field = field.lookup_member(field_name)
except AttributeError:
raise LookUpError('Cannot resolve subfield or operator {} '
'on the field {}'.format(
field_name, field.name))
if not new_field and isinstance(field, ComplexBaseField):
if hasattr(field.field, 'document_type') and cls._dynamic \
and field.field.document_type._dynamic:

View File

@@ -290,6 +290,7 @@ class ComplexBaseField(BaseField):
return value
if self.field:
self.field._auto_dereference = self._auto_dereference
value_dict = dict([(key, self.field.to_python(item))
for key, item in value.items()])
else:
@@ -424,8 +425,11 @@ class ObjectIdField(BaseField):
"""
def to_python(self, value):
if not isinstance(value, ObjectId):
value = ObjectId(value)
try:
if not isinstance(value, ObjectId):
value = ObjectId(value)
except:
pass
return value
def to_mongo(self, value):

View File

@@ -385,15 +385,17 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
new_class._auto_id_field = getattr(parent_doc_cls,
'_auto_id_field', False)
if not new_class._meta.get('id_field'):
# After 0.10, find not existing names, instead of overwriting
id_name, id_db_name = cls.get_auto_id_names(new_class)
new_class._auto_id_field = True
new_class._meta['id_field'] = 'id'
new_class._fields['id'] = ObjectIdField(db_field='_id')
new_class._fields['id'].name = 'id'
new_class.id = new_class._fields['id']
# Prepend id field to _fields_ordered
if 'id' in new_class._fields and 'id' not in new_class._fields_ordered:
new_class._fields_ordered = ('id', ) + new_class._fields_ordered
new_class._meta['id_field'] = id_name
new_class._fields[id_name] = ObjectIdField(db_field=id_db_name)
new_class._fields[id_name].name = id_name
new_class.id = new_class._fields[id_name]
new_class._db_field_map[id_name] = id_db_name
new_class._reverse_db_field_map[id_db_name] = id_name
# Prepend id field to _fields_ordered
new_class._fields_ordered = (id_name, ) + new_class._fields_ordered
# Merge in exceptions with parent hierarchy
exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned)
@@ -408,6 +410,19 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
return new_class
def get_auto_id_names(self):
id_name, id_db_name = ('id', '_id')
if id_name not in self._fields and \
id_db_name not in (v.db_field for v in self._fields.values()):
return id_name, id_db_name
id_basename, id_db_basename, i = 'auto_id', '_auto_id', 0
while id_name in self._fields or \
id_db_name in (v.db_field for v in self._fields.values()):
id_name = '{0}_{1}'.format(id_basename, i)
id_db_name = '{0}_{1}'.format(id_db_basename, i)
i += 1
return id_name, id_db_name
class MetaDict(dict):

View File

@@ -128,21 +128,25 @@ class DeReference(object):
"""
object_map = {}
for collection, dbrefs in self.reference_map.iteritems():
refs = [dbref for dbref in dbrefs
if unicode(dbref).encode('utf-8') not in object_map]
if hasattr(collection, 'objects'): # We have a document class for the refs
col_name = collection._get_collection_name()
refs = [dbref for dbref in dbrefs
if (col_name, dbref) not in object_map]
references = collection.objects.in_bulk(refs)
for key, doc in references.iteritems():
object_map[key] = doc
object_map[(col_name, key)] = doc
else: # Generic reference: use the refs data to convert to document
if isinstance(doc_type, (ListField, DictField, MapField,)):
continue
refs = [dbref for dbref in dbrefs
if (collection, dbref) not in object_map]
if doc_type:
references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
for ref in references:
doc = doc_type._from_son(ref)
object_map[doc.id] = doc
object_map[(collection, doc.id)] = doc
else:
references = get_db()[collection].find({'_id': {'$in': refs}})
for ref in references:
@@ -154,7 +158,7 @@ class DeReference(object):
for x in collection.split('_')))._from_son(ref)
else:
doc = doc_type._from_son(ref)
object_map[doc.id] = doc
object_map[(collection, doc.id)] = doc
return object_map
def _attach_objects(self, items, depth=0, instance=None, name=None):
@@ -180,7 +184,8 @@ class DeReference(object):
if isinstance(items, (dict, SON)):
if '_ref' in items:
return self.object_map.get(items['_ref'].id, items)
return self.object_map.get(
(items['_ref'].collection, items['_ref'].id), items)
elif '_cls' in items:
doc = get_document(items['_cls'])._from_son(items)
_cls = doc._data.pop('_cls', None)
@@ -216,9 +221,11 @@ class DeReference(object):
for field_name, field in v._fields.iteritems():
v = data[k]._data.get(field_name, None)
if isinstance(v, (DBRef)):
data[k]._data[field_name] = self.object_map.get(v.id, v)
data[k]._data[field_name] = self.object_map.get(
(v.collection, v.id), v)
elif isinstance(v, (dict, SON)) and '_ref' in v:
data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
data[k]._data[field_name] = self.object_map.get(
(v['_ref'].collection , v['_ref'].id), v)
elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
item_name = "{0}.{1}.{2}".format(name, k, field_name)
data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)
@@ -226,7 +233,7 @@ class DeReference(object):
item_name = '%s.%s' % (name, k) if name else name
data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
elif hasattr(v, 'id'):
data[k] = self.object_map.get(v.id, v)
data[k] = self.object_map.get((v.collection, v.id), v)
if instance and name:
if is_list:

View File

@@ -1,4 +1,4 @@
import warnings
import pymongo
import re
@@ -17,6 +17,7 @@ from mongoengine.base import (
get_document
)
from mongoengine.errors import InvalidQueryError, InvalidDocumentError
from mongoengine.python_support import IS_PYMONGO_3
from mongoengine.queryset import (OperationError, NotUniqueError,
QuerySet, transform)
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
@@ -113,9 +114,11 @@ class Document(BaseDocument):
specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta`
dictionary. :attr:`max_documents` is the maximum number of documents that
is allowed to be stored in the collection, and :attr:`max_size` is the
maximum size of the collection in bytes. If :attr:`max_size` is not
maximum size of the collection in bytes. :attr:`max_size` is rounded up
to the next multiple of 256 by MongoDB internally and mongoengine before.
Use also a multiple of 256 to avoid confusions. If :attr:`max_size` is not
specified and :attr:`max_documents` is, :attr:`max_size` defaults to
10000000 bytes (10MB).
10485760 bytes (10MB).
Indexes may be created by specifying :attr:`indexes` in the :attr:`meta`
dictionary. The value should be a list of field names or tuples of field
@@ -136,7 +139,7 @@ class Document(BaseDocument):
By default, any extra attribute existing in stored data but not declared
in your model will raise a :class:`~mongoengine.FieldDoesNotExist` error.
This can be disabled by setting :attr:`strict` to ``False``
in the :attr:`meta` dictionnary.
in the :attr:`meta` dictionary.
"""
# The __metaclass__ attribute is removed by 2to3 when running with Python3
@@ -144,13 +147,15 @@ class Document(BaseDocument):
my_metaclass = TopLevelDocumentMetaclass
__metaclass__ = TopLevelDocumentMetaclass
__slots__ = ('__objects')
__slots__ = ('__objects',)
def pk():
"""Primary key alias
"""
def fget(self):
if 'id_field' not in self._meta:
return None
return getattr(self, self._meta['id_field'])
def fset(self, value):
@@ -171,10 +176,13 @@ class Document(BaseDocument):
db = cls._get_db()
collection_name = cls._get_collection_name()
# Create collection as a capped collection if specified
if cls._meta['max_size'] or cls._meta['max_documents']:
if cls._meta.get('max_size') or cls._meta.get('max_documents'):
# Get max document limit and max byte size from meta
max_size = cls._meta['max_size'] or 10000000 # 10MB default
max_documents = cls._meta['max_documents']
max_size = cls._meta.get('max_size') or 10 * 2 ** 20 # 10MB default
max_documents = cls._meta.get('max_documents')
# Round up to next 256 bytes as MongoDB would do it to avoid exception
if max_size % 256:
max_size = (max_size // 256 + 1) * 256
if collection_name in db.collection_names():
cls._collection = db[collection_name]
@@ -265,7 +273,8 @@ class Document(BaseDocument):
to cascading saves. Implies ``cascade=True``.
:param _refs: A list of processed references used in cascading saves
:param save_condition: only perform save if matching record in db
satisfies condition(s) (e.g., version number)
satisfies condition(s) (e.g. version number).
Raises :class:`OperationError` if the conditions are not satisfied
.. versionchanged:: 0.5
In existing documents it only saves changed fields using
@@ -283,6 +292,8 @@ class Document(BaseDocument):
.. versionchanged:: 0.8.5
Optional save_condition that only overwrites existing documents
if the condition is satisfied in the current db record.
.. versionchanged:: 0.10
:class:`OperationError` exception raised if save_condition fails.
"""
signals.pre_save.send(self.__class__, document=self)
@@ -347,6 +358,9 @@ class Document(BaseDocument):
upsert = save_condition is None
last_error = collection.update(select_dict, update_query,
upsert=upsert, **write_concern)
if not upsert and last_error['nModified'] == 0:
raise OperationError('Race condition preventing'
' document update detected')
created = is_new_object(last_error)
if cascade is None:
@@ -635,23 +649,51 @@ class Document(BaseDocument):
db = cls._get_db()
db.drop_collection(cls._get_collection_name())
@classmethod
def create_index(cls, keys, background=False, **kwargs):
"""Creates the given indexes if required.
:param keys: a single index key or a list of index keys (to
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
:param background: Allows index creation in the background
"""
index_spec = cls._build_index_spec(keys)
index_spec = index_spec.copy()
fields = index_spec.pop('fields')
drop_dups = kwargs.get('drop_dups', False)
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
elif not IS_PYMONGO_3:
index_spec['drop_dups'] = drop_dups
index_spec['background'] = background
index_spec.update(kwargs)
if IS_PYMONGO_3:
return cls._get_collection().create_index(fields, **index_spec)
else:
return cls._get_collection().ensure_index(fields, **index_spec)
@classmethod
def ensure_index(cls, key_or_list, drop_dups=False, background=False,
**kwargs):
"""Ensure that the given indexes are in place.
"""Ensure that the given indexes are in place. Deprecated in favour
of create_index.
:param key_or_list: a single index key or a list of index keys (to
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
:param background: Allows index creation in the background
:param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value
will be removed if PyMongo3+ is used
"""
index_spec = cls._build_index_spec(key_or_list)
index_spec = index_spec.copy()
fields = index_spec.pop('fields')
index_spec['drop_dups'] = drop_dups
index_spec['background'] = background
index_spec.update(kwargs)
return cls._get_collection().ensure_index(fields, **index_spec)
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
elif not IS_PYMONGO_3:
kwargs.update({'drop_dups': drop_dups})
return cls.create_index(key_or_list, background=background, **kwargs)
@classmethod
def ensure_indexes(cls):
@@ -666,6 +708,9 @@ class Document(BaseDocument):
drop_dups = cls._meta.get('index_drop_dups', False)
index_opts = cls._meta.get('index_opts') or {}
index_cls = cls._meta.get('index_cls', True)
if IS_PYMONGO_3 and drop_dups:
msg = "drop_dups is deprecated and is removed when using PyMongo 3+."
warnings.warn(msg, DeprecationWarning)
collection = cls._get_collection()
# 746: when connection is via mongos, the read preference is not necessarily an indication that
@@ -694,8 +739,11 @@ class Document(BaseDocument):
if 'cls' in opts:
del opts['cls']
collection.ensure_index(fields, background=background,
drop_dups=drop_dups, **opts)
if IS_PYMONGO_3:
collection.create_index(fields, background=background, **opts)
else:
collection.ensure_index(fields, background=background,
drop_dups=drop_dups, **opts)
# If _cls is being used (for polymorphism), it needs an index,
# only if another index doesn't begin with _cls
@@ -707,8 +755,12 @@ class Document(BaseDocument):
if 'cls' in index_opts:
del index_opts['cls']
collection.ensure_index('_cls', background=background,
**index_opts)
if IS_PYMONGO_3:
collection.create_index('_cls', background=background,
**index_opts)
else:
collection.ensure_index('_cls', background=background,
**index_opts)
@classmethod
def list_indexes(cls, go_up=True, go_down=True):

View File

@@ -119,22 +119,31 @@ class URLField(StringField):
"""
_URL_REGEX = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
# domain...
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
r'^(?:[a-z0-9\.\-]*)://' # scheme is validated separately
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|' # domain...
r'localhost|' # localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
_URL_SCHEMES = ['http', 'https', 'ftp', 'ftps']
def __init__(self, verify_exists=False, url_regex=None, **kwargs):
def __init__(self, verify_exists=False, url_regex=None, schemes=None, **kwargs):
self.verify_exists = verify_exists
self.url_regex = url_regex or self._URL_REGEX
self.schemes = schemes or self._URL_SCHEMES
super(URLField, self).__init__(**kwargs)
def validate(self, value):
# Check first if the scheme is valid
scheme = value.split('://')[0].lower()
if scheme not in self.schemes:
self.error('Invalid scheme {} in URL: {}'.format(scheme, value))
return
# Then check full URL
if not self.url_regex.match(value):
self.error('Invalid URL: %s' % value)
self.error('Invalid URL: {}'.format(value))
return
if self.verify_exists:
@@ -162,7 +171,7 @@ class EmailField(StringField):
# quoted-string
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
# domain (max length of an ICAAN TLD is 22 characters)
r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,22}$', re.IGNORECASE
r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
)
def validate(self, value):
@@ -546,7 +555,7 @@ class EmbeddedDocumentField(BaseField):
def to_python(self, value):
if not isinstance(value, self.document_type):
return self.document_type._from_son(value)
return self.document_type._from_son(value, _auto_dereference=self._auto_dereference)
return value
def to_mongo(self, value, use_db_field=True, fields=[]):
@@ -1004,6 +1013,7 @@ class CachedReferenceField(BaseField):
"""
A referencefield with cache fields to purpose pseudo-joins
.. versionadded:: 0.9
"""
@@ -1674,12 +1684,21 @@ class SequenceField(BaseField):
cluster of machines, it is easier to create an object ID than have
global, uniformly increasing sequence numbers.
:param collection_name: Name of the counter collection (default 'mongoengine.counters')
:param sequence_name: Name of the sequence in the collection (default 'ClassName.counter')
:param value_decorator: Any callable to use as a counter (default int)
Use any callable as `value_decorator` to transform calculated counter into
any value suitable for your needs, e.g. string or hexadecimal
representation of the default integer counter value.
.. note::
In case the counter is defined in the abstract document, it will be
common to all inherited documents and the default sequence name will
be the class name of the abstract document.
.. versionadded:: 0.5
.. versionchanged:: 0.8 added `value_decorator`
"""
@@ -1694,7 +1713,7 @@ class SequenceField(BaseField):
self.sequence_name = sequence_name
self.value_decorator = (callable(value_decorator) and
value_decorator or self.VALUE_DECORATOR)
return super(SequenceField, self).__init__(*args, **kwargs)
super(SequenceField, self).__init__(*args, **kwargs)
def generate(self):
"""
@@ -1740,7 +1759,7 @@ class SequenceField(BaseField):
if self.sequence_name:
return self.sequence_name
owner = self.owner_document
if issubclass(owner, Document):
if issubclass(owner, Document) and not owner._meta.get('abstract'):
return owner._get_collection_name()
else:
return ''.join('_%s' % c if c.isupper() else c

View File

@@ -258,54 +258,6 @@ class BaseQuerySet(object):
"""
return self._document(**kwargs).save()
def get_or_create(self, write_concern=None, auto_save=True,
*q_objs, **query):
"""Retrieve unique object or create, if it doesn't exist. Returns a
tuple of ``(object, created)``, where ``object`` is the retrieved or
created object and ``created`` is a boolean specifying whether a new
object was created. Raises
:class:`~mongoengine.queryset.MultipleObjectsReturned` or
`DocumentName.MultipleObjectsReturned` if multiple results are found.
A new document will be created if the document doesn't exists; a
dictionary of default values for the new document may be provided as a
keyword argument called :attr:`defaults`.
.. note:: This requires two separate operations and therefore a
race condition exists. Because there are no transactions in
mongoDB other approaches should be investigated, to ensure you
don't accidentally duplicate data when using this method. This is
now scheduled to be removed before 1.0
:param write_concern: optional extra keyword arguments used if we
have to create a new document.
Passes any write_concern onto :meth:`~mongoengine.Document.save`
:param auto_save: if the object is to be saved automatically if
not found.
.. deprecated:: 0.8
.. versionchanged:: 0.6 - added `auto_save`
.. versionadded:: 0.3
"""
msg = ("get_or_create is scheduled to be deprecated. The approach is "
"flawed without transactions. Upserts should be preferred.")
warnings.warn(msg, DeprecationWarning)
defaults = query.get('defaults', {})
if 'defaults' in query:
del query['defaults']
try:
doc = self.get(*q_objs, **query)
return doc, False
except self._document.DoesNotExist:
query.update(defaults)
doc = self._document(**query)
if auto_save:
doc.save(write_concern=write_concern)
return doc, True
def first(self):
"""Retrieve the first object matching the query.
"""
@@ -1296,6 +1248,27 @@ class BaseQuerySet(object):
else:
return 0
def aggregate_sum(self, field):
"""Sum over the values of the specified field.
:param field: the field to sum over; use dot-notation to refer to
embedded document fields
This method is more performant than the regular `sum`, because it uses
the aggregation framework instead of map-reduce.
"""
result = self._document._get_collection().aggregate([
{ '$match': self._query },
{ '$group': { '_id': 'sum', 'total': { '$sum': '$' + field } } }
])
if IS_PYMONGO_3:
result = list(result)
else:
result = result.get('result')
if result:
return result[0]['total']
return 0
def average(self, field):
"""Average over the values of the specified field.
@@ -1351,6 +1324,27 @@ class BaseQuerySet(object):
else:
return 0
def aggregate_average(self, field):
"""Average over the values of the specified field.
:param field: the field to average over; use dot-notation to refer to
embedded document fields
This method is more performant than the regular `average`, because it
uses the aggregation framework instead of map-reduce.
"""
result = self._document._get_collection().aggregate([
{ '$match': self._query },
{ '$group': { '_id': 'avg', 'total': { '$avg': '$' + field } } }
])
if IS_PYMONGO_3:
result = list(result)
else:
result = result.get('result')
if result:
return result[0]['total']
return 0
def item_frequencies(self, field, normalize=False, map_reduce=True):
"""Returns a dictionary of all items present in a field across
the whole queried set of documents, and their corresponding frequency.

View File

@@ -44,7 +44,7 @@ def query(_doc_cls=None, _field_operation=False, **query):
if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
op = parts.pop()
#if user escape field name by __
# if user escape field name by __
if len(parts) > 1 and parts[-1] == "":
parts.pop()