pub/mongoengine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge in upstream dev

Browse Source
This commit is contained in:
Colin Howe
2011-06-08 12:33:36 +01:00
parent 7c62fdc0b8 3c88faa889
commit 4174918476
19 changed files with 1060 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
mongoengine/__init__.py
View File

@@ -6,9 +6,11 @@ import connection
from connection import *
import queryset
from queryset import *
import signals
from signals import *
__all__ = (document.__all__ + fields.__all__ + connection.__all__ +
queryset.__all__)
queryset.__all__ + signals.__all__)
__author__ = 'Harry Marr'

89
mongoengine/base.py
View File

@@ -2,9 +2,12 @@ from queryset import QuerySet, QuerySetManager
from queryset import DoesNotExist, MultipleObjectsReturned
from queryset import DO_NOTHING
from mongoengine import signals
import sys
import pymongo
import pymongo.objectid
from operator import itemgetter
class NotRegistered(Exception):
@@ -126,6 +129,88 @@ class BaseField(object):
self.validate(value)
class DereferenceBaseField(BaseField):
"""Handles the lazy dereferencing of a queryset. Will dereference all
items in a list / dict rather than one at a time.
"""
def __get__(self, instance, owner):
"""Descriptor to automatically dereference references.
"""
from fields import ReferenceField, GenericReferenceField
from connection import _get_db
if instance is None:
# Document class being used rather than a document object
return self
# Get value from document instance if available
value_list = instance._data.get(self.name)
if not value_list:
return super(DereferenceBaseField, self).__get__(instance, owner)
is_list = False
if not hasattr(value_list, 'items'):
is_list = True
value_list = dict([(k,v) for k,v in enumerate(value_list)])
if isinstance(self.field, ReferenceField) and value_list:
db = _get_db()
dbref = {}
collections = {}
for k, v in value_list.items():
dbref[k] = v
# Save any DBRefs
if isinstance(v, (pymongo.dbref.DBRef)):
collections.setdefault(v.collection, []).append((k, v))
# For each collection get the references
for collection, dbrefs in collections.items():
id_map = dict([(v.id, k) for k, v in dbrefs])
references = db[collection].find({'_id': {'$in': id_map.keys()}})
for ref in references:
key = id_map[ref['_id']]
dbref[key] = get_document(ref['_cls'])._from_son(ref)
if is_list:
dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))]
instance._data[self.name] = dbref
# Get value from document instance if available
if isinstance(self.field, GenericReferenceField) and value_list:
db = _get_db()
value_list = [(k,v) for k,v in value_list.items()]
dbref = {}
classes = {}
for k, v in value_list:
dbref[k] = v
# Save any DBRefs
if isinstance(v, (dict, pymongo.son.SON)):
classes.setdefault(v['_cls'], []).append((k, v))
# For each collection get the references
for doc_cls, dbrefs in classes.items():
id_map = dict([(v['_ref'].id, k) for k, v in dbrefs])
doc_cls = get_document(doc_cls)
collection = doc_cls._meta['collection']
references = db[collection].find({'_id': {'$in': id_map.keys()}})
for ref in references:
key = id_map[ref['_id']]
dbref[key] = doc_cls._from_son(ref)
if is_list:
dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))]
instance._data[self.name] = dbref
return super(DereferenceBaseField, self).__get__(instance, owner)
class ObjectIdField(BaseField):
"""An field wrapper around MongoDB's ObjectIds.
"""
@@ -382,6 +467,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
class BaseDocument(object):
def __init__(self, **values):
signals.pre_init.send(self, values=values)
self._data = {}
# Assign default values to instance
for attr_name in self._fields.keys():
@@ -395,6 +482,8 @@ class BaseDocument(object):
except AttributeError:
pass
signals.post_init.send(self)
def validate(self):
"""Ensure that all fields' values are valid and that required fields
are present.

10
mongoengine/document.py
View File

@@ -1,3 +1,4 @@
from mongoengine import signals
from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument,
ValidationError)
from queryset import OperationError
@@ -75,6 +76,8 @@ class Document(BaseDocument):
For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers
have recorded the write and will force an fsync on each server being written to.
"""
signals.pre_save.send(self)
if validate:
self.validate()
@@ -82,6 +85,7 @@ class Document(BaseDocument):
write_options = {}
doc = self.to_mongo()
created = '_id' not in doc
try:
collection = self.__class__.objects._collection
if force_insert:
@@ -96,12 +100,16 @@ class Document(BaseDocument):
id_field = self._meta['id_field']
self[id_field] = self._fields[id_field].to_python(object_id)
signals.post_save.send(self, created=created)
def delete(self, safe=False):
"""Delete the :class:`~mongoengine.Document` from the database. This
will only take effect if the document has been previously saved.
:param safe: check if the operation succeeded before returning
"""
signals.pre_delete.send(self)
id_field = self._meta['id_field']
object_id = self._fields[id_field].to_mongo(self[id_field])
try:
@@ -110,6 +118,8 @@ class Document(BaseDocument):
message = u'Could not delete document (%s)' % err.message
raise OperationError(message)
signals.post_delete.send(self)
@classmethod
def register_delete_rule(cls, document_cls, field_name, rule):
"""This method registers the delete rules to apply when removing this

112
mongoengine/fields.py
View File

@@ -1,4 +1,5 @@
from base import BaseField, ObjectIdField, ValidationError, get_document
from base import (BaseField, DereferenceBaseField, ObjectIdField,
ValidationError, get_document)
from queryset import DO_NOTHING
from document import Document, EmbeddedDocument
from connection import _get_db
@@ -12,7 +13,6 @@ import pymongo.binary
import datetime, time
import decimal
import gridfs
import warnings
__all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField',
@@ -118,8 +118,8 @@ class EmailField(StringField):
EMAIL_REGEX = re.compile(
r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string
r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain
r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string
r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain
)
def validate(self, value):
@@ -153,6 +153,7 @@ class IntField(BaseField):
def prepare_query_value(self, op, value):
return int(value)
class FloatField(BaseField):
"""An floating point number field.
"""
@@ -178,6 +179,7 @@ class FloatField(BaseField):
def prepare_query_value(self, op, value):
return float(value)
class DecimalField(BaseField):
"""A fixed-point decimal number field.
@@ -227,6 +229,10 @@ class BooleanField(BaseField):
class DateTimeField(BaseField):
"""A datetime field.
Note: Microseconds are rounded to the nearest millisecond.
Pre UTC microsecond support is effecively broken see
`tests.field.test_datetime` for more information.
"""
def validate(self, value):
@@ -252,21 +258,21 @@ class DateTimeField(BaseField):
else:
usecs = 0
kwargs = {'microsecond': usecs}
try: # Seconds are optional, so try converting seconds first.
try: # Seconds are optional, so try converting seconds first.
return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6],
**kwargs)
except ValueError:
try: # Try without seconds.
try: # Try without seconds.
return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5],
**kwargs)
except ValueError: # Try without hour/minutes/seconds.
except ValueError: # Try without hour/minutes/seconds.
try:
return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3],
**kwargs)
except ValueError:
return None
class EmbeddedDocumentField(BaseField):
"""An embedded document field. Only valid values are subclasses of
:class:`~mongoengine.EmbeddedDocument`.
@@ -314,7 +320,7 @@ class EmbeddedDocumentField(BaseField):
return self.to_mongo(value)
class ListField(BaseField):
class ListField(DereferenceBaseField):
"""A list field that wraps a standard field, allowing multiple instances
of the field to be used as a list in the database.
"""
@@ -330,42 +336,6 @@ class ListField(BaseField):
kwargs.setdefault('default', lambda: [])
super(ListField, self).__init__(**kwargs)
def __get__(self, instance, owner):
"""Descriptor to automatically dereference references.
"""
if instance is None:
# Document class being used rather than a document object
return self
if isinstance(self.field, ReferenceField):
referenced_type = self.field.document_type
# Get value from document instance if available
value_list = instance._data.get(self.name)
if value_list:
deref_list = []
for value in value_list:
# Dereference DBRefs
if isinstance(value, (pymongo.dbref.DBRef)):
value = _get_db().dereference(value)
deref_list.append(referenced_type._from_son(value))
else:
deref_list.append(value)
instance._data[self.name] = deref_list
if isinstance(self.field, GenericReferenceField):
value_list = instance._data.get(self.name)
if value_list:
deref_list = []
for value in value_list:
# Dereference DBRefs
if isinstance(value, (dict, pymongo.son.SON)):
deref_list.append(self.field.dereference(value))
else:
deref_list.append(value)
instance._data[self.name] = deref_list
return super(ListField, self).__get__(instance, owner)
def to_python(self, value):
return [self.field.to_python(item) for item in value]
@@ -459,10 +429,10 @@ class DictField(BaseField):
if op in match_operators and isinstance(value, basestring):
return StringField().prepare_query_value(op, value)
return super(DictField,self).prepare_query_value(op, value)
return super(DictField, self).prepare_query_value(op, value)
class MapField(BaseField):
class MapField(DereferenceBaseField):
"""A field that maps a name to a specified field type. Similar to
a DictField, except the 'value' of each item must match the specified
field type.
@@ -494,47 +464,11 @@ class MapField(BaseField):
except Exception, err:
raise ValidationError('Invalid MapField item (%s)' % str(item))
def __get__(self, instance, owner):
"""Descriptor to automatically dereference references.
"""
if instance is None:
# Document class being used rather than a document object
return self
if isinstance(self.field, ReferenceField):
referenced_type = self.field.document_type
# Get value from document instance if available
value_dict = instance._data.get(self.name)
if value_dict:
deref_dict = []
for key,value in value_dict.iteritems():
# Dereference DBRefs
if isinstance(value, (pymongo.dbref.DBRef)):
value = _get_db().dereference(value)
deref_dict[key] = referenced_type._from_son(value)
else:
deref_dict[key] = value
instance._data[self.name] = deref_dict
if isinstance(self.field, GenericReferenceField):
value_dict = instance._data.get(self.name)
if value_dict:
deref_dict = []
for key,value in value_dict.iteritems():
# Dereference DBRefs
if isinstance(value, (dict, pymongo.son.SON)):
deref_dict[key] = self.field.dereference(value)
else:
deref_dict[key] = value
instance._data[self.name] = deref_dict
return super(MapField, self).__get__(instance, owner)
def to_python(self, value):
return dict( [(key,self.field.to_python(item)) for key,item in value.iteritems()] )
return dict([(key, self.field.to_python(item)) for key, item in value.iteritems()])
def to_mongo(self, value):
return dict( [(key,self.field.to_mongo(item)) for key,item in value.iteritems()] )
return dict([(key, self.field.to_mongo(item)) for key, item in value.iteritems()])
def prepare_query_value(self, op, value):
if op not in ('set', 'unset'):
@@ -752,11 +686,11 @@ class GridFSProxy(object):
self.newfile = self.fs.new_file(**kwargs)
self.grid_id = self.newfile._id
def put(self, file, **kwargs):
def put(self, file_obj, **kwargs):
if self.grid_id:
raise GridFSError('This document already has a file. Either delete '
'it or call replace to overwrite it')
self.grid_id = self.fs.put(file, **kwargs)
self.grid_id = self.fs.put(file_obj, **kwargs)
def write(self, string):
if self.grid_id:
@@ -785,9 +719,9 @@ class GridFSProxy(object):
self.grid_id = None
self.gridout = None
def replace(self, file, **kwargs):
def replace(self, file_obj, **kwargs):
self.delete()
self.put(file, **kwargs)
self.put(file_obj, **kwargs)
def close(self):
if self.newfile:

124
mongoengine/queryset.py
View File

@@ -336,6 +336,7 @@ class QuerySet(object):
self._snapshot = False
self._timeout = True
self._class_check = True
self._slave_okay = False
# If inheritance is allowed, only return instances and instances of
# subclasses of the class being used
@@ -352,7 +353,7 @@ class QuerySet(object):
copy_props = ('_initial_query', '_query_obj', '_where_clause',
'_loaded_fields', '_ordering', '_snapshot',
'_timeout', '_limit', '_skip')
'_timeout', '_limit', '_skip', '_slave_okay')
for prop in copy_props:
val = getattr(self, prop)
@@ -376,21 +377,27 @@ class QuerySet(object):
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
"""
index_list = QuerySet._build_index_spec(self._document, key_or_list)
self._collection.ensure_index(index_list, drop_dups=drop_dups,
background=background)
index_spec = QuerySet._build_index_spec(self._document, key_or_list)
self._collection.ensure_index(
index_spec['fields'],
drop_dups=drop_dups,
background=background,
sparse=index_spec.get('sparse', False),
unique=index_spec.get('unique', False))
return self
@classmethod
def _build_index_spec(cls, doc_cls, key_or_list):
def _build_index_spec(cls, doc_cls, spec):
"""Build a PyMongo index spec from a MongoEngine index spec.
"""
if isinstance(key_or_list, basestring):
key_or_list = [key_or_list]
if isinstance(spec, basestring):
spec = {'fields': [spec]}
if isinstance(spec, (list, tuple)):
spec = {'fields': spec}
index_list = []
use_types = doc_cls._meta.get('allow_inheritance', True)
for key in key_or_list:
for key in spec['fields']:
# Get direction from + or -
direction = pymongo.ASCENDING
if key.startswith("-"):
@@ -410,15 +417,22 @@ class QuerySet(object):
if use_types and not all(f._index_with_types for f in fields):
use_types = False
# If _types is being used, create an index for it
# If _types is being used, prepend it to every specified index
index_types = doc_cls._meta.get('index_types', True)
allow_inheritance = doc_cls._meta.get('allow_inheritance')
if index_types and allow_inheritance and use_types:
if spec.get('types', index_types) and allow_inheritance and use_types:
index_list.insert(0, ('_types', 1))
return index_list
spec['fields'] = index_list
def __call__(self, q_obj=None, class_check=True, **query):
if spec.get('sparse', False) and len(spec['fields']) > 1:
raise ValueError(
'Sparse indexes can only have one field in them. '
'See https://jira.mongodb.org/browse/SERVER-2193')
return spec
def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query):
"""Filter the selected documents by calling the
:class:`~mongoengine.queryset.QuerySet` with a query.
@@ -428,6 +442,8 @@ class QuerySet(object):
objects, only the last one will be used
:param class_check: If set to False bypass class name check when
querying collection
:param slave_okay: if True, allows this query to be run against a
replica secondary.
:param query: Django-style query keyword arguments
"""
query = Q(**query)
@@ -468,9 +484,12 @@ class QuerySet(object):
# Ensure document-defined indexes are created
if self._document._meta['indexes']:
for key_or_list in self._document._meta['indexes']:
self._collection.ensure_index(key_or_list,
background=background, **index_opts)
for spec in self._document._meta['indexes']:
opts = index_opts.copy()
opts['unique'] = spec.get('unique', False)
opts['sparse'] = spec.get('sparse', False)
self._collection.ensure_index(spec['fields'],
background=background, **opts)
# If _types is being used (for polymorphism), it needs an index
if index_types and '_types' in self._query:
@@ -486,17 +505,23 @@ class QuerySet(object):
return self._collection_obj
@property
def _cursor_args(self):
cursor_args = {
'snapshot': self._snapshot,
'timeout': self._timeout,
'slave_okay': self._slave_okay
}
if self._loaded_fields:
cursor_args['fields'] = self._loaded_fields.as_dict()
return cursor_args
@property
def _cursor(self):
if self._cursor_obj is None:
cursor_args = {
'snapshot': self._snapshot,
'timeout': self._timeout,
}
if self._loaded_fields:
cursor_args['fields'] = self._loaded_fields.as_dict()
self._cursor_obj = self._collection.find(self._query,
**cursor_args)
**self._cursor_args)
# Apply where clauses to cursor
if self._where_clause:
self._cursor_obj.where(self._where_clause)
@@ -705,6 +730,46 @@ class QuerySet(object):
result = None
return result
def insert(self, doc_or_docs, load_bulk=True):
"""bulk insert documents
:param docs_or_doc: a document or list of documents to be inserted
:param load_bulk (optional): If True returns the list of document instances
By default returns document instances, set ``load_bulk`` to False to
return just ``ObjectIds``
.. versionadded:: 0.5
"""
from document import Document
docs = doc_or_docs
return_one = False
if isinstance(docs, Document) or issubclass(docs.__class__, Document):
return_one = True
docs = [docs]
raw = []
for doc in docs:
if not isinstance(doc, self._document):
msg = "Some documents inserted aren't instances of %s" % str(self._document)
raise OperationError(msg)
if doc.pk:
msg = "Some documents have ObjectIds use doc.update() instead"
raise OperationError(msg)
raw.append(doc.to_mongo())
ids = self._collection.insert(raw)
if not load_bulk:
return return_one and ids[0] or ids
documents = self.in_bulk(ids)
results = []
for obj_id in ids:
results.append(documents.get(obj_id))
return return_one and results[0] or results
def with_id(self, object_id):
"""Retrieve the object matching the id provided.
@@ -713,7 +778,7 @@ class QuerySet(object):
id_field = self._document._meta['id_field']
object_id = self._document._fields[id_field].to_mongo(object_id)
result = self._collection.find_one({'_id': object_id})
result = self._collection.find_one({'_id': object_id}, **self._cursor_args)
if result is not None:
result = self._document._from_son(result)
return result
@@ -729,7 +794,8 @@ class QuerySet(object):
"""
doc_map = {}
docs = self._collection.find({'_id': {'$in': object_ids}})
docs = self._collection.find({'_id': {'$in': object_ids}},
**self._cursor_args)
for doc in docs:
doc_map[doc['_id']] = self._document._from_son(doc)
@@ -1026,6 +1092,7 @@ class QuerySet(object):
:param enabled: whether or not snapshot mode is enabled
"""
self._snapshot = enabled
return self
def timeout(self, enabled):
"""Enable or disable the default mongod timeout when querying.
@@ -1033,6 +1100,15 @@ class QuerySet(object):
:param enabled: whether or not the timeout is used
"""
self._timeout = enabled
return self
def slave_okay(self, enabled):
"""Enable or disable the slave_okay when querying.
:param enabled: whether or not the slave_okay is enabled
"""
self._slave_okay = enabled
return self
def delete(self, safe=False):
"""Delete the documents matched by the query.

44
mongoengine/signals.py Normal file
View File

@@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
__all__ = ['pre_init', 'post_init', 'pre_save', 'post_save',
'pre_delete', 'post_delete']
signals_available = False
try:
from blinker import Namespace
signals_available = True
except ImportError:
class Namespace(object):
def signal(self, name, doc=None):
return _FakeSignal(name, doc)
class _FakeSignal(object):
"""If blinker is unavailable, create a fake class with the same
interface that allows sending of signals but will fail with an
error on anything else. Instead of doing anything on send, it
will just ignore the arguments and do nothing instead.
"""
def __init__(self, name, doc=None):
self.name = name
self.__doc__ = doc
def _fail(self, *args, **kwargs):
raise RuntimeError('signalling support is unavailable '
'because the blinker library is '
'not installed.')
send = lambda *a, **kw: None
connect = disconnect = has_receivers_for = receivers_for = \
temporarily_connected_to = _fail
del _fail
# the namespace for code signals. If you are not mongoengine code, do
# not put signals in here. Create your own namespace instead.
_signals = Namespace()
pre_init = _signals.signal('pre_init')
post_init = _signals.signal('post_init')
pre_save = _signals.signal('pre_save')
post_save = _signals.signal('post_save')
pre_delete = _signals.signal('pre_delete')
post_delete = _signals.signal('post_delete')

59
mongoengine/tests.py Normal file
View File

@@ -0,0 +1,59 @@
from mongoengine.connection import _get_db
class query_counter(object):
""" Query_counter contextmanager to get the number of queries. """
def __init__(self):
""" Construct the query_counter. """
self.counter = 0
self.db = _get_db()
def __enter__(self):
""" On every with block we need to drop the profile collection. """
self.db.set_profiling_level(0)
self.db.system.profile.drop()
self.db.set_profiling_level(2)
return self
def __exit__(self, t, value, traceback):
""" Reset the profiling level. """
self.db.set_profiling_level(0)
def __eq__(self, value):
""" == Compare querycounter. """
return value == self._get_count()
def __ne__(self, value):
""" != Compare querycounter. """
return not self.__eq__(value)
def __lt__(self, value):
""" < Compare querycounter. """
return self._get_count() < value
def __le__(self, value):
""" <= Compare querycounter. """
return self._get_count() <= value
def __gt__(self, value):
""" > Compare querycounter. """
return self._get_count() > value
def __ge__(self, value):
""" >= Compare querycounter. """
return self._get_count() >= value
def __int__(self):
""" int representation. """
return self._get_count()
def __repr__(self):
""" repr query_counter as the number of queries. """
return u"%s" % self._get_count()
def _get_count(self):
""" Get the number of queries. """
count = self.db.system.profile.find().count() - self.counter
self.counter += 1
return count