merged conflicts
This commit is contained in:
blackbrrr
commit
22a6ec7794
@ -22,7 +22,7 @@ sys.path.append(os.path.abspath('..'))
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = ['sphinx.ext.autodoc']
|
||||
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
@ -115,3 +115,41 @@ class Document(BaseDocument):
|
||||
"""
|
||||
db = _get_db()
|
||||
db.drop_collection(cls._meta['collection'])
|
||||
|
||||
|
||||
class MapReduceDocument(object):
|
||||
"""A document returned from a map/reduce query.
|
||||
|
||||
:param collection: An instance of :class:`~pymongo.Collection`
|
||||
:param key: Document/result key, often an instance of
|
||||
:class:`~pymongo.objectid.ObjectId`. If supplied as
|
||||
an ``ObjectId`` found in the given ``collection``,
|
||||
the object can be accessed via the ``object`` property.
|
||||
:param value: The result(s) for this key.
|
||||
|
||||
.. versionadded:: 0.2.2
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, document, collection, key, value):
|
||||
self._document = document
|
||||
self._collection = collection
|
||||
self.key = key
|
||||
self.value = value
|
||||
|
||||
@property
|
||||
def object(self):
|
||||
"""Lazy-load the object referenced by ``self.key``. If ``self.key``
|
||||
is not an ``ObjectId``, simply return ``self.key``.
|
||||
"""
|
||||
if not isinstance(self.key, (pymongo.objectid.ObjectId)):
|
||||
try:
|
||||
self.key = pymongo.objectid.ObjectId(self.key)
|
||||
except:
|
||||
return self.key
|
||||
if not hasattr(self, "_key_object"):
|
||||
self._key_object = self._document.objects.with_id(self.key)
|
||||
return self._key_object
|
||||
return self._key_object
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ REPR_OUTPUT_SIZE = 20
|
||||
|
||||
|
||||
class DoesNotExist(Exception):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class MultipleObjectsReturned(Exception):
|
||||
@ -30,6 +30,7 @@ class OperationError(Exception):
|
||||
|
||||
RE_TYPE = type(re.compile(''))
|
||||
|
||||
|
||||
class Q(object):
|
||||
|
||||
OR = '||'
|
||||
@ -103,7 +104,7 @@ class Q(object):
|
||||
js.append(operation_js)
|
||||
else:
|
||||
# Construct the JS for this field
|
||||
value, field_js = self._build_op_js(op, key, value, value_name)
|
||||
(op, key, value, value_name)
|
||||
js_scope[value_name] = value
|
||||
js.append(field_js)
|
||||
return ' && '.join(js)
|
||||
@ -360,7 +361,7 @@ class QuerySet(object):
|
||||
may be provided as a keyword argument called :attr:`defaults`.
|
||||
"""
|
||||
defaults = query.get('defaults', {})
|
||||
if query.has_key('defaults'):
|
||||
if 'defaults' in query:
|
||||
del query['defaults']
|
||||
|
||||
self.__call__(*q_objs, **query)
|
||||
@ -430,6 +431,70 @@ class QuerySet(object):
|
||||
def __len__(self):
|
||||
return self.count()
|
||||
|
||||
def map_reduce(self, map_f, reduce_f, finalize_f=None, limit=None,
|
||||
scope=None, keep_temp=False):
|
||||
"""Perform a map/reduce query using the current query spec
|
||||
and ordering. While ``map_reduce`` respects ``QuerySet`` chaining,
|
||||
it must be the last call made, as it does not return a maleable
|
||||
``QuerySet``.
|
||||
|
||||
See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce`
|
||||
and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced`
|
||||
tests in ``tests.queryset.QuerySetTest`` for usage examples.
|
||||
|
||||
:param map_f: map function, as :class:`~pymongo.code.Code` or string
|
||||
:param reduce_f: reduce function, as
|
||||
:class:`~pymongo.code.Code` or string
|
||||
:param finalize_f: finalize function, an optional function that
|
||||
performs any post-reduction processing.
|
||||
:param scope: values to insert into map/reduce global scope. Optional.
|
||||
:param limit: number of objects from current query to provide
|
||||
to map/reduce method
|
||||
:param keep_temp: keep temporary table (boolean, default ``True``)
|
||||
|
||||
Returns an iterator yielding
|
||||
:class:`~mongoengine.document.MapReduceDocument`.
|
||||
|
||||
.. note:: Map/Reduce requires server version **>= 1.1.1**. The PyMongo
|
||||
:meth:`~pymongo.collection.Collection.map_reduce` helper requires
|
||||
PyMongo version **>= 1.2**.
|
||||
|
||||
.. versionadded:: 0.2.2
|
||||
|
||||
"""
|
||||
from document import MapReduceDocument
|
||||
|
||||
if not hasattr(self._collection, "map_reduce"):
|
||||
raise NotImplementedError("Requires MongoDB >= 1.1.1")
|
||||
|
||||
if not isinstance(map_f, pymongo.code.Code):
|
||||
map_f = pymongo.code.Code(map_f)
|
||||
if not isinstance(reduce_f, pymongo.code.Code):
|
||||
reduce_f = pymongo.code.Code(reduce_f)
|
||||
|
||||
mr_args = {'query': self._query, 'keeptemp': keep_temp}
|
||||
|
||||
if finalize_f:
|
||||
if not isinstance(finalize_f, pymongo.code.Code):
|
||||
finalize_f = pymongo.code.Code(finalize_f)
|
||||
mr_args['finalize'] = finalize_f
|
||||
|
||||
if scope:
|
||||
mr_args['scope'] = scope
|
||||
|
||||
if limit:
|
||||
mr_args['limit'] = limit
|
||||
|
||||
results = self._collection.map_reduce(map_f, reduce_f, **mr_args)
|
||||
results = results.find()
|
||||
|
||||
if self._ordering:
|
||||
results = results.sort(self._ordering)
|
||||
|
||||
for doc in results:
|
||||
yield MapReduceDocument(self._document, self._collection,
|
||||
doc['_id'], doc['value'])
|
||||
|
||||
def limit(self, n):
|
||||
"""Limit the number of returned documents to `n`. This may also be
|
||||
achieved using array-slicing syntax (e.g. ``User.objects[:5]``).
|
||||
@ -441,6 +506,7 @@ class QuerySet(object):
|
||||
else:
|
||||
self._cursor.limit(n)
|
||||
self._limit = n
|
||||
|
||||
# Return self to allow chaining
|
||||
return self
|
||||
|
||||
@ -516,6 +582,7 @@ class QuerySet(object):
|
||||
key = key[1:]
|
||||
key_list.append((key, direction))
|
||||
|
||||
self._ordering = key_list
|
||||
self._cursor.sort(key_list)
|
||||
return self
|
||||
|
||||
@ -827,6 +894,7 @@ class QuerySetManager(object):
|
||||
queryset = self._manager_func(owner, queryset)
|
||||
return queryset
|
||||
|
||||
|
||||
def queryset_manager(func):
|
||||
"""Decorator that allows you to define custom QuerySet managers on
|
||||
:class:`~mongoengine.Document` classes. The manager must be a function that
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
import unittest
|
||||
import pymongo
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from mongoengine.queryset import (QuerySet, MultipleObjectsReturned,
|
||||
DoesNotExist)
|
||||
@ -619,6 +622,163 @@ class QuerySetTest(unittest.TestCase):
|
||||
ages = [p.age for p in self.Person.objects.order_by('-name')]
|
||||
self.assertEqual(ages, [30, 40, 20])
|
||||
|
||||
def test_map_reduce(self):
|
||||
"""Ensure map/reduce is both mapping and reducing.
|
||||
"""
|
||||
class Song(Document):
|
||||
artists = ListField(StringField())
|
||||
title = StringField()
|
||||
is_cover = BooleanField()
|
||||
|
||||
Song.drop_collection()
|
||||
|
||||
Song(title="Gloria", is_cover=True, artists=['Patti Smith']).save()
|
||||
Song(title="Redondo beach", is_cover=False,
|
||||
artists=['Patti Smith']).save()
|
||||
Song(title="My Generation", is_cover=True,
|
||||
artists=['Patti Smith', 'John Cale']).save()
|
||||
|
||||
map_f = """
|
||||
function() {
|
||||
this.artists.forEach(function(artist) {
|
||||
emit(artist, 1);
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
reduce_f = """
|
||||
function(key, values) {
|
||||
var total = 0;
|
||||
for(var i=0; i<values.length; i++) {
|
||||
total += values[i];
|
||||
}
|
||||
return total;
|
||||
}
|
||||
"""
|
||||
|
||||
# ensure both artists are found
|
||||
results = Song.objects.map_reduce(map_f, reduce_f)
|
||||
results = list(results)
|
||||
self.assertEqual(len(results), 2)
|
||||
|
||||
# query for a count of Songs per artist, ordered by -count.
|
||||
# Patti Smith has 3 song credits, and should therefore be first.
|
||||
results = Song.objects.order_by("-value").map_reduce(map_f, reduce_f)
|
||||
results = list(results)
|
||||
self.assertEqual(results[0].key, "Patti Smith")
|
||||
self.assertEqual(results[0].value, 3.0)
|
||||
|
||||
Song.drop_collection()
|
||||
|
||||
def test_map_reduce_finalize(self):
|
||||
"""Ensure that map, reduce, and finalize run and introduce "scope"
|
||||
by simulating "hotness" ranking with Reddit algorithm.
|
||||
"""
|
||||
from time import mktime
|
||||
|
||||
class Link(Document):
|
||||
title = StringField()
|
||||
up_votes = IntField()
|
||||
down_votes = IntField()
|
||||
submitted = DateTimeField()
|
||||
|
||||
Link.drop_collection()
|
||||
|
||||
now = datetime.utcnow()
|
||||
|
||||
# Note: Test data taken from a custom Reddit homepage on
|
||||
# Fri, 12 Feb 2010 14:36:00 -0600. Link ordering should
|
||||
# reflect order of insertion below.
|
||||
|
||||
Link(title = "Google Buzz auto-followed a woman's abusive ex ...",
|
||||
up_votes = 1079,
|
||||
down_votes = 553,
|
||||
submitted = now-timedelta(hours=4)).save()
|
||||
Link(title = "We did it! Barbie is a computer engineer.",
|
||||
up_votes = 481,
|
||||
down_votes = 124,
|
||||
submitted = now-timedelta(hours=2)).save()
|
||||
Link(title = "This Is A Mosquito Getting Killed By A Laser",
|
||||
up_votes = 1446,
|
||||
down_votes = 530,
|
||||
submitted=now-timedelta(hours=13)).save()
|
||||
Link(title = "Arabic flashcards land physics student in jail.",
|
||||
up_votes = 215,
|
||||
down_votes = 105,
|
||||
submitted = now-timedelta(hours=6)).save()
|
||||
Link(title = "The Burger Lab: Presenting, the Flood Burger",
|
||||
up_votes = 48,
|
||||
down_votes = 17,
|
||||
submitted = now-timedelta(hours=5)).save()
|
||||
Link(title="How to see polarization with the naked eye",
|
||||
up_votes = 74,
|
||||
down_votes = 13,
|
||||
submitted = now-timedelta(hours=10)).save()
|
||||
|
||||
map_f = """
|
||||
function() {
|
||||
emit(this._id, {up_delta: this.up_votes - this.down_votes,
|
||||
sub_date: this.submitted.getTime() / 1000})
|
||||
}
|
||||
"""
|
||||
|
||||
reduce_f = """
|
||||
function(key, values) {
|
||||
data = values[0];
|
||||
|
||||
x = data.up_delta;
|
||||
|
||||
// calculate time diff between reddit epoch and submission
|
||||
sec_since_epoch = data.sub_date - reddit_epoch;
|
||||
|
||||
// calculate 'Y'
|
||||
if(x > 0) {
|
||||
y = 1;
|
||||
} else if (x = 0) {
|
||||
y = 0;
|
||||
} else {
|
||||
y = -1;
|
||||
}
|
||||
|
||||
// calculate 'Z', the maximal value
|
||||
if(Math.abs(x) >= 1) {
|
||||
z = Math.abs(x);
|
||||
} else {
|
||||
z = 1;
|
||||
}
|
||||
|
||||
return {x: x, y: y, z: z, t_s: sec_since_epoch};
|
||||
}
|
||||
"""
|
||||
|
||||
finalize_f = """
|
||||
function(key, value) {
|
||||
// f(sec_since_epoch,y,z) = log10(z) + ((y*sec_since_epoch) / 45000)
|
||||
z_10 = Math.log(value.z) / Math.log(10);
|
||||
weight = z_10 + ((value.y * value.t_s) / 45000);
|
||||
return weight;
|
||||
}
|
||||
"""
|
||||
|
||||
reddit_epoch = mktime(datetime(2005, 12, 8, 7, 46, 43).timetuple())
|
||||
scope = {'reddit_epoch': reddit_epoch}
|
||||
|
||||
# ensure both artists are found
|
||||
results = Link.objects.order_by("-value")
|
||||
results = results.map_reduce(map_f,
|
||||
reduce_f,
|
||||
finalize_f=finalize_f,
|
||||
scope=scope)
|
||||
results = list(results)
|
||||
|
||||
# assert troublesome Buzz article is ranked 1st
|
||||
self.assertTrue(results[0].object.title.startswith("Google Buzz"))
|
||||
|
||||
# assert laser vision is ranked last
|
||||
self.assertTrue(results[-1].object.title.startswith("How to see"))
|
||||
|
||||
Link.drop_collection()
|
||||
|
||||
def test_item_frequencies(self):
|
||||
"""Ensure that item frequencies are properly generated from lists.
|
||||
"""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user