From d0e0b291df4d8c1501f5caad491b758a8f87ef64 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Thu, 25 Feb 2010 17:20:52 +0000 Subject: [PATCH 1/6] Implementation and tests for exec_js field substitution --- mongoengine/queryset.py | 18 ++++++++++++++ tests/queryset.py | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 563c25ac..3ac8a37a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,6 +1,7 @@ from connection import _get_db import pymongo +import re import copy @@ -593,6 +594,21 @@ class QuerySet(object): def __iter__(self): return self + def _sub_js_fields(self, code): + """When fields are specified with [~fieldname] syntax, where + *fieldname* is the Python name of a field, *fieldname* will be + substituted for the MongoDB name of the field (specified using the + :attr:`name` keyword argument in a field's constructor). + """ + def field_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = QuerySet._lookup_field(self._document, field_name) + # Substitute the correct name for the field into the javascript + return '["%s"]' % fields[-1].name + + return re.sub('\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be provided, which will be translated to their correct names and supplied @@ -608,6 +624,8 @@ class QuerySet(object): :param options: options that you want available to the function (accessed in Javascript through the ``options`` object) """ + code = self._sub_js_fields(code) + fields = [QuerySet._translate_field_name(self._document, f) for f in fields] collection = self._document._meta['collection'] diff --git a/tests/queryset.py b/tests/queryset.py index 61cb0537..d287efad 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -387,6 +387,58 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_exec_js_field_sub(self): + """Ensure that field substitutions occur properly in exec_js functions. + """ + class Comment(EmbeddedDocument): + content = StringField(name='body') + + class BlogPost(Document): + name = StringField(name='doc-name') + comments = ListField(EmbeddedDocumentField(Comment), name='cmnts') + + BlogPost.drop_collection() + + comments1 = [Comment(content='cool'), Comment(content='yay')] + post1 = BlogPost(name='post1', comments=comments1) + post1.save() + + comments2 = [Comment(content='nice stuff')] + post2 = BlogPost(name='post2', comments=comments2) + post2.save() + + code = """ + function getComments() { + var comments = []; + db[collection].find(query).forEach(function(doc) { + var docComments = doc[~comments]; + for (var i = 0; i < docComments.length; i++) { + comments.push({ + 'document': doc[~name], + 'comment': doc[~comments][i][~comments.content] + }); + } + }); + return comments; + } + """ + + sub_code = BlogPost.objects._sub_js_fields(code) + code_chunks = ['doc["cmnts"];', 'doc["doc-name"],', + 'doc["cmnts"][i]["body"]'] + for chunk in code_chunks: + self.assertTrue(chunk in sub_code) + + results = BlogPost.objects.exec_js(code) + expected_results = [ + {u'comment': u'cool', u'document': u'post1'}, + {u'comment': u'yay', u'document': u'post1'}, + {u'comment': u'nice stuff', u'document': u'post2'}, + ] + self.assertEqual(results, expected_results) + + BlogPost.drop_collection() + def test_delete(self): """Ensure that documents are properly deleted from the database. """ From ab2d019349f22ead61b6612996ff90e066adf33f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 26 Feb 2010 13:23:15 +0000 Subject: [PATCH 2/6] Added server-side js docs --- docs/guide/querying.rst | 103 ++++++++++++++++++++++++++++++++++++++++ mongoengine/queryset.py | 11 ++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 0742244d..d57b8b60 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -232,6 +232,109 @@ calling it with keyword arguments:: natively supported by MongoDB -- they are compiled to Javascript and sent to the server for execution. +Server-side javascript execution +================================ +Javascript functions may be written and sent to the server for execution. The +result of this is the return value of the Javascript function. This +functionality is accessed through the +:meth:`~mongoengine.queryset.QuerySet.exec_js` method on +:meth:`~mongoengine.queryset.QuerySet` objects. Pass in a string containing a +Javascript function as the first argument. + +The remaining positional arguments are names of fields that will be passed into +you Javascript function as its arguments. This allows functions to be written +that may be executed on any field in a collection (e.g. the +:meth:`~mongoengine.queryset.QuerySet.sum` method, which accepts the name of +the field to sum over as its argument). Note that field names passed in in this +manner are automatically translated to the names used on the database (set +using the :attr:`name` keyword argument to a field constructor). + +Keyword arguments to :meth:`~mongoengine.queryset.QuerySet.exec_js` are +combined into an object called :attr:`options`, which is available in the +Javascript function. This may be used for defining specific parameters for your +function. + +Some variables are made available in the scope of the Javascript function: + +* ``collection`` -- the name of the collection that corresponds to the + :class:`~mongoengine.Document` class that is being used; this should be + used to get the :class:`Collection` object from :attr:`db` in Javascript + code +* ``query`` -- the query that has been generated by the + :class:`~mongoengine.queryset.QuerySet` object; this may be passed into + the :meth:`find` method on a :class:`Collection` object in the Javascript + function +* ``options`` -- an object containing the keyword arguments passed into + :meth:`~mongoengine.queryset.QuerySet.exec_js` + +The following example demonstrates the intended usage of +:meth:`~mongoengine.queryset.QuerySet.exec_js` by defining a function that sums +over a field on a document (this functionality is already available throught +:meth:`~mongoengine.queryset.QuerySet.sum` but is shown here for sake of +example):: + + def sum_field(document, field_name, include_negatives=True): + code = """ + function(sumField) { + var total = 0.0; + db[collection].find(query).forEach(function(doc) { + var val = doc[sumField]; + if (val >= 0.0 || options.includeNegatives) { + total += val; + } + }); + return total; + } + """ + options = {'includeNegatives': include_negatives} + return document.objects.exec_js(code, field_name, **options) + +As fields in MongoEngine may use different names in the database (set using the +:attr:`name` keyword argument to a :class:`Field` constructor), a mechanism +exists for replacing MongoEngine field names with the database field names in +Javascript code. When accessing a field on a collection object, use +square-bracket notation, and prefix the MongoEngine field name with a tilde. +The field name that follows the tilde will be translated to the name used in +the database. Note that when referring to fields on embedded documents, +the name of the :class:`~mongoengine.EmbeddedDocumentField`, followed by a dot, +should be used before the name of the field on the embedded document. The +following example shows how the substitutions are made:: + + class Comment(EmbeddedDocument): + content = StringField(name='body') + + class BlogPost(Document): + title = StringField(name='doctitle') + comments = ListField(EmbeddedDocumentField(Comment), name='cs') + + # Returns a list of dictionaries. Each dictionary contains a value named + # "document", which corresponds to the "title" field on a BlogPost, and + # "comment", which corresponds to an individual comment. The substitutions + # made are shown in the comments. + BlogPost.objects.exec_js(""" + function() { + var comments = []; + db[collection].find(query).forEach(function(doc) { + // doc[~comments] -> doc["cs"] + var docComments = doc[~comments]; + + for (var i = 0; i < docComments.length; i++) { + // doc[~comments][i] -> doc["cs"][i] + var comment = doc[~comments][i]; + + comments.push({ + // doc[~title] -> doc["doctitle"] + 'document': doc[~title], + + // comment[~comments.content] -> comment["body"] + 'comment': comment[~comments.content] + }); + } + }); + return comments; + } + """) + .. _guide-atomic-updates: Atomic updates diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3ac8a37a..9592236d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -373,7 +373,7 @@ class QuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - :param object_ids: a list or tuple of ``ObjectId``s + :param object_ids: a list or tuple of ``ObjectId``\ s :rtype: dict of ObjectIds as keys and collection-specific Document subclasses as values. """ @@ -455,7 +455,7 @@ class QuerySet(object): post = BlogPost.objects(...).only("title") - :param *fields: fields to include + :param fields: fields to include """ self._loaded_fields = list(fields) return self @@ -618,6 +618,13 @@ class QuerySet(object): current query; and ``options``, which is an object containing any options specified as keyword arguments. + As fields in MongoEngine may use different names in the database (set + using the :attr:`name` keyword argument to a :class:`Field` + constructor), a mechanism exists for replacing MongoEngine field names + with the database field names in Javascript code. When accessing a + field, use square-bracket notation, and prefix the MongoEngine field + name with a tilde (~). + :param code: a string of Javascript code to execute :param fields: fields that you will be using in your function, which will be passed in to your function as arguments From 66520c77f86c0ebf41e5b33df92c4bb5becd2471 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 26 Feb 2010 13:43:32 +0000 Subject: [PATCH 3/6] Added regex match operators with test --- mongoengine/fields.py | 19 ++++++++++++++++++- mongoengine/queryset.py | 10 +++++++--- tests/queryset.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2f800fc5..6bca1b2a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -39,8 +39,25 @@ class StringField(BaseField): def lookup_member(self, member_name): return None + def prepare_query_value(self, op, value): + if not isinstance(op, basestring): + return value -class URLField(BaseField): + if op.lstrip('i') in ('startswith', 'endswith', 'contains'): + flags = 0 + if op.startswith('i'): + flags = re.IGNORECASE + op = op.lstrip('i') + + regex = r'%s' + if op == 'startswith': + regex = r'^%s' + elif op == 'endswith': + regex = r'%s$' + value = re.compile(regex % value, flags) + return value + +class URLField(StringField): """A field that validates input as a URL. """ diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9592236d..4d8fb7ff 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -275,13 +275,15 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists'] + match_operators = ['contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith'] mongo_query = {} for key, value in query.items(): parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in operators: + if parts[-1] in operators + match_operators: op = parts.pop() if _doc_cls: @@ -291,13 +293,15 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'ne', 'gt', 'gte', 'lt', 'lte'): + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] + singular_ops += match_operators + if op in singular_ops: value = field.prepare_query_value(op, value) elif op in ('in', 'nin', 'all'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] - if op: + if op and op not in match_operators: value = {'$' + op: value} key = '.'.join(parts) diff --git a/tests/queryset.py b/tests/queryset.py index d287efad..5b434e95 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -186,6 +186,41 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age=50) self.assertEqual(person.name, "User C") + def test_regex_query_shortcuts(self): + """Ensure that contains, startswith, endswith, etc work. + """ + person = self.Person(name='Guido van Rossum') + person.save() + + # Test contains + obj = self.Person.objects(name__contains='van').first() + self.assertEqual(obj, person) + obj = self.Person.objects(name__contains='Van').first() + self.assertEqual(obj, None) + + # Test icontains + obj = self.Person.objects(name__icontains='Van').first() + self.assertEqual(obj, person) + + # Test startswith + obj = self.Person.objects(name__startswith='Guido').first() + self.assertEqual(obj, person) + obj = self.Person.objects(name__startswith='guido').first() + self.assertEqual(obj, None) + + # Test istartswith + obj = self.Person.objects(name__istartswith='guido').first() + self.assertEqual(obj, person) + + # Test endswith + obj = self.Person.objects(name__endswith='Rossum').first() + self.assertEqual(obj, person) + obj = self.Person.objects(name__endswith='rossuM').first() + self.assertEqual(obj, None) + + # Test iendswith + obj = self.Person.objects(name__iendswith='rossuM').first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. From 719a653375b6b337729173622c4d197aa1abfed7 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 26 Feb 2010 13:48:00 +0000 Subject: [PATCH 4/6] Added match operator docs --- docs/guide/querying.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index d57b8b60..9d6ec827 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -68,6 +68,16 @@ Available operators are as follows: * ``size`` -- the size of the array is * ``exists`` -- value for field exists +The following operators are available as shortcuts to querying with regular +expressions: + +* ``contains`` -- string field contains value +* ``icontains`` -- string field contains value (case insensitive) +* ``startswith`` -- string field starts with value +* ``istartswith`` -- string field starts with value (case insensitive) +* ``endswith`` -- string field ends with value +* ``iendswith`` -- string field ends with value (case insensitive) + Limiting and skipping results ============================= Just as with traditional ORMs, you may limit the number of results returned, or From 0b1c50662667027283f4c2fede189872dbdd5c55 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 26 Feb 2010 16:46:07 +0000 Subject: [PATCH 5/6] Added Q object support for regexes (inc. operator shortcuts) --- mongoengine/queryset.py | 36 ++++++++++++++++++++++++++---------- tests/queryset.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4d8fb7ff..2d146ef9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -28,6 +28,8 @@ class OperationError(Exception): pass +RE_TYPE = type(re.compile('')) + class Q(object): OR = '||' @@ -47,6 +49,8 @@ class Q(object): 'return this.%(field)s.indexOf(a) != -1 })'), 'size': 'this.%(field)s.length == %(value)s', 'exists': 'this.%(field)s != null', + 'regex_eq': '%(value)s.test(this.%(field)s)', + 'regex_ne': '!%(value)s.test(this.%(field)s)', } def __init__(self, **query): @@ -91,24 +95,36 @@ class Q(object): for j, (op, value) in enumerate(value.items()): # Create a custom variable name for this operator op_value_name = '%so%s' % (value_name, j) + # Construct the JS that uses this op + operation_js = self._build_op_js(op, key, value, + op_value_name) # Update the js scope with the value for this op js_scope[op_value_name] = value - # Construct the JS that uses this op - operation_js = Q.OPERATORS[op.strip('$')] % { - 'field': key, - 'value': op_value_name - } js.append(operation_js) else: - js_scope[value_name] = value # Construct the JS for this field - field_js = Q.OPERATORS[op.strip('$')] % { - 'field': key, - 'value': value_name - } + field_js = self._build_op_js(op, key, value, value_name) + js_scope[value_name] = value js.append(field_js) return ' && '.join(js) + def _build_op_js(self, op, key, value, value_name): + """Substitute the values in to the correct chunk of Javascript. + """ + if isinstance(value, RE_TYPE): + # Regexes are handled specially + if op.strip('$') == 'ne': + op_js = Q.OPERATORS['regex_ne'] + else: + op_js = Q.OPERATORS['regex_eq'] + else: + op_js = Q.OPERATORS[op.strip('$')] + # Perform the substitution + operation_js = op_js % { + 'field': key, + 'value': value_name + } + return operation_js class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, diff --git a/tests/queryset.py b/tests/queryset.py index 5b434e95..ee8d09ef 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -197,30 +197,48 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(name__contains='Van').first() self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__contains='van')).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__contains='Van')).first() + self.assertEqual(obj, None) # Test icontains obj = self.Person.objects(name__icontains='Van').first() self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__icontains='Van')).first() + self.assertEqual(obj, person) # Test startswith obj = self.Person.objects(name__startswith='Guido').first() self.assertEqual(obj, person) obj = self.Person.objects(name__startswith='guido').first() self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__startswith='Guido')).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__startswith='guido')).first() + self.assertEqual(obj, None) # Test istartswith obj = self.Person.objects(name__istartswith='guido').first() self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__istartswith='guido')).first() + self.assertEqual(obj, person) # Test endswith obj = self.Person.objects(name__endswith='Rossum').first() self.assertEqual(obj, person) obj = self.Person.objects(name__endswith='rossuM').first() self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__endswith='Rossum')).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__endswith='rossuM')).first() + self.assertEqual(obj, None) # Test iendswith obj = self.Person.objects(name__iendswith='rossuM').first() self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__iendswith='rossuM')).first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. @@ -378,6 +396,26 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + def test_q_regex(self): + """Ensure that Q objects can be queried using regexes. + """ + person = self.Person(name='Guido van Rossum') + person.save() + + import re + obj = self.Person.objects(Q(name=re.compile('^Gui'))).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name=re.compile('^gui'))).first() + self.assertEqual(obj, None) + + obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() + self.assertEqual(obj, person) + + obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() + self.assertEqual(obj, None) + def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. """ From 6e77e328551a17beaed3a033a20a8d2480ae6610 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 26 Feb 2010 17:13:19 +0000 Subject: [PATCH 6/6] Fixed Q object ObjectId comparison issue --- mongoengine/queryset.py | 13 +++++++++---- tests/queryset.py | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2d146ef9..de343e82 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -96,14 +96,14 @@ class Q(object): # Create a custom variable name for this operator op_value_name = '%so%s' % (value_name, j) # Construct the JS that uses this op - operation_js = self._build_op_js(op, key, value, - op_value_name) + value, operation_js = self._build_op_js(op, key, value, + op_value_name) # Update the js scope with the value for this op js_scope[op_value_name] = value js.append(operation_js) else: # Construct the JS for this field - field_js = self._build_op_js(op, key, value, value_name) + value, field_js = self._build_op_js(op, key, value, value_name) js_scope[value_name] = value js.append(field_js) return ' && '.join(js) @@ -119,12 +119,17 @@ class Q(object): op_js = Q.OPERATORS['regex_eq'] else: op_js = Q.OPERATORS[op.strip('$')] + + # Comparing two ObjectIds in Javascript doesn't work.. + if isinstance(value, pymongo.objectid.ObjectId): + value = str(value) + # Perform the substitution operation_js = op_js % { 'field': key, 'value': value_name } - return operation_js + return value, operation_js class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, diff --git a/tests/queryset.py b/tests/queryset.py index ee8d09ef..0189ff55 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -376,6 +376,11 @@ class QuerySetTest(unittest.TestCase): post6 = BlogPost(published=False) post6.save() + # Check ObjectId lookup works + obj = BlogPost.objects(id=post1.id).first() + self.assertEqual(obj, post1) + + # Check Q object combination date = datetime(2010, 1, 10) q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) posts = [post.id for post in q]