Simplified Q objects

Removed QueryTreeTransformerVisitor (#98) (#171)

docs/changelog.rst

@@ -24,6 +24,7 @@ Changes in 0.8.X
 - Added support for multiple slices and made slicing chainable. (#170) (#190) (#191)
 - Fixed GridFSProxy __getattr__ behaviour (#196)
 - Fix Django timezone support (#151)
+- Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171)
 
 Changes in 0.7.9
 ================

mongoengine/queryset/visitor.py

@@ -55,57 +55,6 @@ class SimplificationVisitor(QNodeVisitor):
         return combined_query
 
 
-class QueryTreeTransformerVisitor(QNodeVisitor):
-    """Transforms the query tree in to a form that may be used with MongoDB.
-    """
-
-    def visit_combination(self, combination):
-        if combination.operation == combination.AND:
-            # MongoDB doesn't allow us to have too many $or operations in our
-            # queries, so the aim is to move the ORs up the tree to one
-            # 'master' $or. Firstly, we must find all the necessary parts (part
-            # of an AND combination or just standard Q object), and store them
-            # separately from the OR parts.
-            or_groups = []
-            and_parts = []
-            for node in combination.children:
-                if isinstance(node, QCombination):
-                    if node.operation == node.OR:
-                        # Any of the children in an $or component may cause
-                        # the query to succeed
-                        or_groups.append(node.children)
-                    elif node.operation == node.AND:
-                        and_parts.append(node)
-                elif isinstance(node, Q):
-                    and_parts.append(node)
-
-            # Now we combine the parts into a usable query. AND together all of
-            # the necessary parts. Then for each $or part, create a new query
-            # that ANDs the necessary part with the $or part.
-            clauses = []
-            for or_group in product(*or_groups):
-                q_object = reduce(lambda a, b: a & b, and_parts, Q())
-                q_object = reduce(lambda a, b: a & b, or_group, q_object)
-                clauses.append(q_object)
-            # Finally, $or the generated clauses in to one query. Each of the
-            # clauses is sufficient for the query to succeed.
-            return reduce(lambda a, b: a | b, clauses, Q())
-
-        if combination.operation == combination.OR:
-            children = []
-            # Crush any nested ORs in to this combination as MongoDB doesn't
-            # support nested $or operations
-            for node in combination.children:
-                if (isinstance(node, QCombination) and
-                    node.operation == combination.OR):
-                    children += node.children
-                else:
-                    children.append(node)
-            combination.children = children
-
-        return combination
-
-
 class QueryCompilerVisitor(QNodeVisitor):
     """Compiles the nodes in a query tree to a PyMongo-compatible query
     dictionary.
@@ -115,45 +64,14 @@ class QueryCompilerVisitor(QNodeVisitor):
         self.document = document
 
     def visit_combination(self, combination):
+        operator = "$and"
         if combination.operation == combination.OR:
-            return {'$or': combination.children}
+            operator = "$or"
-        elif combination.operation == combination.AND:
+        return {operator: combination.children}
-            return self._mongo_query_conjunction(combination.children)
-        return combination
 
     def visit_query(self, query):
         return transform.query(self.document, **query.query)
 
-    def _mongo_query_conjunction(self, queries):
-        """Merges Mongo query dicts - effectively &ing them together.
-        """
-        combined_query = {}
-        for query in queries:
-            for field, ops in query.items():
-                if field not in combined_query:
-                    combined_query[field] = ops
-                else:
-                    # The field is already present in the query the only way
-                    # we can merge is if both the existing value and the new
-                    # value are operation dicts, reject anything else
-                    if (not isinstance(combined_query[field], dict) or
-                        not isinstance(ops, dict)):
-                        message = 'Conflicting values for ' + field
-                        raise InvalidQueryError(message)
-
-                    current_ops = set(combined_query[field].keys())
-                    new_ops = set(ops.keys())
-                    # Make sure that the same operation isn't applied more than
-                    # once to a single field
-                    intersection = current_ops.intersection(new_ops)
-                    if intersection:
-                        msg = 'Duplicate query conditions: '
-                        raise InvalidQueryError(msg + ', '.join(intersection))
-
-                    # Right! We've got two non-overlapping dicts of operations!
-                    combined_query[field].update(copy.deepcopy(ops))
-        return combined_query
-
 
 class QNode(object):
     """Base class for nodes in query trees.
@@ -164,7 +82,6 @@ class QNode(object):
 
     def to_query(self, document):
         query = self.accept(SimplificationVisitor())
-        query = query.accept(QueryTreeTransformerVisitor())
         query = query.accept(QueryCompilerVisitor(document))
         return query
 
@@ -205,7 +122,8 @@ class QCombination(QNode):
             # If the child is a combination of the same type, we can merge its
             # children directly into this combinations children
             if isinstance(node, QCombination) and node.operation == operation:
-                self.children += node.children
+                # self.children += node.children
+                self.children.append(node)
             else:
                 self.children.append(node)
 

tests/queryset/visitor.py

@@ -115,29 +115,31 @@ class QTest(unittest.TestCase):
             x = IntField()
             y = BooleanField()
 
+        TestDoc.drop_collection()
+
         query = (Q(x__gt=0) | Q(x__exists=False))
         query &= Q(x__lt=100)
-        self.assertEqual(query.to_query(TestDoc), {
+        self.assertEqual(query.to_query(TestDoc), {'$and': [
-            '$or': [
+            {'$or': [{'x': {'$gt': 0}},
-                {'x': {'$lt': 100, '$gt': 0}},
+                     {'x': {'$exists': False}}]},
-                {'x': {'$lt': 100, '$exists': False}},
+            {'x': {'$lt': 100}}]
-            ]
         })
 
         q1 = (Q(x__gt=0) | Q(x__exists=False))
         q2 = (Q(x__lt=100) | Q(y=True))
         query = (q1 & q2).to_query(TestDoc)
 
-        self.assertEqual(['$or'], query.keys())
+        TestDoc(x=101).save()
-        conditions = [
+        TestDoc(x=10).save()
-            {'x': {'$lt': 100, '$gt': 0}},
+        TestDoc(y=True).save()
-            {'x': {'$lt': 100, '$exists': False}},
+
-            {'x': {'$gt': 0}, 'y': True},
+        self.assertEqual(query,
-            {'x': {'$exists': False}, 'y': True},
+        {'$and': [
-        ]
+            {'$or': [{'x': {'$gt': 0}}, {'x': {'$exists': False}}]},
-        self.assertEqual(len(conditions), len(query['$or']))
+            {'$or': [{'x': {'$lt': 100}}, {'y': True}]}
-        for condition in conditions:
+        ]})
-            self.assertTrue(condition in query['$or'])
+
+        self.assertEqual(2, TestDoc.objects(q1 & q2).count())
 
     def test_or_and_or_combination(self):
         """Ensure that Q-objects handle ORing ANDed ORed components. :)
@@ -146,20 +148,40 @@ class QTest(unittest.TestCase):
             x = IntField()
             y = BooleanField()
 
+        TestDoc.drop_collection()
+        TestDoc(x=-1, y=True).save()
+        TestDoc(x=101, y=True).save()
+        TestDoc(x=99, y=False).save()
+        TestDoc(x=101, y=False).save()
+
         q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False)))
         q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False)))
         query = (q1 | q2).to_query(TestDoc)
 
-        self.assertEqual(['$or'], query.keys())
+        self.assertEqual(query,
-        conditions = [
+            {'$or': [
-            {'x': {'$gt': 0}, 'y': True},
+                {'$and': [{'x': {'$gt': 0}},
-            {'x': {'$gt': 0}, 'y': {'$exists': False}},
+                          {'$or': [{'y': True}, {'y': {'$exists': False}}]}]},
-            {'x': {'$lt': 100}, 'y':False},
+                {'$and': [{'x': {'$lt': 100}},
-            {'x': {'$lt': 100}, 'y': {'$exists': False}},
+                          {'$or': [{'y': False}, {'y': {'$exists': False}}]}]}
-        ]
+            ]}
-        self.assertEqual(len(conditions), len(query['$or']))
+        )
-        for condition in conditions:
+
-            self.assertTrue(condition in query['$or'])
+        self.assertEqual(2, TestDoc.objects(q1 | q2).count())
+
+    def test_multiple_occurence_in_field(self):
+        class Test(Document):
+            name = StringField(max_length=40)
+            title = StringField(max_length=40)
+
+        q1 = Q(name__contains='te') | Q(title__contains='te')
+        q2 = Q(name__contains='12') | Q(title__contains='12')
+
+        q3 = q1 & q2
+
+        query = q3.to_query(Test)
+        self.assertEqual(query["$and"][0], q1.to_query(Test))
+        self.assertEqual(query["$and"][1], q2.to_query(Test))
 
     def test_q_clone(self):