From 576629f825587029059f831957142c63df525f9d Mon Sep 17 00:00:00 2001 From: lihorne Date: Wed, 28 Jan 2015 08:09:23 -0500 Subject: [PATCH 1/8] Added support for $minDistance query --- mongoengine/queryset/transform.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index f6cfa87e..3171681b 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -15,7 +15,7 @@ COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not', 'elemMatch', 'type') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere', - 'max_distance', 'geo_within', 'geo_within_box', + 'max_distance', 'min_distance', 'geo_within', 'geo_within_box', 'geo_within_polygon', 'geo_within_center', 'geo_within_sphere', 'geo_intersects') STRING_OPERATORS = ('contains', 'icontains', 'startswith', @@ -128,21 +128,29 @@ def query(_doc_cls=None, _field_operation=False, **query): mongo_query[key].update(value) # $maxDistance needs to come last - convert to SON value_dict = mongo_query[key] - if '$maxDistance' in value_dict and '$near' in value_dict: + if ('$maxDistance' in value_dict and '$near' in value_dict): value_son = SON() if isinstance(value_dict['$near'], dict): for k, v in value_dict.iteritems(): - if k == '$maxDistance': + if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v - value_son['$near'] = SON(value_son['$near']) - value_son['$near']['$maxDistance'] = value_dict['$maxDistance'] + if (get_connection().max_wire_version <= 1): + value_son['$maxDistance'] = value_dict[ + '$maxDistance'] + else: + value_son['$near'] = SON(value_son['$near']) + value_son['$near'][ + '$maxDistance'] = value_dict['$maxDistance'] else: for k, v in value_dict.iteritems(): - if k == '$maxDistance': + if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v - value_son['$maxDistance'] = value_dict['$maxDistance'] + if '$maxDistance' in value_dict: + value_son['$maxDistance'] = value_dict['$maxDistance'] + if '$minDistance' in value_dict: + value_son['$minDistance'] = value_dict['$minDistance'] mongo_query[key] = value_son else: @@ -312,6 +320,8 @@ def _geo_operator(field, op, value): value = {'$within': {'$box': value}} elif op == "max_distance": value = {'$maxDistance': value} + elif op == "min_distance": + value = {'$minDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented for a GeoPointField" % op) @@ -332,6 +342,8 @@ def _geo_operator(field, op, value): value = {'$near': _infer_geometry(value)} elif op == "max_distance": value = {'$maxDistance': value} + elif op == "min_distance": + value = {'$minDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented for a %s " % (op, field._name)) From 2d57dc056533c75fdd988698bea14db1de3c17e1 Mon Sep 17 00:00:00 2001 From: Liam Horne Date: Wed, 28 Jan 2015 08:16:31 -0500 Subject: [PATCH 2/8] Fixed an indentation mistake --- mongoengine/queryset/transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 3171681b..637e1b4b 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -147,10 +147,10 @@ def query(_doc_cls=None, _field_operation=False, **query): if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v - if '$maxDistance' in value_dict: - value_son['$maxDistance'] = value_dict['$maxDistance'] - if '$minDistance' in value_dict: - value_son['$minDistance'] = value_dict['$minDistance'] + if '$maxDistance' in value_dict: + value_son['$maxDistance'] = value_dict['$maxDistance'] + if '$minDistance' in value_dict: + value_son['$minDistance'] = value_dict['$minDistance'] mongo_query[key] = value_son else: From 5efabdcea39922c1723d7978c35bc603975c3f8d Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Sun, 21 Jun 2015 03:03:50 +0200 Subject: [PATCH 3/8] Added tests, documentation and simplified code --- docs/changelog.rst | 1 + docs/guide/querying.rst | 6 ++-- mongoengine/queryset/transform.py | 50 +++++++++++++------------------ tests/queryset/geo.py | 31 ++++++++++++++++++- tests/queryset/queryset.py | 14 +++++++++ 5 files changed, 70 insertions(+), 32 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0dcd1e08..ac683750 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Changes in 0.9.X - DEV - Added `BaseQuerySet.aggregate_sum` and `BaseQuerySet.aggregate_average` methods. - Fix for delete with write_concern {'w': 0}. #1008 - Allow dynamic lookup for more than two parts. #882 +- Added support for min_distance on geo queries. #831 Changes in 0.9.0 ================ diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 1cde82cb..688fc9e9 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -146,9 +146,10 @@ The following were added in MongoEngine 0.8 for loc.objects(point__near=[40, 5]) loc.objects(point__near={"type": "Point", "coordinates": [40, 5]}) - You can also set the maximum distance in meters as well:: + You can also set the maximum and/or the minimum distance in meters as well:: loc.objects(point__near=[40, 5], point__max_distance=1000) + loc.objects(point__near=[40, 5], point__min_distance=100) The older 2D indexes are still supported with the :class:`~mongoengine.fields.GeoPointField`: @@ -168,7 +169,8 @@ The older 2D indexes are still supported with the * ``max_distance`` -- can be added to your location queries to set a maximum distance. - +* ``min_distance`` -- can be added to your location queries to set a minimum + distance. Querying lists -------------- diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 637e1b4b..91915703 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -6,7 +6,7 @@ from bson import SON from mongoengine.base.fields import UPDATE_OPERATORS from mongoengine.connection import get_connection from mongoengine.common import _import_class -from mongoengine.errors import InvalidQueryError +from mongoengine.errors import InvalidQueryError, LookUpError __all__ = ('query', 'update') @@ -44,8 +44,8 @@ def query(_doc_cls=None, _field_operation=False, **query): if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() - # if user escape field name by __ - if len(parts) > 1 and parts[-1] == "": + #if user escape field name by __ + if len(parts) > 1 and parts[-1]=="": parts.pop() negate = False @@ -126,32 +126,28 @@ def query(_doc_cls=None, _field_operation=False, **query): elif key in mongo_query: if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) - # $maxDistance needs to come last - convert to SON + # $max/minDistance needs to come last - convert to SON value_dict = mongo_query[key] - if ('$maxDistance' in value_dict and '$near' in value_dict): + if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and '$near' in value_dict: value_son = SON() - if isinstance(value_dict['$near'], dict): - for k, v in value_dict.iteritems(): - if k == '$maxDistance' or k == '$minDistance': - continue - value_son[k] = v - if (get_connection().max_wire_version <= 1): - value_son['$maxDistance'] = value_dict[ - '$maxDistance'] - else: - value_son['$near'] = SON(value_son['$near']) + for k, v in value_dict.iteritems(): + if k == '$maxDistance' or k == '$minDistance': + continue + value_son[k] = v + if isinstance(value_dict['$near'], dict) and\ + get_connection().max_wire_version > 1: + value_son['$near'] = SON(value_son['$near']) + if '$maxDistance' in value_dict: value_son['$near'][ '$maxDistance'] = value_dict['$maxDistance'] + if '$minDistance' in value_dict: + value_son['$near'][ + '$minDistance'] = value_dict['$minDistance'] else: - for k, v in value_dict.iteritems(): - if k == '$maxDistance' or k == '$minDistance': - continue - value_son[k] = v if '$maxDistance' in value_dict: value_son['$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: value_son['$minDistance'] = value_dict['$minDistance'] - mongo_query[key] = value_son else: # Store for manually merging later @@ -305,7 +301,11 @@ def update(_doc_cls=None, **update): def _geo_operator(field, op, value): """Helper to return the query for a given geo query""" - if field._geo_index == pymongo.GEO2D: + if op == "max_distance": + value = {'$maxDistance': value} + elif op == "min_distance": + value = {'$minDistance': value} + elif field._geo_index == pymongo.GEO2D: if op == "within_distance": value = {'$within': {'$center': value}} elif op == "within_spherical_distance": @@ -318,10 +318,6 @@ def _geo_operator(field, op, value): value = {'$nearSphere': value} elif op == 'within_box': value = {'$within': {'$box': value}} - elif op == "max_distance": - value = {'$maxDistance': value} - elif op == "min_distance": - value = {'$minDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented for a GeoPointField" % op) @@ -340,10 +336,6 @@ def _geo_operator(field, op, value): value = {"$geoIntersects": _infer_geometry(value)} elif op == "near": value = {'$near': _infer_geometry(value)} - elif op == "max_distance": - value = {'$maxDistance': value} - elif op == "min_distance": - value = {'$minDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented for a %s " % (op, field._name)) diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py index 12e96a04..889dac38 100644 --- a/tests/queryset/geo.py +++ b/tests/queryset/geo.py @@ -70,6 +70,11 @@ class GeoQueriesTest(unittest.TestCase): self.assertEqual(events.count(), 1) self.assertEqual(events[0], event2) + # find events at least 10 degrees away of san francisco + point = [-122.415579, 37.7566023] + events = Event.objects(location__near=point, location__min_distance=10) + self.assertEqual(events.count(), 2) + # find events within 10 degrees of san francisco point_and_distance = [[-122.415579, 37.7566023], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -171,7 +176,7 @@ class GeoQueriesTest(unittest.TestCase): # Same behavior for _within_spherical_distance points = Point.objects( - location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] + location__within_spherical_distance=[[-122, 37.5], 60 / earth_radius] ) self.assertEqual(points.count(), 2) @@ -186,6 +191,16 @@ class GeoQueriesTest(unittest.TestCase): self.assertEqual(points.count(), 2) + # Test query works with max_distance being farer from one point + points = Point.objects(location__near_sphere=[-122, 37.8], + location__min_distance=60 / earth_radius) + self.assertEqual(points.count(), 1) + + # Test query works with min_distance being farer from one point + points = Point.objects(location__near_sphere=[-122, 37.8], + location__min_distance=60 / earth_radius) + self.assertEqual(points.count(), 1) + # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) @@ -268,6 +283,20 @@ class GeoQueriesTest(unittest.TestCase): self.assertEqual(events.count(), 2) self.assertEqual(events[0], event3) + # ensure min_distance and max_distance combine well + events = Event.objects(location__near=[-87.67892, 41.9120459], + location__min_distance=1000, + location__max_distance=10000).order_by("-date") + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event3) + + # ensure ordering is respected by "near" + events = Event.objects(location__near=[-87.67892, 41.9120459], + # location__min_distance=10000 + location__min_distance=10000).order_by("-date") + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + # check that within_box works box = [(-125.0, 35.0), (-100.0, 40.0)] events = Event.objects(location__geo_within_box=box) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index aaa14ca0..4f00e1c6 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -4763,5 +4763,19 @@ class QuerySetTest(unittest.TestCase): for p in Person.objects(): self.assertEqual(p.name, 'a') + def test_last_field_name_like_operator(self): + class EmbeddedItem(EmbeddedDocument): + type = StringField() + + class Doc(Document): + item = EmbeddedDocumentField(EmbeddedItem) + + Doc.drop_collection() + + doc = Doc(item=EmbeddedItem(type="axe")) + doc.save() + + self.assertEqual(1, Doc.objects(item__type__="axe").count()) + if __name__ == '__main__': unittest.main() From d96fcdb35cfcf083997c07a8f92afa022abf95df Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Sun, 21 Jun 2015 03:24:05 +0200 Subject: [PATCH 4/8] Fixed problem of ordering when using near_sphere operator --- mongoengine/queryset/transform.py | 3 ++- tests/queryset/geo.py | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 91915703..7ede8212 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -128,7 +128,8 @@ def query(_doc_cls=None, _field_operation=False, **query): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON value_dict = mongo_query[key] - if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and '$near' in value_dict: + if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \ + ('$near' in value_dict or '$nearSphere' in value_dict): value_son = SON() for k, v in value_dict.iteritems(): if k == '$maxDistance' or k == '$minDistance': diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py index 889dac38..4a800114 100644 --- a/tests/queryset/geo.py +++ b/tests/queryset/geo.py @@ -182,13 +182,6 @@ class GeoQueriesTest(unittest.TestCase): points = Point.objects(location__near_sphere=[-122, 37.5], location__max_distance=60 / earth_radius) - # This test is sometimes failing with Mongo internals non-sense. - # See https://travis-ci.org/MongoEngine/mongoengine/builds/58729101 - try: - points.count() - except OperationFailure: - raise SkipTest("Sometimes MongoDB ignores its capacities on maxDistance") - self.assertEqual(points.count(), 2) # Test query works with max_distance being farer from one point From 95165aa92f817f23b287925b02a1756a94085208 Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Sun, 21 Jun 2015 13:29:54 +0200 Subject: [PATCH 5/8] Logic and test adaptations for MongoDB < 3 --- mongoengine/queryset/transform.py | 24 ++++++++++++++---------- tests/queryset/geo.py | 7 ++++++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 7ede8212..dc1c2f58 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -135,16 +135,20 @@ def query(_doc_cls=None, _field_operation=False, **query): if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v - if isinstance(value_dict['$near'], dict) and\ - get_connection().max_wire_version > 1: - value_son['$near'] = SON(value_son['$near']) - if '$maxDistance' in value_dict: - value_son['$near'][ - '$maxDistance'] = value_dict['$maxDistance'] - if '$minDistance' in value_dict: - value_son['$near'][ - '$minDistance'] = value_dict['$minDistance'] - else: + # seems only required for 2.6= 1: + value_son[near_op] = SON(value_son[near_op]) + if '$maxDistance' in value_dict: + value_son[near_op][ + '$maxDistance'] = value_dict['$maxDistance'] + if '$minDistance' in value_dict: + value_son[near_op][ + '$minDistance'] = value_dict['$minDistance'] + near_embedded = True + if not near_embedded: if '$maxDistance' in value_dict: value_son['$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py index 4a800114..2ab99dd8 100644 --- a/tests/queryset/geo.py +++ b/tests/queryset/geo.py @@ -73,7 +73,12 @@ class GeoQueriesTest(unittest.TestCase): # find events at least 10 degrees away of san francisco point = [-122.415579, 37.7566023] events = Event.objects(location__near=point, location__min_distance=10) - self.assertEqual(events.count(), 2) + # The following real test passes on MongoDB 3 but minDistance seems + # buggy on older MongoDB versions + if get_connection().server_info()['versionArray'][0] > 2: + self.assertEqual(events.count(), 2) + else: + self.assertTrue(events.count() >= 2) # find events within 10 degrees of san francisco point_and_distance = [[-122.415579, 37.7566023], 10] From 40f6df7160c5f140ca40deed04d40386e7c0db27 Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Sun, 21 Jun 2015 13:41:41 +0200 Subject: [PATCH 6/8] Adapted one more test for MongoDB < 3 --- mongoengine/queryset/transform.py | 2 +- tests/queryset/geo.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index dc1c2f58..4ff38769 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -6,7 +6,7 @@ from bson import SON from mongoengine.base.fields import UPDATE_OPERATORS from mongoengine.connection import get_connection from mongoengine.common import _import_class -from mongoengine.errors import InvalidQueryError, LookUpError +from mongoengine.errors import InvalidQueryError __all__ = ('query', 'update') diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py index 2ab99dd8..9aac44f5 100644 --- a/tests/queryset/geo.py +++ b/tests/queryset/geo.py @@ -189,15 +189,23 @@ class GeoQueriesTest(unittest.TestCase): location__max_distance=60 / earth_radius) self.assertEqual(points.count(), 2) - # Test query works with max_distance being farer from one point + # Test query works with max_distance, being farer from one point points = Point.objects(location__near_sphere=[-122, 37.8], - location__min_distance=60 / earth_radius) + location__max_distance=60 / earth_radius) + close_point = points.first() self.assertEqual(points.count(), 1) - # Test query works with min_distance being farer from one point + # Test query works with min_distance, being farer from one point points = Point.objects(location__near_sphere=[-122, 37.8], location__min_distance=60 / earth_radius) - self.assertEqual(points.count(), 1) + # The following real test passes on MongoDB 3 but minDistance seems + # buggy on older MongoDB versions + if get_connection().server_info()['versionArray'][0] > 2: + self.assertEqual(points.count(), 1) + far_point = points.first() + self.assertNotEqual(close_point, far_point) + else: + self.assertTrue(points.count() >= 1) # Finds both points, but orders the north point first because it's # closer to the reference point to the north. From 9063b559c4683ff2607cf929b40e46d4f85205c6 Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Mon, 22 Jun 2015 16:40:50 +0200 Subject: [PATCH 7/8] Fix for PyMongo3+ --- mongoengine/queryset/transform.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 4ff38769..e7f5988c 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -7,6 +7,7 @@ from mongoengine.base.fields import UPDATE_OPERATORS from mongoengine.connection import get_connection from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError +from mongoengine.python_support import IS_PYMONGO_3 __all__ = ('query', 'update') @@ -135,11 +136,12 @@ def query(_doc_cls=None, _field_operation=False, **query): if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v - # seems only required for 2.6== 2.6, may fail when combining + # PyMongo 3+ and MongoDB < 2.6 near_embedded = False for near_op in ('$near', '$nearSphere'): - if isinstance(value_dict.get(near_op), dict) and \ - get_connection().max_wire_version > 1: + if isinstance(value_dict.get(near_op), dict) and ( + IS_PYMONGO_3 or get_connection().max_wire_version > 1): value_son[near_op] = SON(value_son[near_op]) if '$maxDistance' in value_dict: value_son[near_op][ From 5c807f3dc86264cd80b63d74974a387c473d27f7 Mon Sep 17 00:00:00 2001 From: Matthieu Rigal Date: Mon, 22 Jun 2015 16:41:36 +0200 Subject: [PATCH 8/8] Various test adjustments to improve stability independantly of execution order --- tests/test_connection.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 4a02696a..e9477b79 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -54,11 +54,10 @@ class ConnectionTest(unittest.TestCase): def test_sharing_connections(self): """Ensure that connections are shared when the connection settings are exactly the same """ - connect('mongoenginetest', alias='testdb1') - + connect('mongoenginetests', alias='testdb1') expected_connection = get_connection('testdb1') - connect('mongoenginetest', alias='testdb2') + connect('mongoenginetests', alias='testdb2') actual_connection = get_connection('testdb2') # Handle PyMongo 3+ Async Connection @@ -96,8 +95,7 @@ class ConnectionTest(unittest.TestCase): c.mongoenginetest.system.users.remove({}) def test_connect_uri_without_db(self): - """Ensure that the connect() method works properly with uri's - without database_name + """Ensure connect() method works properly with uri's without database_name """ c = connect(db='mongoenginetest', alias='admin') c.admin.system.users.remove({}) @@ -130,28 +128,27 @@ class ConnectionTest(unittest.TestCase): # Create users c = connect('mongoenginetest') c.admin.system.users.remove({}) - c.admin.add_user('username', 'password') + c.admin.add_user('username2', 'password') # Authentication fails without "authSource" if IS_PYMONGO_3: - test_conn = connect('mongoenginetest', alias='test2', - host='mongodb://username:password@localhost/mongoenginetest') + test_conn = connect('mongoenginetest', alias='test1', + host='mongodb://username2:password@localhost/mongoenginetest') self.assertRaises(OperationFailure, test_conn.server_info) else: self.assertRaises( ConnectionError, connect, 'mongoenginetest', alias='test1', - host='mongodb://username:password@localhost/mongoenginetest' + host='mongodb://username2:password@localhost/mongoenginetest' ) self.assertRaises(ConnectionError, get_db, 'test1') # Authentication succeeds with "authSource" test_conn2 = connect( 'mongoenginetest', alias='test2', - host=('mongodb://username:password@localhost/' + host=('mongodb://username2:password@localhost/' 'mongoenginetest?authSource=admin') ) # This will fail starting from MongoDB 2.6+ - # test_conn2.server_info() db = get_db('test2') self.assertTrue(isinstance(db, pymongo.database.Database)) self.assertEqual(db.name, 'mongoenginetest')