From 8f288fe45875050ac39985dcf9bcc304b6c1f15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 29 Sep 2019 22:48:46 +0200 Subject: [PATCH 01/79] add mongodb 4.0 to travis and docs --- .travis.yml | 10 ++++++---- README.rst | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54a6befd..21321841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ # with a very large number of jobs, hence we only test a subset of all the # combinations: # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, and PyPy. +# tested against Python v2.7, v3.5, v3.6, v3.7, PyPy and PyPy3. # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo # combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. @@ -30,15 +30,16 @@ dist: xenial env: global: + - MONGODB_4_0=4.0.12 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 + - PYMONGO_3_9=3.9 - PYMONGO_3_6=3.6 - PYMONGO_3_4=3.4 matrix: - - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_6} + - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9} matrix: - # Finish the build as soon as one job fails fast_finish: true @@ -47,7 +48,8 @@ matrix: env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - + - python: 3.7 + env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_9} install: # Install Mongo diff --git a/README.rst b/README.rst index 679980f8..82b32893 100644 --- a/README.rst +++ b/README.rst @@ -26,10 +26,10 @@ an `API reference `_. Supported MongoDB Versions ========================== -MongoEngine is currently tested against MongoDB v3.4 and v3.6. Future versions +MongoEngine is currently tested against MongoDB v3.4, v3.6 and v4.0. Future versions should be supported as well, but aren't actively tested at the moment. Make sure to open an issue or submit a pull request if you experience any problems -with MongoDB version > 3.6. +with MongoDB version > 4.0. Installation ============ From b61c8cd104975b5fb47681387abc443c8c8430b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 1 Oct 2019 22:17:19 +0200 Subject: [PATCH 02/79] fix tox envs --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a1ae8444..a7921c61 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36} +envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39} [testenv] commands = @@ -8,5 +8,6 @@ deps = nose mg34: pymongo>=3.4,<3.5 mg36: pymongo>=3.6,<3.7 + mg39: pymongo>=3.9,<4.0 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs From 17151f67c2cdd98e97f3f4f539af167b84927ee2 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:53:39 +0100 Subject: [PATCH 03/79] Reformat repo with pre-commit hooks --- docs/django.rst | 4 ++-- docs/faq.rst | 1 - docs/guide/connecting.rst | 2 +- docs/guide/mongomock.rst | 4 ++-- docs/guide/signals.rst | 4 ++-- docs/guide/text-indexes.rst | 6 +++--- docs/index.rst | 1 - mongoengine/queryset/base.py | 4 +++- python-mongoengine.spec | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index b8a52165..d43a205e 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -13,7 +13,7 @@ Help Wanted! The MongoEngine team is looking for help contributing and maintaining a new Django extension for MongoEngine! If you have Django experience and would like -to help contribute to the project, please get in touch on the -`mailing list `_ or by +to help contribute to the project, please get in touch on the +`mailing list `_ or by simply contributing on `GitHub `_. diff --git a/docs/faq.rst b/docs/faq.rst index 27cd6937..49c73023 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -10,4 +10,3 @@ If this is a requirement for your project, check the alternative: `uMongo`_ and .. _uMongo: https://umongo.readthedocs.io/ .. _MotorEngine: https://motorengine.readthedocs.io/ - diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index aac13902..ac2146a6 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -86,7 +86,7 @@ using 3 different databases to store data:: connect(alias='user-db-alias', db='user-db') connect(alias='book-db-alias', db='book-db') connect(alias='users-books-db-alias', db='users-books-db') - + class User(Document): name = StringField() diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst index d70ee6a6..9f199ce5 100644 --- a/docs/guide/mongomock.rst +++ b/docs/guide/mongomock.rst @@ -2,10 +2,10 @@ Use mongomock for testing ============================== -`mongomock `_ is a package to do just +`mongomock `_ is a package to do just what the name implies, mocking a mongo database. -To use with mongoengine, simply specify mongomock when connecting with +To use with mongoengine, simply specify mongomock when connecting with mongoengine: .. code-block:: python diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 06bccb3b..e5214610 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -44,8 +44,8 @@ Available signals include: `post_save` Called within :meth:`~mongoengine.Document.save` after most actions - (validation, insert/update, and cascades, but not clearing dirty flags) have - completed successfully. Passed the additional boolean keyword argument + (validation, insert/update, and cascades, but not clearing dirty flags) have + completed successfully. Passed the additional boolean keyword argument `created` to indicate if the save was an insert or an update. `pre_delete` diff --git a/docs/guide/text-indexes.rst b/docs/guide/text-indexes.rst index 92a4471a..a5eaf7d8 100644 --- a/docs/guide/text-indexes.rst +++ b/docs/guide/text-indexes.rst @@ -8,7 +8,7 @@ After MongoDB 2.4 version, supports search documents by text indexes. Defining a Document with text index =================================== Use the *$* prefix to set a text index, Look the declaration:: - + class News(Document): title = StringField() content = StringField() @@ -35,10 +35,10 @@ Saving a document:: content="Various improvements").save() Next, start a text search using :attr:`QuerySet.search_text` method:: - + document = News.objects.search_text('testing').first() document.title # may be: "Using mongodb text search" - + document = News.objects.search_text('released').first() document.title # may be: "MongoEngine 0.9 released" diff --git a/docs/index.rst b/docs/index.rst index 662968d4..15f3c590 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -91,4 +91,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a648391e..a09cbf99 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1193,7 +1193,9 @@ class BaseQuerySet(object): validate_read_preference("read_preference", read_preference) queryset = self.clone() queryset._read_preference = read_preference - queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_preference + queryset._cursor_obj = ( + None + ) # we need to re-create the cursor object whenever we apply read_preference return queryset def scalar(self, *fields): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index eddb488d..635c779f 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog -* See: http://docs.mongoengine.org/en/latest/changelog.html \ No newline at end of file +* See: http://docs.mongoengine.org/en/latest/changelog.html From 5b9f2bac87cafab072e224f9f5c0caa123d26c8f Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:54:20 +0100 Subject: [PATCH 04/79] Add pre-commit --- .pre-commit-config.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..cac25e41 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: https://github.com/ambv/black + rev: 19.3b0 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.2.3 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: flake8 From 648b28876d152be2a6e1f28b79ddb4f82e05e78b Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 00:55:16 +0100 Subject: [PATCH 05/79] Rename requirements-lint.txt to requirements-dev.txt --- requirements-lint.txt => requirements-dev.txt | 4 ++++ 1 file changed, 4 insertions(+) rename requirements-lint.txt => requirements-dev.txt (51%) diff --git a/requirements-lint.txt b/requirements-dev.txt similarity index 51% rename from requirements-lint.txt rename to requirements-dev.txt index 9dc6123b..e57131c5 100644 --- a/requirements-lint.txt +++ b/requirements-dev.txt @@ -1,3 +1,7 @@ black flake8 flake8-import-order +pre-commit +pytest +ipdb +ipython From abc159b7b95380d0da6f9bc7cef427a31f3139e6 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 01:10:54 +0100 Subject: [PATCH 06/79] Update Contributing, changelog --- CONTRIBUTING.rst | 24 ++++++++++++++++++++++-- docs/changelog.rst | 2 ++ requirements-dev.txt | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 56bae31f..b571acf1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -31,8 +31,28 @@ build. You should ensure that your code is properly converted by Style Guide ----------- -MongoEngine uses `black `_ for code -formatting. +MongoEngine uses various tools to maintain a common coding style. + +To install all development tools, simply run the following commands: + +.. code-block:: console + + $ python -m pip install -r requirements-dev.txt + + +You can install `pre-commit `_ into your git hooks, +to automatically check and fix any formatting issue before creating a +git commit. + +Simply run the following command: + +.. code-block:: console + + $ pre-commit install + +See the ``.pre-commit-config.yaml`` configuration file for more information +on how it works. + Testing ------- diff --git a/docs/changelog.rst b/docs/changelog.rst index 99081957..933d0231 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ Development - In bulk write insert, the detailed error message would raise in exception. - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 +- Added pre-commit +- Renamed requirements-lint.txt to requirements-dev.txt Changes in 0.18.2 ================= diff --git a/requirements-dev.txt b/requirements-dev.txt index e57131c5..ee788e7a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,4 @@ pre-commit pytest ipdb ipython +tox From 37ffeafeff79e615ca0aa3140b5db638a1cd252b Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 01:13:05 +0100 Subject: [PATCH 07/79] Replace 'pip' with 'python -m pip install' in docs --- README.rst | 6 +++--- docs/guide/installing.rst | 2 +- docs/index.rst | 2 +- docs/tutorial.rst | 2 +- docs/upgrade.rst | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 853d8fbe..bae7c52c 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ with MongoDB version > 3.6. Installation ============ We recommend the use of `virtualenv `_ and of -`pip `_. You can then use ``pip install -U mongoengine``. +`pip `_. You can then use ``python -m pip install -U mongoengine``. You may also have `setuptools `_ and thus you can use ``easy_install -U mongoengine``. Another option is `pipenv `_. You can then use ``pipenv install mongoengine`` @@ -44,7 +44,7 @@ run ``python setup.py install``. Dependencies ============ -All of the dependencies can easily be installed via `pip `_. +All of the dependencies can easily be installed via `python -m pip `_. At the very least, you'll need these two packages to use MongoEngine: - pymongo>=3.4 @@ -126,7 +126,7 @@ installed in your environment and then: .. code-block:: shell # Install tox - $ pip install tox + $ python -m pip install tox # Run the test suites $ tox diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst index b89d48f0..2c962ad9 100644 --- a/docs/guide/installing.rst +++ b/docs/guide/installing.rst @@ -12,7 +12,7 @@ MongoEngine is available on PyPI, so you can use :program:`pip`: .. code-block:: console - $ pip install mongoengine + $ python -m pip install mongoengine Alternatively, if you don't have setuptools installed, `download it from PyPi `_ and run diff --git a/docs/index.rst b/docs/index.rst index 15f3c590..a42ff857 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ MongoDB. To install it, simply run .. code-block:: console - $ pip install -U mongoengine + $ python -m pip install -U mongoengine :doc:`tutorial` A quick tutorial building a tumblelog to get you up and running with diff --git a/docs/tutorial.rst b/docs/tutorial.rst index bcd0d17f..b7885c34 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -18,7 +18,7 @@ location --- running it locally will be easier, but if that is not an option then it may be run on a remote server. If you haven't installed MongoEngine, simply use pip to install it like so:: - $ pip install mongoengine + $ python -m pip install mongoengine Before we can start using MongoEngine, we need to tell it how to connect to our instance of :program:`mongod`. For this we use the :func:`~mongoengine.connect` diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 082dbadc..285bf24c 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -85,10 +85,10 @@ by default from now on. The 0.8.7 package on pypi was corrupted. If upgrading from 0.8.7 to 0.9.0 please follow: :: - pip uninstall pymongo - pip uninstall mongoengine - pip install pymongo==2.8 - pip install mongoengine + python -m pip uninstall pymongo + python -m pip uninstall mongoengine + python -m pip install pymongo==2.8 + python -m pip install mongoengine 0.8.7 ***** From d3d7f0e670b09fdd571fa6f2cf18ee36b793b6c7 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 01:18:29 +0100 Subject: [PATCH 08/79] Changelog --- CONTRIBUTING.rst | 2 +- docs/changelog.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b571acf1..27759f8f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -44,7 +44,7 @@ You can install `pre-commit `_ into your git hooks, to automatically check and fix any formatting issue before creating a git commit. -Simply run the following command: +To enable ``pre-commit`` simply run: .. code-block:: console diff --git a/docs/changelog.rst b/docs/changelog.rst index 933d0231..38f1a85e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,8 +25,8 @@ Development - In bulk write insert, the detailed error message would raise in exception. - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 -- Added pre-commit -- Renamed requirements-lint.txt to requirements-dev.txt +- Added pre-commit #2212 +- Renamed requirements-lint.txt to requirements-dev.txt #2212 Changes in 0.18.2 ================= From 90fecc56ddf53872f72774b75d27090acd6315ff Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Thu, 5 Dec 2019 01:33:35 +0100 Subject: [PATCH 09/79] Reformat with black --- mongoengine/queryset/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a09cbf99..a648391e 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1193,9 +1193,7 @@ class BaseQuerySet(object): validate_read_preference("read_preference", read_preference) queryset = self.clone() queryset._read_preference = read_preference - queryset._cursor_obj = ( - None - ) # we need to re-create the cursor object whenever we apply read_preference + queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_preference return queryset def scalar(self, *fields): From 2ca905b6e53794a4725d7519b8aca64f4d2ebd31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 14 Jun 2019 23:30:01 +0200 Subject: [PATCH 10/79] Finalize python2/3 codebase compatibility and get rid of 2to3 --- mongoengine/base/datastructures.py | 4 ++-- mongoengine/base/document.py | 4 ++-- mongoengine/fields.py | 8 +------- mongoengine/queryset/base.py | 2 +- setup.py | 4 ++-- tests/fields/test_embedded_document_field.py | 10 ++++++---- tests/fields/test_long_field.py | 6 +----- tests/fields/test_sequence_field.py | 14 +++++++------- tests/fields/test_url_field.py | 7 ++++--- tests/queryset/test_queryset.py | 8 ++++---- tests/test_signals.py | 2 +- 11 files changed, 31 insertions(+), 38 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index d1b5ae76..a9abf6ea 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -422,10 +422,10 @@ class StrictDict(object): return len(list(iteritems(self))) def __eq__(self, other): - return self.items() == other.items() + return list(self.items()) == list(other.items()) def __ne__(self, other): - return self.items() != other.items() + return list(self.items()) != list(other.items()) @classmethod def create(cls, allowed_keys): diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index ad691362..4948e672 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -92,7 +92,7 @@ class BaseDocument(object): # if so raise an Exception. if not self._dynamic and (self._meta.get("strict", True) or _created): _undefined_fields = set(values.keys()) - set( - self._fields.keys() + ["id", "pk", "_cls", "_text_score"] + list(self._fields.keys()) + ["id", "pk", "_cls", "_text_score"] ) if _undefined_fields: msg = ('The fields "{0}" do not exist on the document "{1}"').format( @@ -670,7 +670,7 @@ class BaseDocument(object): del set_data["_id"] # Determine if any changed items were actually unset. - for path, value in set_data.items(): + for path, value in list(set_data.items()): if value or isinstance( value, (numbers.Number, bool) ): # Account for 0 and True that are truthy diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f8f527a3..bef85dd7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,6 +8,7 @@ import uuid from operator import itemgetter from bson import Binary, DBRef, ObjectId, SON +from bson.int64 import Int64 import gridfs import pymongo from pymongo import ReturnDocument @@ -21,11 +22,6 @@ except ImportError: else: import dateutil.parser -try: - from bson.int64 import Int64 -except ImportError: - Int64 = long - from mongoengine.base import ( BaseDocument, @@ -53,8 +49,6 @@ except ImportError: ImageOps = None if six.PY3: - # Useless as long as 2to3 gets executed - # as it turns `long` into `int` blindly long = int diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a648391e..d3176050 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -989,7 +989,7 @@ class BaseQuerySet(object): .. versionchanged:: 0.5 - Added subfield support """ fields = {f: QueryFieldList.ONLY for f in fields} - self.only_fields = fields.keys() + self.only_fields = list(fields.keys()) return self.fields(True, **fields) def exclude(self, *fields): diff --git a/setup.py b/setup.py index ceb5afad..c6e99f53 100644 --- a/setup.py +++ b/setup.py @@ -118,8 +118,8 @@ extra_opts = { "Pillow>=2.0.0", ], } + if sys.version_info[0] == 3: - extra_opts["use_2to3"] = True if "test" in sys.argv: extra_opts["packages"] = find_packages() extra_opts["package_data"] = { @@ -143,7 +143,7 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=["pymongo>=3.4", "six>=1.10.0"], + install_requires=['pymongo>=3.4', 'six', 'future'], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/fields/test_embedded_document_field.py b/tests/fields/test_embedded_document_field.py index eeddac1e..b80f4d8c 100644 --- a/tests/fields/test_embedded_document_field.py +++ b/tests/fields/test_embedded_document_field.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from builtins import str + import pytest from mongoengine import ( @@ -75,7 +77,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase): # Test non exiting attribute with pytest.raises(InvalidQueryError) as exc_info: Person.objects(settings__notexist="bar").first() - assert unicode(exc_info.value) == u'Cannot resolve field "notexist"' + assert str(exc_info.value) == u'Cannot resolve field "notexist"' with pytest.raises(LookUpError): Person.objects.only("settings.notexist") @@ -111,7 +113,7 @@ class TestEmbeddedDocumentField(MongoDBTestCase): # Test non exiting attribute with pytest.raises(InvalidQueryError) as exc_info: assert Person.objects(settings__notexist="bar").first().id == p.id - assert unicode(exc_info.value) == u'Cannot resolve field "notexist"' + assert str(exc_info.value) == u'Cannot resolve field "notexist"' # Test existing attribute assert Person.objects(settings__base_foo="basefoo").first().id == p.id @@ -319,7 +321,7 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase): # Test non exiting attribute with pytest.raises(InvalidQueryError) as exc_info: Person.objects(settings__notexist="bar").first() - assert unicode(exc_info.value) == u'Cannot resolve field "notexist"' + assert str(exc_info.value) == u'Cannot resolve field "notexist"' with pytest.raises(LookUpError): Person.objects.only("settings.notexist") @@ -347,7 +349,7 @@ class TestGenericEmbeddedDocumentField(MongoDBTestCase): # Test non exiting attribute with pytest.raises(InvalidQueryError) as exc_info: assert Person.objects(settings__notexist="bar").first().id == p.id - assert unicode(exc_info.value) == u'Cannot resolve field "notexist"' + assert str(exc_info.value) == u'Cannot resolve field "notexist"' # Test existing attribute assert Person.objects(settings__base_foo="basefoo").first().id == p.id diff --git a/tests/fields/test_long_field.py b/tests/fields/test_long_field.py index da4f04c8..b39a714c 100644 --- a/tests/fields/test_long_field.py +++ b/tests/fields/test_long_field.py @@ -1,12 +1,8 @@ # -*- coding: utf-8 -*- import pytest +from bson.int64 import Int64 import six -try: - from bson.int64 import Int64 -except ImportError: - Int64 = long - from mongoengine import * from mongoengine.connection import get_db diff --git a/tests/fields/test_sequence_field.py b/tests/fields/test_sequence_field.py index aa83f710..f96f6b06 100644 --- a/tests/fields/test_sequence_field.py +++ b/tests/fields/test_sequence_field.py @@ -21,7 +21,7 @@ class TestSequenceField(MongoDBTestCase): assert c["next"] == 10 ids = [i.id for i in Person.objects] - assert ids == range(1, 11) + assert ids == list(range(1, 11)) c = self.db["mongoengine.counters"].find_one({"_id": "person.id"}) assert c["next"] == 10 @@ -76,7 +76,7 @@ class TestSequenceField(MongoDBTestCase): assert c["next"] == 10 ids = [i.id for i in Person.objects] - assert ids == range(1, 11) + assert ids == list(range(1, 11)) c = self.db["mongoengine.counters"].find_one({"_id": "jelly.id"}) assert c["next"] == 10 @@ -101,10 +101,10 @@ class TestSequenceField(MongoDBTestCase): assert c["next"] == 10 ids = [i.id for i in Person.objects] - assert ids == range(1, 11) + assert ids == list(range(1, 11)) counters = [i.counter for i in Person.objects] - assert counters == range(1, 11) + assert counters == list(range(1, 11)) c = self.db["mongoengine.counters"].find_one({"_id": "person.id"}) assert c["next"] == 10 @@ -166,10 +166,10 @@ class TestSequenceField(MongoDBTestCase): assert c["next"] == 10 ids = [i.id for i in Person.objects] - assert ids == range(1, 11) + assert ids == list(range(1, 11)) id = [i.id for i in Animal.objects] - assert id == range(1, 11) + assert id == list(range(1, 11)) c = self.db["mongoengine.counters"].find_one({"_id": "person.id"}) assert c["next"] == 10 @@ -193,7 +193,7 @@ class TestSequenceField(MongoDBTestCase): assert c["next"] == 10 ids = [i.id for i in Person.objects] - assert ids == map(str, range(1, 11)) + assert ids == [str(i) for i in range(1, 11)] c = self.db["mongoengine.counters"].find_one({"_id": "person.id"}) assert c["next"] == 10 diff --git a/tests/fields/test_url_field.py b/tests/fields/test_url_field.py index 948a4788..98f5d4fd 100644 --- a/tests/fields/test_url_field.py +++ b/tests/fields/test_url_field.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import pytest +from builtins import str + from mongoengine import * from tests.utils import MongoDBTestCase @@ -35,9 +37,8 @@ class TestURLField(MongoDBTestCase): with pytest.raises(ValidationError) as exc_info: link.validate() assert ( - unicode(exc_info.value) - == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])" - ) + str(exc_info.exception) + == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])") def test_url_scheme_validation(self): """Ensure that URLFields validate urls with specific schemes properly. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 7812ab66..49dab169 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -110,7 +110,7 @@ class TestQueryset(unittest.TestCase): # Filter people by age people = self.Person.objects(age=20) assert people.count() == 1 - person = people.next() + person = next(people) assert person == user_a assert person.name == "User A" assert person.age == 20 @@ -2768,7 +2768,7 @@ class TestQueryset(unittest.TestCase): ) # start a map/reduce - cursor.next() + next(cursor) results = Person.objects.map_reduce( map_f=map_person, @@ -4395,7 +4395,7 @@ class TestQueryset(unittest.TestCase): # Use a query to filter the people found to just person1 people = self.Person.objects(age=20).scalar("name") assert people.count() == 1 - person = people.next() + person = next(people) assert person == "User A" # Test limit @@ -5309,7 +5309,7 @@ class TestQueryset(unittest.TestCase): if not test: raise AssertionError("Cursor has data and returned False") - queryset.next() + next(queryset) if not queryset: raise AssertionError( "Cursor has data and it must returns True, even in the last item." diff --git a/tests/test_signals.py b/tests/test_signals.py index d79eaf75..13ea5a05 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -58,7 +58,7 @@ class TestSignal(unittest.TestCase): @classmethod def post_save(cls, sender, document, **kwargs): - dirty_keys = document._delta()[0].keys() + document._delta()[1].keys() + dirty_keys = list(document._delta()[0].keys()) + list(document._delta()[1].keys()) signal_output.append("post_save signal, %s" % document) signal_output.append("post_save dirty keys, %s" % dirty_keys) if kwargs.pop("created", False): From 009f9a2b14fb1bde3465ea702784c1381d617c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 15 Jun 2019 12:24:13 +0200 Subject: [PATCH 11/79] set dist as xenial to avoid relying on flaky travis default dist --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index cbf34cde..0a96e762 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ python: dist: xenial +dist: xenial + env: global: - MONGODB_3_4=3.4.17 From a3e432eb68db6be7944ad1cf96c0526d7a10d39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 15 Jun 2019 14:37:47 +0200 Subject: [PATCH 12/79] remove references to '2to3' in doc, travis, etc --- .landscape.yml | 9 ++------- .travis.yml | 4 ++-- CONTRIBUTING.rst | 18 +++++++++++++----- benchmarks/test_inserts.py | 7 +++++++ mongoengine/document.py | 4 ---- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.landscape.yml b/.landscape.yml index a27bbb03..4f13a5eb 100644 --- a/.landscape.yml +++ b/.landscape.yml @@ -5,17 +5,12 @@ pylint: options: additional-builtins: - # add xrange and long as valid built-ins. In Python 3, xrange is - # translated into range and long is translated into int via 2to3 (see - # "use_2to3" in setup.py). This should be removed when we drop Python - # 2 support (which probably won't happen any time soon). - - xrange + # add long as valid built-ins. - long pyflakes: disable: - # undefined variables are already covered by pylint (and exclude - # xrange & long) + # undefined variables are already covered by pylint (and exclude long) - F821 ignore-paths: diff --git a/.travis.yml b/.travis.yml index 0a96e762..ecd5163d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ install: before_script: - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork - - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi # Run flake8 for Python 2.7 only + - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then flake8 .; else echo "flake8 only runs on py37"; fi # Run flake8 for Python 3.7 only - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then black --check .; else echo "black only runs on py37"; fi # Run black for Python 3.7 only - mongo --eval 'db.version();' # Make sure mongo is awake @@ -84,7 +84,7 @@ script: # 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible # code in a separate dir and runs tests on that. after_success: -- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; else echo "coveralls only sent for py27"; fi +- coveralls --verbose notifications: irc: irc.freenode.org#mongoengine diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 56bae31f..a71c2cec 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,11 +22,19 @@ Supported Interpreters MongoEngine supports CPython 2.7 and newer. Language features not supported by all interpreters can not be used. -The codebase is written in python 2 so you must be using python 2 -when developing new features. Compatibility of the library with Python 3 -relies on the 2to3 package that gets executed as part of the installation -build. You should ensure that your code is properly converted by -`2to3 `_. +The codebase is written in a compatible manner for python 2 & 3 so it +is important that this is taken into account when it comes to discrepencies +between the 2 versions (check this https://python-future.org/compatible_idioms.html). +Travis runs run the tests against the different versions as a safety net. + +Python 2/3 compatibility +---------------------- + +The codebase is written in a compatible manner for python 2 & 3 so it +is important that this is taken into account when it comes to discrepencies +between the 2 versions (check this https://python-future.org/compatible_idioms.html). +Travis runs run the tests against the different versions as a safety net. + Style Guide ----------- diff --git a/benchmarks/test_inserts.py b/benchmarks/test_inserts.py index fd017bae..af6399f7 100644 --- a/benchmarks/test_inserts.py +++ b/benchmarks/test_inserts.py @@ -3,13 +3,17 @@ import timeit def main(): setup = """ +from builtins import range + from pymongo import MongoClient + connection = MongoClient() connection.drop_database('mongoengine_benchmark_test') """ stmt = """ from pymongo import MongoClient + connection = MongoClient() db = connection.mongoengine_benchmark_test @@ -55,7 +59,10 @@ myNoddys = noddy.find() print("{}s".format(t.timeit(1))) setup = """ +from builtins import range + from pymongo import MongoClient + connection = MongoClient() connection.drop_database('mongoengine_benchmark_test') connection.close() diff --git a/mongoengine/document.py b/mongoengine/document.py index 23968f17..fc6b9f16 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -71,7 +71,6 @@ class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): __slots__ = ("_instance",) - # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 my_metaclass = DocumentMetaclass @@ -156,7 +155,6 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): in the :attr:`meta` dictionary. """ - # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 my_metaclass = TopLevelDocumentMetaclass @@ -1045,7 +1043,6 @@ class DynamicDocument(six.with_metaclass(TopLevelDocumentMetaclass, Document)): There is one caveat on Dynamic Documents: undeclared fields cannot start with `_` """ - # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 my_metaclass = TopLevelDocumentMetaclass @@ -1069,7 +1066,6 @@ class DynamicEmbeddedDocument(six.with_metaclass(DocumentMetaclass, EmbeddedDocu information about dynamic documents. """ - # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 my_metaclass = DocumentMetaclass From 44b7f792fead6a3fa61dcb75a9a4d3c7ecdc90b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 16 Jun 2019 22:45:04 +0200 Subject: [PATCH 13/79] fix benchmarks prints --- benchmarks/test_basic_doc_ops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/test_basic_doc_ops.py b/benchmarks/test_basic_doc_ops.py index e840f97a..c74594fe 100644 --- a/benchmarks/test_basic_doc_ops.py +++ b/benchmarks/test_basic_doc_ops.py @@ -135,7 +135,6 @@ def test_big_doc(): % (timeit(create_and_delete_company, 10) * 10 ** 3) ) - if __name__ == "__main__": test_basic() print("-" * 100) From 7e0ba1b3359f494a750400042245662a58ae472c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 16 Jun 2019 23:09:46 +0200 Subject: [PATCH 14/79] clean remaining references to 2to3 --- .travis.yml | 3 --- CONTRIBUTING.rst | 4 ---- 2 files changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ecd5163d..de32e6bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,9 +80,6 @@ before_script: script: - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine" -# For now only submit coveralls for Python v2.7. Python v3.x currently shows -# 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible -# code in a separate dir and runs tests on that. after_success: - coveralls --verbose diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a71c2cec..445319e5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,10 +22,6 @@ Supported Interpreters MongoEngine supports CPython 2.7 and newer. Language features not supported by all interpreters can not be used. -The codebase is written in a compatible manner for python 2 & 3 so it -is important that this is taken into account when it comes to discrepencies -between the 2 versions (check this https://python-future.org/compatible_idioms.html). -Travis runs run the tests against the different versions as a safety net. Python 2/3 compatibility ---------------------- From 82af5e4a1920578fbd9dc36de8c1760311fb9837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 18 Jun 2019 22:55:51 +0200 Subject: [PATCH 15/79] fix small finding from review on py2py3 compat --- mongoengine/base/document.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 4948e672..4e4df92f 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -3,6 +3,7 @@ import numbers from functools import partial from bson import DBRef, ObjectId, SON, json_util +from future.utils import listitems import pymongo import six from six import iteritems @@ -670,7 +671,7 @@ class BaseDocument(object): del set_data["_id"] # Determine if any changed items were actually unset. - for path, value in list(set_data.items()): + for path, value in listitems(set_data): if value or isinstance( value, (numbers.Number, bool) ): # Account for 0 and True that are truthy From 64c0cace85ae528892409b9f34c343aff1bfbe60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 19 Jul 2019 21:50:47 +0200 Subject: [PATCH 16/79] fix from code review regarding py2py3 compat --- CONTRIBUTING.rst | 4 ++-- mongoengine/base/datastructures.py | 5 +++-- tests/fields/test_sequence_field.py | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 445319e5..b04ae968 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -28,8 +28,8 @@ Python 2/3 compatibility The codebase is written in a compatible manner for python 2 & 3 so it is important that this is taken into account when it comes to discrepencies -between the 2 versions (check this https://python-future.org/compatible_idioms.html). -Travis runs run the tests against the different versions as a safety net. +between the two versions (see https://python-future.org/compatible_idioms.html). +Travis runs the tests against different Python versions as a safety net. Style Guide diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index a9abf6ea..f8b4cd92 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,6 +1,7 @@ import weakref from bson import DBRef +from future.utils import listitems import six from six import iteritems @@ -422,10 +423,10 @@ class StrictDict(object): return len(list(iteritems(self))) def __eq__(self, other): - return list(self.items()) == list(other.items()) + return listitems(self) == listitems(other) def __ne__(self, other): - return list(self.items()) != list(other.items()) + return not(self == other) @classmethod def create(cls, allowed_keys): diff --git a/tests/fields/test_sequence_field.py b/tests/fields/test_sequence_field.py index f96f6b06..8e6615a1 100644 --- a/tests/fields/test_sequence_field.py +++ b/tests/fields/test_sequence_field.py @@ -267,12 +267,12 @@ class TestSequenceField(MongoDBTestCase): foo = Foo(name="Foo") foo.save() - assert not ( - "base.counter" in self.db["mongoengine.counters"].find().distinct("_id") + assert ( + "base.counter" not in self.db["mongoengine.counters"].find().distinct("_id") ) - assert ("foo.counter" and "bar.counter") in self.db[ - "mongoengine.counters" - ].find().distinct("_id") + existing_counters = self.db["mongoengine.counters"].find().distinct("_id") + assert "foo.counter" in existing_counters + assert "bar.counter" in existing_counters assert foo.counter == bar.counter assert foo._fields["counter"].owner_document == Foo assert bar._fields["counter"].owner_document == Bar From 45a7520fc3c9b22452d78ee2b5ae9c11f616b6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 21 Jul 2019 22:54:14 +0200 Subject: [PATCH 17/79] make use past.builtins.long --- mongoengine/fields.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bef85dd7..562cc906 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -10,6 +10,7 @@ from operator import itemgetter from bson import Binary, DBRef, ObjectId, SON from bson.int64 import Int64 import gridfs +from past.builtins import long import pymongo from pymongo import ReturnDocument import six @@ -48,9 +49,6 @@ except ImportError: Image = None ImageOps = None -if six.PY3: - long = int - __all__ = ( "StringField", From 6bc4e602bb5fef6c48ccc3233645eed729f2d9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 14 Dec 2019 00:05:48 +0100 Subject: [PATCH 18/79] additional fix --- tests/fields/test_url_field.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fields/test_url_field.py b/tests/fields/test_url_field.py index 98f5d4fd..477bced7 100644 --- a/tests/fields/test_url_field.py +++ b/tests/fields/test_url_field.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- -import pytest - from builtins import str -from mongoengine import * +import pytest +from mongoengine import * from tests.utils import MongoDBTestCase @@ -37,8 +36,9 @@ class TestURLField(MongoDBTestCase): with pytest.raises(ValidationError) as exc_info: link.validate() assert ( - str(exc_info.exception) - == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])") + str(exc_info.value) + == u"ValidationError (Link:None) (Invalid URL: http://\u043f\u0440\u0438\u0432\u0435\u0442.com: ['url'])" + ) def test_url_scheme_validation(self): """Ensure that URLFields validate urls with specific schemes properly. From 9166ba91d713924f91983cfbe959c4981e4f0711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 14 Dec 2019 15:39:02 +0100 Subject: [PATCH 19/79] fix minor styling issue --- benchmarks/test_basic_doc_ops.py | 1 + mongoengine/base/datastructures.py | 2 +- setup.py | 2 +- tests/fields/test_sequence_field.py | 4 ++-- tests/test_signals.py | 4 +++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/test_basic_doc_ops.py b/benchmarks/test_basic_doc_ops.py index c74594fe..e840f97a 100644 --- a/benchmarks/test_basic_doc_ops.py +++ b/benchmarks/test_basic_doc_ops.py @@ -135,6 +135,7 @@ def test_big_doc(): % (timeit(create_and_delete_company, 10) * 10 ** 3) ) + if __name__ == "__main__": test_basic() print("-" * 100) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index f8b4cd92..8c93f596 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -426,7 +426,7 @@ class StrictDict(object): return listitems(self) == listitems(other) def __ne__(self, other): - return not(self == other) + return not (self == other) @classmethod def create(cls, allowed_keys): diff --git a/setup.py b/setup.py index c6e99f53..19cd4be7 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=['pymongo>=3.4', 'six', 'future'], + install_requires=["pymongo>=3.4", "six", "future"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/fields/test_sequence_field.py b/tests/fields/test_sequence_field.py index 8e6615a1..81d648fd 100644 --- a/tests/fields/test_sequence_field.py +++ b/tests/fields/test_sequence_field.py @@ -267,8 +267,8 @@ class TestSequenceField(MongoDBTestCase): foo = Foo(name="Foo") foo.save() - assert ( - "base.counter" not in self.db["mongoengine.counters"].find().distinct("_id") + assert "base.counter" not in self.db["mongoengine.counters"].find().distinct( + "_id" ) existing_counters = self.db["mongoengine.counters"].find().distinct("_id") assert "foo.counter" in existing_counters diff --git a/tests/test_signals.py b/tests/test_signals.py index 13ea5a05..451e01ff 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -58,7 +58,9 @@ class TestSignal(unittest.TestCase): @classmethod def post_save(cls, sender, document, **kwargs): - dirty_keys = list(document._delta()[0].keys()) + list(document._delta()[1].keys()) + dirty_keys = list(document._delta()[0].keys()) + list( + document._delta()[1].keys() + ) signal_output.append("post_save signal, %s" % document) signal_output.append("post_save dirty keys, %s" % dirty_keys) if kwargs.pop("created", False): From d8c0631dabe7d900691ec9c25366c45f3112c50e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 14 Dec 2019 21:23:28 +0100 Subject: [PATCH 20/79] added entry in changelog for py2py3 compatibility --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bc01a403..bac5ee17 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- Codebase is now compatible with both Python2 and Python3 (no more relying on 2to3 during installation) #2087 - Documentation improvements: - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver. - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 From d73846213977b2fb1e382b38d86406bdf9a7e81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 11 Jan 2020 23:15:30 +0100 Subject: [PATCH 21/79] Fix bug introduced in -1.19 related to DictField validate failing without default connection --- docs/changelog.rst | 5 ++++ mongoengine/fields.py | 14 +++++----- mongoengine/mongodb_support.py | 2 +- tests/fields/test_dict_field.py | 48 ++++++++++++++++++++------------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b8e6ae56..2b532da9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,11 @@ Development =========== - (Fill this out as you fix issues and develop your features). +Changes in 0.19.2 +================= +- DictField validate failed without default connection (bug introduced in 0.19.0) #2239 + + Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7ec8c0f3..d502dba3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1088,14 +1088,12 @@ class DictField(ComplexBaseField): msg = "Invalid dictionary key - documents must have only string keys" self.error(msg) - curr_mongo_ver = get_mongodb_version() - - if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value): - self.error( - 'Invalid dictionary key name - keys may not contain "."' - ' or startswith "$" characters' - ) - elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value): + # Following condition applies to MongoDB >= 3.6 + # older Mongo has stricter constraints but + # it will be rejected upon insertion anyway + # Having a validation that depends on the MongoDB version + # is not straightforward as the field isn't aware of the connected Mongo + if key_starts_with_dollar(value): self.error( 'Invalid dictionary key name - keys may not startswith "$" characters' ) diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 5d437fef..522f064e 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -11,7 +11,7 @@ MONGODB_36 = (3, 6) def get_mongodb_version(): - """Return the version of the connected mongoDB (first 2 digits) + """Return the version of the default connected mongoDB (first 2 digits) :return: tuple(int, int) """ diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index 44e628f6..6850cd58 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import pytest +from bson import InvalidDocument from mongoengine import * from mongoengine.base import BaseDict @@ -19,22 +20,24 @@ class TestDictField(MongoDBTestCase): post = BlogPost(info=info).save() assert get_as_pymongo(post) == {"_id": post.id, "info": info} - def test_general_things(self): - """Ensure that dict types work as expected.""" + def test_validate_invalid_type(self): + class BlogPost(Document): + info = DictField() + BlogPost.drop_collection() + + invalid_infos = ["my post", ["test", "test"], {1: "test"}] + for invalid_info in invalid_infos: + with pytest.raises(ValidationError): + BlogPost(info=invalid_info).validate() + + def test_keys_with_dots_or_dollars(self): class BlogPost(Document): info = DictField() BlogPost.drop_collection() post = BlogPost() - post.info = "my post" - with pytest.raises(ValidationError): - post.validate() - - post.info = ["test", "test"] - with pytest.raises(ValidationError): - post.validate() post.info = {"$title": "test"} with pytest.raises(ValidationError): @@ -48,25 +51,34 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() - post.info = {1: "test"} - with pytest.raises(ValidationError): - post.validate() - post.info = {"nested": {"the.title": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + # MongoDB < 3.6 rejects dots + # To avoid checking the mongodb version from the DictField class + # we rely on MongoDB to reject the data during the save + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() post.info = {"dollar_and_dot": {"te$st.test": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() - post.info = {"title": "test"} + def test_general_things(self): + """Ensure that dict types work as expected.""" + + class BlogPost(Document): + info = DictField() + + BlogPost.drop_collection() + + post = BlogPost(info={"title": "test"}) post.save() post = BlogPost() From 18b68f1b8034457384daa5bdbd7d780055e21690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:29:18 +0100 Subject: [PATCH 22/79] update travis mongo 4.0 to latest 4.0.13 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f7880649..a7d6da1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ dist: xenial env: global: - - MONGODB_4_0=4.0.12 + - MONGODB_4_0=4.0.13 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 - PYMONGO_3_10=3.10 From e0565ddac5cd62f9fd05c69187b49a20bb644035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:31:28 +0100 Subject: [PATCH 23/79] update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b8e6ae56..d924a2c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- Add Mongo 4.0 to Travis Changes in 0.19.1 ================= From 605de59bd08d9d4f54f695c7f73a1458975d479f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 12 Jan 2020 21:37:32 +0100 Subject: [PATCH 24/79] improve travis + fix tox mg310 --- .travis.yml | 9 +++++---- tox.ini | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7d6da1a..62bbacb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,13 +31,14 @@ dist: xenial env: global: - - MONGODB_4_0=4.0.13 - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 - - PYMONGO_3_10=3.10 - - PYMONGO_3_9=3.9 - - PYMONGO_3_6=3.6 + - MONGODB_4_0=4.0.13 + - PYMONGO_3_4=3.4 + - PYMONGO_3_6=3.6 + - PYMONGO_3_9=3.9 + - PYMONGO_3_10=3.10 matrix: - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} diff --git a/tox.ini b/tox.ini index c3789b7d..396817ca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36, mg39} +envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310} [testenv] commands = From 72de6d67c7f91d9e1278d1503e87766c4e4708c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 3 Jan 2020 09:40:00 +0100 Subject: [PATCH 25/79] Bump dev status classifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5ba84e06..5c6451aa 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ version_line = list(filter(lambda l: l.startswith("VERSION"), open(init)))[0] VERSION = get_version(eval(version_line.split("=")[-1])) CLASSIFIERS = [ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", From 57db68dc040acae8070667a2840f945d3c40669a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 13 Jan 2020 23:34:43 +0100 Subject: [PATCH 26/79] update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..add37120 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- Bump development Status classifier to Production/Stable #2232 Changes in 0.19.1 ================= From 86e965f854763863de2d12e54b4416b4761df55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 30 Dec 2019 21:58:57 +0100 Subject: [PATCH 27/79] remove very old deprecated method --- docs/changelog.rst | 3 ++- mongoengine/queryset/base.py | 20 -------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..b96a85ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,8 @@ Changes in 0.19.1 - Requires Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 - +- BREAKING CHANGE: Removed Queryset._ensure_indexes and Queryset.ensure_indexes that were deprecated in 2013. + Document.ensure_indexes still exists Changes in 0.19.0 ================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..805a3d0a 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1958,23 +1958,3 @@ class BaseQuerySet(object): setattr(queryset, "_" + method_name, val) return queryset - - # Deprecated - def ensure_index(self, **kwargs): - """Deprecated use :func:`Document.ensure_index`""" - msg = ( - "Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ( - "Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() From 095217e7977b933c53697e9736a6a8bcb37830cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 13 Jan 2020 23:53:24 +0100 Subject: [PATCH 28/79] remove methods that were derecated in 2013... --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b96a85ed..5fe34f91 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,14 +7,15 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- BREAKING CHANGE: Removed ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes`` that were deprecated in 2013. + ``Document.ensure_indexes`` still exists and is the right method to use Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 -- BREAKING CHANGE: Removed Queryset._ensure_indexes and Queryset.ensure_indexes that were deprecated in 2013. - Document.ensure_indexes still exists + Changes in 0.19.0 ================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 From 38703acc299740ea52a73cca3e222bc050754223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Thu, 23 Jan 2020 23:33:23 +0100 Subject: [PATCH 29/79] fix complex datetime field invalid string set --- mongoengine/fields.py | 5 ++++- tests/fields/test_complex_datetime_field.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7ec8c0f3..3eff0325 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -684,7 +684,10 @@ class ComplexDateTimeField(StringField): super(ComplexDateTimeField, self).__set__(instance, value) value = instance._data[self.name] if value is not None: - instance._data[self.name] = self._convert_from_datetime(value) + if isinstance(value, datetime.datetime): + instance._data[self.name] = self._convert_from_datetime(value) + else: + instance._data[self.name] = value def validate(self, value): value = self.to_python(value) diff --git a/tests/fields/test_complex_datetime_field.py b/tests/fields/test_complex_datetime_field.py index f0a6b96e..699032cc 100644 --- a/tests/fields/test_complex_datetime_field.py +++ b/tests/fields/test_complex_datetime_field.py @@ -4,6 +4,8 @@ import itertools import math import re +import pytest + from mongoengine import * from tests.utils import MongoDBTestCase @@ -191,3 +193,18 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): fetched_log = Log.objects.with_id(log.id) assert fetched_log.timestamp >= NOW + + def test_setting_bad_value_does_not_raise_unless_validate_is_called(self): + # test regression of #2253 + + class Log(Document): + timestamp = ComplexDateTimeField() + + Log.drop_collection() + + log = Log(timestamp="garbage") + with pytest.raises(ValidationError): + log.validate() + + with pytest.raises(ValidationError): + log.save() From 2d6a4c4b9043d5cf4fe029870fceb9ecd47ce9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Thu, 23 Jan 2020 23:36:03 +0100 Subject: [PATCH 30/79] update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..11e6c063 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- Fix error when setting a string as a ComplexDateTimeField #2253 Changes in 0.19.1 ================= From 8e17e42e26d944d90bdd16bfacb8d07879768055 Mon Sep 17 00:00:00 2001 From: Agustin Date: Fri, 24 Jan 2020 13:11:07 -0300 Subject: [PATCH 31/79] Allow setting read_concern --- mongoengine/context_managers.py | 8 ++++++ mongoengine/queryset/base.py | 31 +++++++++++++++++---- tests/queryset/test_queryset.py | 49 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 1592ceef..0f6c8698 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -257,3 +257,11 @@ def set_write_concern(collection, write_concerns): combined_concerns = dict(collection.write_concern.document.items()) combined_concerns.update(write_concerns) yield collection.with_options(write_concern=WriteConcern(**combined_concerns)) + + +@contextmanager +def set_read_write_concern(collection, write_concerns, read_concern): + combined_write_concerns = dict(collection.write_concern.document.items()) + combined_write_concerns.update(write_concerns) + + yield collection.with_options(write_concern=WriteConcern(**combined_write_concerns), read_concern=read_concern) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..0b76235c 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -11,6 +11,7 @@ import pymongo import pymongo.errors from pymongo.collection import ReturnDocument from pymongo.common import validate_read_preference +from pymongo.read_concern import ReadConcern import six from six import iteritems @@ -18,7 +19,7 @@ from mongoengine import signals from mongoengine.base import get_document from mongoengine.common import _import_class from mongoengine.connection import get_db -from mongoengine.context_managers import set_write_concern, switch_db +from mongoengine.context_managers import set_write_concern, set_read_write_concern, switch_db from mongoengine.errors import ( BulkWriteError, InvalidQueryError, @@ -62,6 +63,7 @@ class BaseQuerySet(object): self._timeout = True self._slave_okay = False self._read_preference = None + self._read_concern = None self._iter = False self._scalar = [] self._none = False @@ -490,7 +492,7 @@ class BaseQuerySet(object): return result.deleted_count def update( - self, upsert=False, multi=True, write_concern=None, full_result=False, **update + self, upsert=False, multi=True, write_concern=None, read_concern=None, full_result=False, **update ): """Perform an atomic update on the fields matched by the query. @@ -502,6 +504,7 @@ class BaseQuerySet(object): ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. + :param read_concern: Override the read concern for the operation :param full_result: Return the associated ``pymongo.UpdateResult`` rather than just the number updated items :param update: Django-style update keyword arguments @@ -528,7 +531,7 @@ class BaseQuerySet(object): else: update["$set"] = {"_cls": queryset._document._class_name} try: - with set_write_concern(queryset._collection, write_concern) as collection: + with set_read_write_concern(queryset._collection, write_concern, read_concern) as collection: update_func = collection.update_one if multi: update_func = collection.update_many @@ -545,7 +548,7 @@ class BaseQuerySet(object): raise OperationError(message) raise OperationError(u"Update failed (%s)" % six.text_type(err)) - def upsert_one(self, write_concern=None, **update): + def upsert_one(self, write_concern=None, read_concern=None, **update): """Overwrite or add the first document matched by the query. :param write_concern: Extra keyword arguments are passed down which @@ -554,6 +557,7 @@ class BaseQuerySet(object): ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. + :param read_concern: Override the read concern for the operation :param update: Django-style update keyword arguments :returns the new or overwritten document @@ -565,6 +569,7 @@ class BaseQuerySet(object): multi=False, upsert=True, write_concern=write_concern, + read_concern=read_concern, full_result=True, **update ) @@ -1196,6 +1201,20 @@ class BaseQuerySet(object): queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_preference return queryset + def read_concern(self, read_concern): + """Change the read_concern when querying. + + :param read_concern: override ReplicaSetConnection-level + preference. + """ + if read_concern is not None and not isinstance(read_concern, ReadConcern): + raise TypeError("%r is not a read concern." % (read_concern,)) + + queryset = self.clone() + queryset._read_concern = read_concern + queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_concern + return queryset + def scalar(self, *fields): """Instead of returning Document instances, return either a specific value or a tuple of values in order. @@ -1642,9 +1661,9 @@ class BaseQuerySet(object): # XXX In PyMongo 3+, we define the read preference on a collection # level, not a cursor level. Thus, we need to get a cloned collection # object using `with_options` first. - if self._read_preference is not None: + if self._read_preference is not None or self._read_concern is not None: self._cursor_obj = self._collection.with_options( - read_preference=self._read_preference + read_preference=self._read_preference, read_concern=self._read_concern ).find(self._query, **self._cursor_args) else: self._cursor_obj = self._collection.find(self._query, **self._cursor_args) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index b30350e6..c238752d 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -7,6 +7,7 @@ from decimal import Decimal from bson import DBRef, ObjectId import pymongo +from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference from pymongo.results import UpdateResult import pytest @@ -4658,6 +4659,54 @@ class TestQueryset(unittest.TestCase): ) assert_read_pref(bars, ReadPreference.SECONDARY_PREFERRED) + def test_read_concern(self): + class Bar(Document): + txt = StringField() + + meta = {"indexes": ["txt"]} + + Bar.drop_collection() + bar = Bar.objects.create(txt="xyz") + + bars = list(Bar.objects.read_concern(None)) + assert bars == [bar] + + bars = Bar.objects.read_concern(ReadConcern(level='local')) + assert bars._read_concern == ReadConcern(level='local') + assert ( + bars._cursor.collection.read_concern + == ReadConcern(level='local') + ) + + # Make sure that `.read_concern(...)` does accept string values. + with pytest.raises(TypeError): + Bar.objects.read_concern('local') + + def assert_read_concern(qs, expected_read_concern): + assert qs._read_concern == expected_read_concern + assert qs._cursor.collection.read_concern == expected_read_concern + + # Make sure read concern is respected after a `.skip(...)`. + bars = Bar.objects.skip(1).read_concern(ReadConcern('majority')) + assert_read_concern(bars, ReadConcern('majority')) + + # Make sure read concern is respected after a `.limit(...)`. + bars = Bar.objects.limit(1).read_concern(ReadConcern('majority')) + assert_read_concern(bars, ReadConcern('majority')) + + # Make sure read concern is respected after an `.order_by(...)`. + bars = Bar.objects.order_by("txt").read_concern( + ReadConcern('majority') + ) + assert_read_concern(bars, ReadConcern('majority')) + + # Make sure read concern is respected after a `.hint(...)`. + bars = Bar.objects.hint([("txt", 1)]).read_concern( + ReadConcern('majority') + ) + assert_read_concern(bars, ReadConcern('majority')) + + def test_json_simple(self): class Embedded(EmbeddedDocument): string = StringField() From 450658d7ac2b26be7849f8f0599fff06bb7e6ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 4 Feb 2020 22:51:02 +0100 Subject: [PATCH 32/79] fix indirect library version that dropped python2 support recently --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5ba84e06..5cba5d9e 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,10 @@ CLASSIFIERS = [ "Topic :: Software Development :: Libraries :: Python Modules", ] +PYTHON_VERSION = sys.version_info[0] +PY3 = PYTHON_VERSION == 3 +PY2 = PYTHON_VERSION == 2 + extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), "tests_require": [ @@ -116,9 +120,10 @@ extra_opts = { "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support + "zipp<2.0.0", # (dependency of pytest) dropped python2 support ], } -if sys.version_info[0] == 3: +if PY3: extra_opts["use_2to3"] = True if "test" in sys.argv: extra_opts["packages"] = find_packages() From 4bca3de42f08a5ca4fdd0b4021d6a54f25e24003 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Fri, 14 Feb 2020 16:43:07 -0300 Subject: [PATCH 33/79] Add support for the elemMatch projection operator. Add basic tests to the fields queryset method. --- mongoengine/queryset/base.py | 2 +- tests/queryset/test_queryset.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..710259df 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1037,7 +1037,7 @@ class BaseQuerySet(object): """ # Check for an operator and transform to mongo-style if there is - operators = ["slice"] + operators = ["slice", "elemMatch"] cleaned_fields = [] for key, value in kwargs.items(): parts = key.split("__") diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index b30350e6..5ebd545f 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4476,6 +4476,57 @@ class TestQueryset(unittest.TestCase): expected = "[u'A1', u'A2']" assert expected == "%s" % sorted(names) + def test_fields(self): + class Bar(EmbeddedDocument): + v = StringField() + z = StringField() + + + class Foo(Document): + x = StringField() + y = IntField() + items = EmbeddedDocumentListField(Bar) + + + Foo.drop_collection() + + Foo(x='foo1', y=1).save() + Foo(x='foo2', y=2, items=[]).save() + Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() + Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() + Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + + foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + + assert all(o.x is not None for o in foos_with_x) + + foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + + assert all(o.y is None for o in foos_with_x) + + foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + + assert foos_with_sliced_items[0].items == [] + assert foos_with_sliced_items[1].items == [] + assert len(foos_with_sliced_items[2].items) == 1 + assert foos_with_sliced_items[2].items[0].z == 'a' + assert len(foos_with_sliced_items[3].items) == 1 + assert foos_with_sliced_items[3].items[0].z == 'a' + assert len(foos_with_sliced_items[4].items) == 1 + assert foos_with_sliced_items[4].items[0].z == 'b' + + foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + + assert foos_with_elem_match_items[0].items == [] + assert foos_with_elem_match_items[1].items == [] + assert foos_with_elem_match_items[2].items == [] + assert len(foos_with_elem_match_items[3].items) == 1 + assert foos_with_elem_match_items[3].items[0].z == 'b' + assert foos_with_elem_match_items[3].items[0].v == 'W' + assert len(foos_with_elem_match_items[4].items) == 1 + assert foos_with_elem_match_items[4].items[0].z == 'b' + + def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() From 81f9b351b3838fe27924937e706300805e9eb82b Mon Sep 17 00:00:00 2001 From: Leonardo Domingues Date: Fri, 21 Feb 2020 19:14:34 -0300 Subject: [PATCH 34/79] Add return info in the save function docstring --- AUTHORS | 1 + mongoengine/document.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b9a81c63..7d3000ce 100644 --- a/AUTHORS +++ b/AUTHORS @@ -255,3 +255,4 @@ that much better: * Filip Kucharczyk (https://github.com/Pacu2) * Eric Timmons (https://github.com/daewok) * Matthew Simpson (https://github.com/mcsimps2) + * Leonardo Domingues (https://github.com/leodmgs) diff --git a/mongoengine/document.py b/mongoengine/document.py index 23968f17..5e812510 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -332,7 +332,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): ): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be - created. + created. Returns the saved object instance. :param force_insert: only try to create a new document, don't allow updates of existing documents. From cfb4943986189a22ae75fd4f378f38f7ec0a77bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 2 Mar 2020 22:49:21 +0100 Subject: [PATCH 35/79] reformat with black --- tests/queryset/test_queryset.py | 53 ++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 5ebd545f..f6d1a916 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4481,51 +4481,68 @@ class TestQueryset(unittest.TestCase): v = StringField() z = StringField() - class Foo(Document): x = StringField() y = IntField() items = EmbeddedDocumentListField(Bar) - Foo.drop_collection() - Foo(x='foo1', y=1).save() - Foo(x='foo2', y=2, items=[]).save() - Foo(x='foo3', y=3, items=[Bar(z='a', v='V')]).save() - Foo(x='foo4', y=4, items=[Bar(z='a', v='V'), Bar(z='b', v='W'), Bar(z='b', v='X'), Bar(z='c', v='V')]).save() - Foo(x='foo5', y=5, items=[Bar(z='b', v='X'), Bar(z='c', v='V'), Bar(z='d', v='V'), Bar(z='e', v='V')]).save() + Foo(x="foo1", y=1).save() + Foo(x="foo2", y=2, items=[]).save() + Foo(x="foo3", y=3, items=[Bar(z="a", v="V")]).save() + Foo( + x="foo4", + y=4, + items=[ + Bar(z="a", v="V"), + Bar(z="b", v="W"), + Bar(z="b", v="X"), + Bar(z="c", v="V"), + ], + ).save() + Foo( + x="foo5", + y=5, + items=[ + Bar(z="b", v="X"), + Bar(z="c", v="V"), + Bar(z="d", v="V"), + Bar(z="e", v="V"), + ], + ).save() - foos_with_x = list(Foo.objects.order_by('y').fields(x=1)) + foos_with_x = list(Foo.objects.order_by("y").fields(x=1)) assert all(o.x is not None for o in foos_with_x) - foos_without_y = list(Foo.objects.order_by('y').fields(y=0)) + foos_without_y = list(Foo.objects.order_by("y").fields(y=0)) assert all(o.y is None for o in foos_with_x) - foos_with_sliced_items = list(Foo.objects.order_by('y').fields(slice__items=1)) + foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1)) assert foos_with_sliced_items[0].items == [] assert foos_with_sliced_items[1].items == [] assert len(foos_with_sliced_items[2].items) == 1 - assert foos_with_sliced_items[2].items[0].z == 'a' + assert foos_with_sliced_items[2].items[0].z == "a" assert len(foos_with_sliced_items[3].items) == 1 - assert foos_with_sliced_items[3].items[0].z == 'a' + assert foos_with_sliced_items[3].items[0].z == "a" assert len(foos_with_sliced_items[4].items) == 1 - assert foos_with_sliced_items[4].items[0].z == 'b' + assert foos_with_sliced_items[4].items[0].z == "b" - foos_with_elem_match_items = list(Foo.objects.order_by('y').fields(elemMatch__items={'z': 'b'})) + foos_with_elem_match_items = list( + Foo.objects.order_by("y").fields(elemMatch__items={"z": "b"}) + ) assert foos_with_elem_match_items[0].items == [] assert foos_with_elem_match_items[1].items == [] assert foos_with_elem_match_items[2].items == [] assert len(foos_with_elem_match_items[3].items) == 1 - assert foos_with_elem_match_items[3].items[0].z == 'b' - assert foos_with_elem_match_items[3].items[0].v == 'W' + assert foos_with_elem_match_items[3].items[0].z == "b" + assert foos_with_elem_match_items[3].items[0].v == "W" assert len(foos_with_elem_match_items[4].items) == 1 - assert foos_with_elem_match_items[4].items[0].z == 'b' - + assert foos_with_elem_match_items[4].items[0].z == "b" def test_elem_match(self): class Foo(EmbeddedDocument): From d287f480e5016090384650515c9342c767c4bbb7 Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Tue, 4 Feb 2020 12:35:03 +0100 Subject: [PATCH 36/79] Fix for combining raw and regular filters --- mongoengine/queryset/transform.py | 4 ++-- tests/queryset/test_transform.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 0b73e99b..659a97e2 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs): key = ".".join(parts) - if op is None or key not in mongo_query: + if key not in mongo_query: mongo_query[key] = value - elif key in mongo_query: + else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 3898809e..8d6c2d06 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -24,6 +24,12 @@ class TestTransform(unittest.TestCase): } assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}} assert transform.query(name__exists=True) == {"name": {"$exists": True}} + assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == { + "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}] + } + assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == { + "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}] + } def test_transform_update(self): class LisDoc(Document): From fda2e2b47ab5085b6ff5b6e27b40794a3fa9e77f Mon Sep 17 00:00:00 2001 From: Filip Kucharczyk Date: Tue, 4 Feb 2020 12:58:25 +0100 Subject: [PATCH 37/79] Update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c1..8dcea62a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 Changes in 0.19.1 ================= From ff4d57032ad366fb766b9625ead6df66bd989675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 3 Mar 2020 23:51:11 +0100 Subject: [PATCH 38/79] reformat w black --- mongoengine/context_managers.py | 4 +++- mongoengine/queryset/base.py | 20 ++++++++++++++++---- tests/queryset/test_queryset.py | 32 ++++++++++++-------------------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 0f6c8698..0c58b57c 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -264,4 +264,6 @@ def set_read_write_concern(collection, write_concerns, read_concern): combined_write_concerns = dict(collection.write_concern.document.items()) combined_write_concerns.update(write_concerns) - yield collection.with_options(write_concern=WriteConcern(**combined_write_concerns), read_concern=read_concern) + yield collection.with_options( + write_concern=WriteConcern(**combined_write_concerns), read_concern=read_concern + ) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0b76235c..0743429c 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -19,7 +19,11 @@ from mongoengine import signals from mongoengine.base import get_document from mongoengine.common import _import_class from mongoengine.connection import get_db -from mongoengine.context_managers import set_write_concern, set_read_write_concern, switch_db +from mongoengine.context_managers import ( + set_write_concern, + set_read_write_concern, + switch_db, +) from mongoengine.errors import ( BulkWriteError, InvalidQueryError, @@ -492,7 +496,13 @@ class BaseQuerySet(object): return result.deleted_count def update( - self, upsert=False, multi=True, write_concern=None, read_concern=None, full_result=False, **update + self, + upsert=False, + multi=True, + write_concern=None, + read_concern=None, + full_result=False, + **update ): """Perform an atomic update on the fields matched by the query. @@ -531,7 +541,9 @@ class BaseQuerySet(object): else: update["$set"] = {"_cls": queryset._document._class_name} try: - with set_read_write_concern(queryset._collection, write_concern, read_concern) as collection: + with set_read_write_concern( + queryset._collection, write_concern, read_concern + ) as collection: update_func = collection.update_one if multi: update_func = collection.update_many @@ -1214,7 +1226,7 @@ class BaseQuerySet(object): queryset._read_concern = read_concern queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_concern return queryset - + def scalar(self, *fields): """Instead of returning Document instances, return either a specific value or a tuple of values in order. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index c238752d..708033f4 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4671,41 +4671,33 @@ class TestQueryset(unittest.TestCase): bars = list(Bar.objects.read_concern(None)) assert bars == [bar] - bars = Bar.objects.read_concern(ReadConcern(level='local')) - assert bars._read_concern == ReadConcern(level='local') - assert ( - bars._cursor.collection.read_concern - == ReadConcern(level='local') - ) + bars = Bar.objects.read_concern(ReadConcern(level="local")) + assert bars._read_concern == ReadConcern(level="local") + assert bars._cursor.collection.read_concern == ReadConcern(level="local") # Make sure that `.read_concern(...)` does accept string values. with pytest.raises(TypeError): - Bar.objects.read_concern('local') + Bar.objects.read_concern("local") def assert_read_concern(qs, expected_read_concern): assert qs._read_concern == expected_read_concern assert qs._cursor.collection.read_concern == expected_read_concern # Make sure read concern is respected after a `.skip(...)`. - bars = Bar.objects.skip(1).read_concern(ReadConcern('majority')) - assert_read_concern(bars, ReadConcern('majority')) + bars = Bar.objects.skip(1).read_concern(ReadConcern("majority")) + assert_read_concern(bars, ReadConcern("majority")) # Make sure read concern is respected after a `.limit(...)`. - bars = Bar.objects.limit(1).read_concern(ReadConcern('majority')) - assert_read_concern(bars, ReadConcern('majority')) + bars = Bar.objects.limit(1).read_concern(ReadConcern("majority")) + assert_read_concern(bars, ReadConcern("majority")) # Make sure read concern is respected after an `.order_by(...)`. - bars = Bar.objects.order_by("txt").read_concern( - ReadConcern('majority') - ) - assert_read_concern(bars, ReadConcern('majority')) + bars = Bar.objects.order_by("txt").read_concern(ReadConcern("majority")) + assert_read_concern(bars, ReadConcern("majority")) # Make sure read concern is respected after a `.hint(...)`. - bars = Bar.objects.hint([("txt", 1)]).read_concern( - ReadConcern('majority') - ) - assert_read_concern(bars, ReadConcern('majority')) - + bars = Bar.objects.hint([("txt", 1)]).read_concern(ReadConcern("majority")) + assert_read_concern(bars, ReadConcern("majority")) def test_json_simple(self): class Embedded(EmbeddedDocument): From 421e3f324fbb8a260eba817717f8391d2c9ecdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 8 Mar 2020 14:58:21 +0100 Subject: [PATCH 39/79] Drop python2 support --- .travis.yml | 35 ++++++++------------ CONTRIBUTING.rst | 18 +++++----- benchmarks/test_inserts.py | 4 --- mongoengine/base/datastructures.py | 16 +-------- mongoengine/base/document.py | 10 ++---- mongoengine/fields.py | 17 ++++------ mongoengine/python_support.py | 23 ------------- mongoengine/queryset/base.py | 4 --- mongoengine/queryset/field_list.py | 2 -- mongoengine/queryset/transform.py | 2 +- mongoengine/queryset/visitor.py | 4 --- setup.py | 17 ++++------ tests/fields/test_binary_field.py | 10 ++---- tests/fields/test_embedded_document_field.py | 2 -- tests/fields/test_file_field.py | 6 ++-- tests/fields/test_url_field.py | 2 -- tests/queryset/test_queryset.py | 5 +-- tests/test_datastructures.py | 6 ++-- tox.ini | 2 +- 19 files changed, 51 insertions(+), 134 deletions(-) delete mode 100644 mongoengine/python_support.py diff --git a/.travis.yml b/.travis.yml index ad659fca..d34f8a36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ # For full coverage, we'd have to test all supported Python, MongoDB, and # PyMongo combinations. However, that would result in an overly long build # with a very large number of jobs, hence we only test a subset of all the -# combinations: -# * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, v3.7, v3.8, PyPy and PyPy3. -# * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo -# combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. -# * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. -# +# combinations. +# * Python3.7, MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, +# Other combinations are tested. See below for the details or check the travis jobs + # We should periodically check MongoDB Server versions supported by MongoDB # Inc., add newly released versions to the test matrix, and remove versions # which have reached their End of Life. See: @@ -16,21 +13,15 @@ # # Reminder: Update README.rst if you change MongoDB versions we test. - language: python +dist: xenial python: -- 2.7 - 3.5 - 3.6 - 3.7 - 3.8 -- pypy - pypy3 -dist: xenial - -dist: xenial - env: global: - MONGODB_3_4=3.4.17 @@ -41,6 +32,8 @@ env: - PYMONGO_3_6=3.6 - PYMONGO_3_9=3.9 - PYMONGO_3_10=3.10 + + - MAIN_PYTHON_VERSION = "3.7" matrix: - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} @@ -49,8 +42,6 @@ matrix: fast_finish: true include: - - python: 2.7 - env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 @@ -74,20 +65,20 @@ install: # tox dryrun to setup the tox venv (we run a mock test). - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder" # Install black for Python v3.7 only. - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then pip install black; fi + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pip install black; fi before_script: - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then flake8 .; else echo "flake8 only runs on py37"; fi # Run flake8 for Python 3.7 only - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then black --check .; else echo "black only runs on py37"; fi # Run black for Python 3.7 only + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then flake8 .; else echo "flake8 only runs on py37"; fi + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then black --check .; else echo "black only runs on py37"; fi - mongo --eval 'db.version();' # Make sure mongo is awake script: - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine" after_success: -- coveralls --verbose +- - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi notifications: irc: irc.freenode.org#mongoengine @@ -109,11 +100,11 @@ deploy: distributions: "sdist bdist_wheel" # Only deploy on tagged commits (aka GitHub releases) and only for the parent - # repo's builds running Python v2.7 along with PyMongo v3.x and MongoDB v3.4. + # repo's builds running Python v3.7 along with PyMongo v3.x and MongoDB v3.4. # We run Travis against many different Python, PyMongo, and MongoDB versions # and we don't want the deploy to occur multiple times). on: tags: true repo: MongoEngine/mongoengine condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4}) - python: 2.7 + python: 3.7 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b04ae968..0deda0fc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -20,23 +20,23 @@ post to the `user group ` Supported Interpreters ---------------------- -MongoEngine supports CPython 2.7 and newer. Language -features not supported by all interpreters can not be used. +MongoEngine supports CPython 3.7 and newer as well as Pypy3. +Language features not supported by all interpreters can not be used. -Python 2/3 compatibility +Python3 codebase ---------------------- -The codebase is written in a compatible manner for python 2 & 3 so it -is important that this is taken into account when it comes to discrepencies -between the two versions (see https://python-future.org/compatible_idioms.html). -Travis runs the tests against different Python versions as a safety net. +Since 0.20, the codebase is exclusively Python 3. + +Earlier versions were exclusively Python2, and was relying on 2to3 to support Python3 installs. +Travis runs the tests against the main Python 3.x versions. Style Guide ----------- -MongoEngine uses `black `_ for code -formatting. +MongoEngine uses `black `_ for code formatting. +Black runs as part of the CI so it will fail in case the code is not formatted properly. Testing ------- diff --git a/benchmarks/test_inserts.py b/benchmarks/test_inserts.py index af6399f7..4ecd48de 100644 --- a/benchmarks/test_inserts.py +++ b/benchmarks/test_inserts.py @@ -3,8 +3,6 @@ import timeit def main(): setup = """ -from builtins import range - from pymongo import MongoClient connection = MongoClient() @@ -59,8 +57,6 @@ myNoddys = noddy.find() print("{}s".format(t.timeit(1))) setup = """ -from builtins import range - from pymongo import MongoClient connection = MongoClient() diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 9f78fec0..f2d64c33 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,7 +1,6 @@ import weakref from bson import DBRef -from future.utils import listitems import six from six import iteritems @@ -181,19 +180,6 @@ class BaseList(list): __iadd__ = mark_as_changed_wrapper(list.__iadd__) __imul__ = mark_as_changed_wrapper(list.__imul__) - if six.PY2: - # Under py3 __setslice__, __delslice__ and __getslice__ - # are replaced by __setitem__, __delitem__ and __getitem__ with a slice as parameter - # so we mimic this under python 2 - def __setslice__(self, i, j, sequence): - return self.__setitem__(slice(i, j), sequence) - - def __delslice__(self, i, j): - return self.__delitem__(slice(i, j)) - - def __getslice__(self, i, j): - return self.__getitem__(slice(i, j)) - def _mark_as_changed(self, key=None): if hasattr(self._instance, "_mark_as_changed"): if key: @@ -426,7 +412,7 @@ class StrictDict(object): return len(list(iteritems(self))) def __eq__(self, other): - return listitems(self) == listitems(other) + return list(self.items()) == list(other.items()) def __ne__(self, other): return not (self == other) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 4e4df92f..73c88774 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,9 +1,9 @@ import copy + import numbers from functools import partial from bson import DBRef, ObjectId, SON, json_util -from future.utils import listitems import pymongo import six from six import iteritems @@ -26,7 +26,6 @@ from mongoengine.errors import ( OperationError, ValidationError, ) -from mongoengine.python_support import Hashable __all__ = ("BaseDocument", "NON_FIELD_ERRORS") @@ -294,10 +293,7 @@ class BaseDocument(object): def __str__(self): # TODO this could be simpler? if hasattr(self, "__unicode__"): - if six.PY3: - return self.__unicode__() - else: - return six.text_type(self).encode("utf-8") + return self.__unicode__() return six.text_type("%s object" % self.__class__.__name__) def __eq__(self, other): @@ -671,7 +667,7 @@ class BaseDocument(object): del set_data["_id"] # Determine if any changed items were actually unset. - for path, value in listitems(set_data): + for path, value in list(set_data.items()): if value or isinstance( value, (numbers.Number, bool) ): # Account for 0 and True that are truthy diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9d7d1d04..5cbf3ae0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -5,12 +5,12 @@ import re import socket import time import uuid +from io import BytesIO from operator import itemgetter from bson import Binary, DBRef, ObjectId, SON from bson.int64 import Int64 import gridfs -from past.builtins import long import pymongo from pymongo import ReturnDocument import six @@ -39,7 +39,6 @@ from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.document import Document, EmbeddedDocument from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version -from mongoengine.python_support import StringIO from mongoengine.queryset import DO_NOTHING from mongoengine.queryset.base import BaseQuerySet from mongoengine.queryset.transform import STRING_OPERATORS @@ -338,7 +337,7 @@ class IntField(BaseField): class LongField(BaseField): - """64-bit integer field.""" + """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)""" def __init__(self, min_value=None, max_value=None, **kwargs): self.min_value, self.max_value = min_value, max_value @@ -346,7 +345,7 @@ class LongField(BaseField): def to_python(self, value): try: - value = long(value) + value = int(value) except (TypeError, ValueError): pass return value @@ -356,7 +355,7 @@ class LongField(BaseField): def validate(self, value): try: - value = long(value) + value = int(value) except (TypeError, ValueError): self.error("%s could not be converted to long" % value) @@ -370,7 +369,7 @@ class LongField(BaseField): if value is None: return value - return super(LongField, self).prepare_query_value(op, long(value)) + return super(LongField, self).prepare_query_value(op, int(value)) class FloatField(BaseField): @@ -1679,8 +1678,6 @@ class GridFSProxy(object): def __bool__(self): return bool(self.grid_id) - __nonzero__ = __bool__ # For Py2 support - def __getstate__(self): self_dict = self.__dict__ self_dict["_fs"] = None @@ -1952,7 +1949,7 @@ class ImageGridFsProxy(GridFSProxy): w, h = img.size - io = StringIO() + io = BytesIO() img.save(io, img_format, progressive=progressive) io.seek(0) @@ -1971,7 +1968,7 @@ class ImageGridFsProxy(GridFSProxy): def _put_thumbnail(self, thumbnail, format, progressive, **kwargs): w, h = thumbnail.size - io = StringIO() + io = BytesIO() thumbnail.save(io, format, progressive=progressive) io.seek(0) diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py deleted file mode 100644 index 57e467db..00000000 --- a/mongoengine/python_support.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Helper functions, constants, and types to aid with Python v2.7 - v3.x support -""" -import six - -# six.BytesIO resolves to StringIO.StringIO in Py2 and io.BytesIO in Py3. -StringIO = six.BytesIO - -# Additionally for Py2, try to use the faster cStringIO, if available -if not six.PY3: - try: - import cStringIO - except ImportError: - pass - else: - StringIO = cStringIO.StringIO - - -if six.PY3: - from collections.abc import Hashable -else: - # raises DeprecationWarnings in Python >=3.7 - from collections import Hashable diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 9f76d92d..b7d0e0ab 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import copy import itertools import re @@ -204,8 +202,6 @@ class BaseQuerySet(object): """Avoid to open all records in an if stmt in Py3.""" return self._has_data() - __nonzero__ = __bool__ # For Py2 support - # Core functions def all(self): diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 5c3ff222..c2618ebd 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -69,8 +69,6 @@ class QueryFieldList(object): def __bool__(self): return bool(self.fields) - __nonzero__ = __bool__ # For Py2 support - def as_dict(self): field_list = {field: self.value for field in self.fields} if self.slice: diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 659a97e2..c179f541 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -10,7 +10,7 @@ from mongoengine.base import UPDATE_OPERATORS from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError -__all__ = ("query", "update") +__all__ = ("query", "update", "STRING_OPERATORS") COMPARISON_OPERATORS = ( "ne", diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 470839c1..72e36ac0 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -143,8 +143,6 @@ class QCombination(QNode): def __bool__(self): return bool(self.children) - __nonzero__ = __bool__ # For Py2 support - def accept(self, visitor): for i in range(len(self.children)): if isinstance(self.children[i], QNode): @@ -180,8 +178,6 @@ class Q(QNode): def __bool__(self): return bool(self.query) - __nonzero__ = __bool__ # For Py2 support - def __eq__(self, other): return self.__class__ == other.__class__ and self.query == other.query diff --git a/setup.py b/setup.py index 418ff7ad..6873284a 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,6 @@ CLASSIFIERS = [ PYTHON_VERSION = sys.version_info[0] PY3 = PYTHON_VERSION == 3 -PY2 = PYTHON_VERSION == 2 extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), @@ -124,14 +123,11 @@ extra_opts = { ], } -if PY3: - if "test" in sys.argv: - extra_opts["packages"] = find_packages() - extra_opts["package_data"] = { - "tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"] - } -else: - extra_opts["tests_require"] += ["python-dateutil"] +if "test" in sys.argv: + extra_opts["packages"] = find_packages() + extra_opts["package_data"] = { + "tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"] + } setup( name="mongoengine", @@ -148,7 +144,8 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0", "future"], + python_requires=">=3.5", + install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py index e2a1b8d6..a653b961 100644 --- a/tests/fields/test_binary_field.py +++ b/tests/fields/test_binary_field.py @@ -123,10 +123,7 @@ class TestBinaryField(MongoDBTestCase): upsert=True, new=True, set__bin_field=BIN_VALUE ) assert doc.some_field == "test" - if six.PY3: - assert doc.bin_field == BIN_VALUE - else: - assert doc.bin_field == Binary(BIN_VALUE) + assert doc.bin_field == BIN_VALUE def test_update_one(self): """Ensures no regression of bug #1127""" @@ -144,7 +141,4 @@ class TestBinaryField(MongoDBTestCase): ) assert n_updated == 1 fetched = MyDocument.objects.with_id(doc.id) - if six.PY3: - assert fetched.bin_field == BIN_VALUE - else: - assert fetched.bin_field == Binary(BIN_VALUE) + assert fetched.bin_field == BIN_VALUE diff --git a/tests/fields/test_embedded_document_field.py b/tests/fields/test_embedded_document_field.py index b80f4d8c..13ca9c0b 100644 --- a/tests/fields/test_embedded_document_field.py +++ b/tests/fields/test_embedded_document_field.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from builtins import str - import pytest from mongoengine import ( diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py index b8ece1a9..5ab6c93f 100644 --- a/tests/fields/test_file_field.py +++ b/tests/fields/test_file_field.py @@ -3,6 +3,7 @@ import copy import os import tempfile import unittest +from io import BytesIO import gridfs import pytest @@ -10,7 +11,6 @@ import six from mongoengine import * from mongoengine.connection import get_db -from mongoengine.python_support import StringIO try: from PIL import Image @@ -30,7 +30,7 @@ TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), "mongodb_leaf.png") def get_file(path): """Use a BytesIO instead of a file to allow to have a one-liner and avoid that the file remains opened""" - bytes_io = StringIO() + bytes_io = BytesIO() with open(path, "rb") as f: bytes_io.write(f.read()) bytes_io.seek(0) @@ -80,7 +80,7 @@ class TestFileField(MongoDBTestCase): PutFile.drop_collection() putfile = PutFile() - putstring = StringIO() + putstring = BytesIO() putstring.write(text) putstring.seek(0) putfile.the_file.put(putstring, content_type=content_type) diff --git a/tests/fields/test_url_field.py b/tests/fields/test_url_field.py index 477bced7..c449e467 100644 --- a/tests/fields/test_url_field.py +++ b/tests/fields/test_url_field.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from builtins import str - import pytest from mongoengine import * diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 17bf7405..7dea6dd5 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4470,10 +4470,7 @@ class TestQueryset(unittest.TestCase): pks = self.Person.objects.order_by("age").scalar("pk")[1:3] names = self.Person.objects.scalar("name").in_bulk(list(pks)).values() - if six.PY3: - expected = "['A1', 'A2']" - else: - expected = "[u'A1', u'A2']" + expected = "['A1', 'A2']" assert expected == "%s" % sorted(names) def test_elem_match(self): diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py index 7b5d7d11..734061ed 100644 --- a/tests/test_datastructures.py +++ b/tests/test_datastructures.py @@ -287,7 +287,7 @@ class TestBaseList: base_list[:] = [ 0, 1, - ] # Will use __setslice__ under py2 and __setitem__ under py3 + ] assert base_list._instance._changed_fields == ["my_name"] assert base_list == [0, 1] @@ -296,13 +296,13 @@ class TestBaseList: base_list[0:2] = [ 1, 0, - ] # Will use __setslice__ under py2 and __setitem__ under py3 + ] assert base_list._instance._changed_fields == ["my_name"] assert base_list == [1, 0, 2] def test___setitem___calls_with_step_slice_mark_as_changed(self): base_list = self._get_baselist([0, 1, 2]) - base_list[0:3:2] = [-1, -2] # uses __setitem__ in both py2 & 3 + base_list[0:3:2] = [-1, -2] # uses __setitem__ assert base_list._instance._changed_fields == ["my_name"] assert base_list == [-1, 1, -2] diff --git a/tox.ini b/tox.ini index 396817ca..675b6d9a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310} +envlist = {py35,pypy3}-{mg34,mg36,mg39,mg310} [testenv] commands = From 03e34299f0d760834386b0b68dc71672b27649bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 11 Mar 2020 21:50:50 +0100 Subject: [PATCH 40/79] clean code related to Py2 + six.text_type & six.string_types --- CONTRIBUTING.rst | 2 +- README.rst | 6 +++ mongoengine/base/datastructures.py | 3 +- mongoengine/base/document.py | 11 ++--- mongoengine/base/fields.py | 26 +++++------ mongoengine/base/metaclasses.py | 18 ++++---- mongoengine/connection.py | 9 ++-- mongoengine/dereference.py | 6 +-- mongoengine/document.py | 10 ++--- mongoengine/errors.py | 5 +-- mongoengine/fields.py | 69 +++++++++++++---------------- mongoengine/queryset/base.py | 30 ++++++------- mongoengine/queryset/transform.py | 4 +- tests/fields/test_date_field.py | 11 ----- tests/fields/test_datetime_field.py | 13 +----- tests/queryset/test_queryset.py | 26 ++++------- 16 files changed, 101 insertions(+), 148 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0deda0fc..d939e2ee 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -28,7 +28,7 @@ Python3 codebase Since 0.20, the codebase is exclusively Python 3. -Earlier versions were exclusively Python2, and was relying on 2to3 to support Python3 installs. +Earlier versions were exclusively Python2, and were relying on 2to3 to support Python3 installs. Travis runs the tests against the main Python 3.x versions. diff --git a/README.rst b/README.rst index b5c95888..1ef1363f 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,8 @@ to both create the virtual environment and install the package. Otherwise, you c download the source from `GitHub `_ and run ``python setup.py install``. +The support for Python2 was dropped with MongoEngine 0.20.0 + Dependencies ============ All of the dependencies can easily be installed via `pip `_. @@ -58,6 +60,10 @@ If you need to use an ``ImageField`` or ``ImageGridFsProxy``: - Pillow>=2.0.0 +If you need to use signals: + +- blinker>=1.3 + Examples ======== Some simple examples of what MongoEngine code looks like: diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index f2d64c33..2a2c7e7d 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,7 +1,6 @@ import weakref from bson import DBRef -import six from six import iteritems from mongoengine.common import _import_class @@ -200,7 +199,7 @@ class EmbeddedDocumentList(BaseList): """ for key, expected_value in kwargs.items(): doc_val = getattr(embedded_doc, key) - if doc_val != expected_value and six.text_type(doc_val) != expected_value: + if doc_val != expected_value and str(doc_val) != expected_value: return False return True diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 73c88774..1254d042 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -294,7 +294,7 @@ class BaseDocument(object): # TODO this could be simpler? if hasattr(self, "__unicode__"): return self.__unicode__() - return six.text_type("%s object" % self.__class__.__name__) + return "%s object" % self.__class__.__name__ def __eq__(self, other): if ( @@ -828,7 +828,7 @@ class BaseDocument(object): @classmethod def _build_index_spec(cls, spec): """Build a PyMongo index spec from a MongoEngine index spec.""" - if isinstance(spec, six.string_types): + if isinstance(spec, str): spec = {"fields": [spec]} elif isinstance(spec, (list, tuple)): spec = {"fields": list(spec)} @@ -925,7 +925,7 @@ class BaseDocument(object): # Add any unique_with fields to the back of the index spec if field.unique_with: - if isinstance(field.unique_with, six.string_types): + if isinstance(field.unique_with, str): field.unique_with = [field.unique_with] # Convert unique_with field names to real field names @@ -1172,9 +1172,6 @@ class BaseDocument(object): else [value] ) return sep.join( - [ - six.text_type(dict(field.choices).get(val, val)) - for val in values or [] - ] + [str(dict(field.choices).get(val, val)) for val in values or []] ) return value diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index cd1039cb..3c88149b 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -4,7 +4,6 @@ import weakref from bson import DBRef, ObjectId, SON import pymongo -import six from six import iteritems from mongoengine.base.common import UPDATE_OPERATORS @@ -92,13 +91,11 @@ class BaseField(object): self._owner_document = None # Make sure db_field is a string (if it's explicitly defined). - if self.db_field is not None and not isinstance( - self.db_field, six.string_types - ): + if self.db_field is not None and not isinstance(self.db_field, str): raise TypeError("db_field should be a string.") # Make sure db_field doesn't contain any forbidden characters. - if isinstance(self.db_field, six.string_types) and ( + if isinstance(self.db_field, str) and ( "." in self.db_field or "\0" in self.db_field or self.db_field.startswith("$") @@ -221,14 +218,12 @@ class BaseField(object): # Choices which are other types of Documents if isinstance(value, (Document, EmbeddedDocument)): if not any(isinstance(value, c) for c in choice_list): - self.error( - "Value must be an instance of %s" % (six.text_type(choice_list)) - ) + self.error("Value must be an instance of %s" % (choice_list)) # Choices which are types other than Documents else: values = value if isinstance(value, (list, tuple)) else [value] if len(set(values) - set(choice_list)): - self.error("Value must be one of %s" % six.text_type(choice_list)) + self.error("Value must be one of %s" % str(choice_list)) def _validate(self, value, **kwargs): # Check the Choices Constraint @@ -345,7 +340,7 @@ class ComplexBaseField(BaseField): def to_python(self, value): """Convert a MongoDB-compatible type to a Python type.""" - if isinstance(value, six.string_types): + if isinstance(value, str): return value if hasattr(value, "to_python"): @@ -399,7 +394,7 @@ class ComplexBaseField(BaseField): EmbeddedDocument = _import_class("EmbeddedDocument") GenericReferenceField = _import_class("GenericReferenceField") - if isinstance(value, six.string_types): + if isinstance(value, str): return value if hasattr(value, "to_mongo"): @@ -513,10 +508,9 @@ class ObjectIdField(BaseField): def to_mongo(self, value): if not isinstance(value, ObjectId): try: - return ObjectId(six.text_type(value)) + return ObjectId(str(value)) except Exception as e: - # e.message attribute has been deprecated since Python 2.6 - self.error(six.text_type(e)) + self.error(str(e)) return value def prepare_query_value(self, op, value): @@ -524,9 +518,9 @@ class ObjectIdField(BaseField): def validate(self, value): try: - ObjectId(six.text_type(value)) + ObjectId(str(value)) except Exception: - self.error("Invalid Object ID") + self.error("Invalid ObjectID") class GeoJsonBaseField(BaseField): diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index e4d26811..0d6e08be 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -1,7 +1,6 @@ import itertools import warnings -import six from six import iteritems, itervalues from mongoengine.base.common import _document_registry @@ -180,14 +179,15 @@ class DocumentMetaclass(type): # module continues to use im_func and im_self, so the code below # copies __func__ into im_func and __self__ into im_self for # classmethod objects in Document derived classes. - if six.PY3: - for val in new_class.__dict__.values(): - if isinstance(val, classmethod): - f = val.__get__(new_class) - if hasattr(f, "__func__") and not hasattr(f, "im_func"): - f.__dict__.update({"im_func": getattr(f, "__func__")}) - if hasattr(f, "__self__") and not hasattr(f, "im_self"): - f.__dict__.update({"im_self": getattr(f, "__self__")}) + # + # Relates to https://github.com/MongoEngine/mongoengine/issues/1107 + # for val in new_class.__dict__.values(): + # if isinstance(val, classmethod): + # f = val.__get__(new_class) + # if hasattr(f, "__func__") and not hasattr(f, "im_func"): + # f.__dict__.update({"im_func": getattr(f, "__func__")}) + # if hasattr(f, "__self__") and not hasattr(f, "im_self"): + # f.__dict__.update({"im_self": getattr(f, "__self__")}) # Handle delete rules for field in itervalues(new_class._fields): diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 4e0c60b0..3f754619 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,6 +1,5 @@ from pymongo import MongoClient, ReadPreference, uri_parser from pymongo.database import _check_name -import six __all__ = [ "DEFAULT_CONNECTION_NAME", @@ -39,8 +38,8 @@ def _check_db_name(name): """Check if a database name is valid. This functionality is copied from pymongo Database class constructor. """ - if not isinstance(name, six.string_types): - raise TypeError("name must be an instance of %s" % six.string_types) + if not isinstance(name, str): + raise TypeError("name must be an instance of %s" % str) elif name != "$external": _check_name(name) @@ -93,7 +92,7 @@ def _get_connection_settings( conn_host = conn_settings["host"] # Host can be a list or a string, so if string, force to a list. - if isinstance(conn_host, six.string_types): + if isinstance(conn_host, str): conn_host = [conn_host] resolved_hosts = [] @@ -148,7 +147,7 @@ def _get_connection_settings( # TODO simplify the code below once we drop support for # PyMongo v3.4. read_pf_mode = uri_options["readpreference"] - if isinstance(read_pf_mode, six.string_types): + if isinstance(read_pf_mode, str): read_pf_mode = read_pf_mode.lower() for preference in read_preferences: if ( diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 9e75f353..01cd5f36 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -30,7 +30,7 @@ class DeReference(object): :class:`~mongoengine.base.ComplexBaseField` :param get: A boolean determining if being called by __get__ """ - if items is None or isinstance(items, six.string_types): + if items is None or isinstance(items, str): return items # cheapest way to convert a queryset to a list @@ -274,9 +274,7 @@ class DeReference(object): (v["_ref"].collection, v["_ref"].id), v ) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: - item_name = six.text_type("{0}.{1}.{2}").format( - name, k, field_name - ) + item_name = "{0}.{1}.{2}".format(name, k, field_name) data[k]._data[field_name] = self._attach_objects( v, depth, instance=instance, name=item_name ) diff --git a/mongoengine/document.py b/mongoengine/document.py index 6a7ced46..13541da8 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -44,7 +44,7 @@ def includes_cls(fields): """Helper function used for ensuring and comparing indexes.""" first_field = None if len(fields): - if isinstance(fields[0], six.string_types): + if isinstance(fields[0], str): first_field = fields[0] elif isinstance(fields[0], (list, tuple)) and len(fields[0]): first_field = fields[0][0] @@ -430,15 +430,15 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): except pymongo.errors.DuplicateKeyError as err: message = u"Tried to save duplicate unique keys (%s)" - raise NotUniqueError(message % six.text_type(err)) + raise NotUniqueError(message % err) except pymongo.errors.OperationFailure as err: message = "Could not save document (%s)" - if re.match("^E1100[01] duplicate key", six.text_type(err)): + if re.match("^E1100[01] duplicate key", str(err)): # E11000 - duplicate key error index # E11001 - duplicate key on update message = u"Tried to save duplicate unique keys (%s)" - raise NotUniqueError(message % six.text_type(err)) - raise OperationError(message % six.text_type(err)) + raise NotUniqueError(message % err) + raise OperationError(message % err) # Make sure we store the PK on this document now that it's saved id_field = self._meta["id_field"] diff --git a/mongoengine/errors.py b/mongoengine/errors.py index b76243d3..1ac01257 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -1,6 +1,5 @@ from collections import defaultdict -import six from six import iteritems __all__ = ( @@ -93,7 +92,7 @@ class ValidationError(AssertionError): self.message = message def __str__(self): - return six.text_type(self.message) + return str(self.message) def __repr__(self): return "%s(%s,)" % (self.__class__.__name__, self.message) @@ -131,7 +130,7 @@ class ValidationError(AssertionError): elif isinstance(source, ValidationError) and source.errors: return build_dict(source.errors) else: - return six.text_type(source) + return str(source) return errors_dict diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5cbf3ae0..b6ddd566 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -108,7 +108,7 @@ class StringField(BaseField): super(StringField, self).__init__(**kwargs) def to_python(self, value): - if isinstance(value, six.text_type): + if isinstance(value, str): return value try: value = value.decode("utf-8") @@ -117,7 +117,7 @@ class StringField(BaseField): return value def validate(self, value): - if not isinstance(value, six.string_types): + if not isinstance(value, str): self.error("StringField only accepts string values") if self.max_length is not None and len(value) > self.max_length: @@ -133,7 +133,7 @@ class StringField(BaseField): return None def prepare_query_value(self, op, value): - if not isinstance(op, six.string_types): + if not isinstance(op, str): return value if op in STRING_OPERATORS: @@ -472,13 +472,13 @@ class DecimalField(BaseField): if value is None: return value if self.force_string: - return six.text_type(self.to_python(value)) + return str(self.to_python(value)) return float(self.to_python(value)) def validate(self, value): if not isinstance(value, decimal.Decimal): - if not isinstance(value, six.string_types): - value = six.text_type(value) + if not isinstance(value, str): + value = str(value) try: value = decimal.Decimal(value) except (TypeError, ValueError, decimal.InvalidOperation) as exc: @@ -543,7 +543,7 @@ class DateTimeField(BaseField): if callable(value): return value() - if not isinstance(value, six.string_types): + if not isinstance(value, str): return None return self._parse_datetime(value) @@ -707,7 +707,7 @@ class EmbeddedDocumentField(BaseField): def __init__(self, document_type, **kwargs): # XXX ValidationError raised outside of the "validate" method. if not ( - isinstance(document_type, six.string_types) + isinstance(document_type, str) or issubclass(document_type, EmbeddedDocument) ): self.error( @@ -720,7 +720,7 @@ class EmbeddedDocumentField(BaseField): @property def document_type(self): - if isinstance(self.document_type_obj, six.string_types): + if isinstance(self.document_type_obj, str): if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: resolved_document_type = self.owner_document else: @@ -846,7 +846,7 @@ class DynamicField(BaseField): """Convert a Python type to a MongoDB compatible type. """ - if isinstance(value, six.string_types): + if isinstance(value, str): return value if hasattr(value, "to_mongo"): @@ -889,7 +889,7 @@ class DynamicField(BaseField): return member_name def prepare_query_value(self, op, value): - if isinstance(value, six.string_types): + if isinstance(value, str): return StringField().prepare_query_value(op, value) return super(DynamicField, self).prepare_query_value(op, self.to_mongo(value)) @@ -954,7 +954,7 @@ class ListField(ComplexBaseField): if ( op in ("set", "unset", None) and hasattr(value, "__iter__") - and not isinstance(value, six.string_types) + and not isinstance(value, str) and not isinstance(value, BaseDocument) ): return [self.field.prepare_query_value(op, v) for v in value] @@ -1026,9 +1026,7 @@ def key_not_string(d): dictionary is not a string. """ for k, v in d.items(): - if not isinstance(k, six.string_types) or ( - isinstance(v, dict) and key_not_string(v) - ): + if not isinstance(k, str) or (isinstance(v, dict) and key_not_string(v)): return True @@ -1107,7 +1105,7 @@ class DictField(ComplexBaseField): "iexact", ] - if op in match_operators and isinstance(value, six.string_types): + if op in match_operators and isinstance(value, str): return StringField().prepare_query_value(op, value) if hasattr( @@ -1194,7 +1192,7 @@ class ReferenceField(BaseField): :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`. """ # XXX ValidationError raised outside of the "validate" method. - if not isinstance(document_type, six.string_types) and not issubclass( + if not isinstance(document_type, str) and not issubclass( document_type, Document ): self.error( @@ -1209,7 +1207,7 @@ class ReferenceField(BaseField): @property def document_type(self): - if isinstance(self.document_type_obj, six.string_types): + if isinstance(self.document_type_obj, str): if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: self.document_type_obj = self.owner_document else: @@ -1325,7 +1323,7 @@ class CachedReferenceField(BaseField): fields = [] # XXX ValidationError raised outside of the "validate" method. - if not isinstance(document_type, six.string_types) and not issubclass( + if not isinstance(document_type, str) and not issubclass( document_type, Document ): self.error( @@ -1370,7 +1368,7 @@ class CachedReferenceField(BaseField): @property def document_type(self): - if isinstance(self.document_type_obj, six.string_types): + if isinstance(self.document_type_obj, str): if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: self.document_type_obj = self.owner_document else: @@ -1498,7 +1496,7 @@ class GenericReferenceField(BaseField): # Keep the choices as a list of allowed Document class names if choices: for choice in choices: - if isinstance(choice, six.string_types): + if isinstance(choice, str): self.choices.append(choice) elif isinstance(choice, type) and issubclass(choice, Document): self.choices.append(choice._class_name) @@ -1507,7 +1505,7 @@ class GenericReferenceField(BaseField): # method. self.error( "Invalid choices provided: must be a list of" - "Document subclasses and/or six.string_typess" + "Document subclasses and/or str" ) def _validate_choices(self, value): @@ -1601,8 +1599,8 @@ class BinaryField(BaseField): def __set__(self, instance, value): """Handle bytearrays in python 3.1""" - if six.PY3 and isinstance(value, bytearray): - value = six.binary_type(value) + if isinstance(value, bytearray): + value = bytes(value) return super(BinaryField, self).__set__(instance, value) def to_mongo(self, value): @@ -1831,7 +1829,7 @@ class FileField(BaseField): key = self.name if ( hasattr(value, "read") and not isinstance(value, GridFSProxy) - ) or isinstance(value, (six.binary_type, six.string_types)): + ) or isinstance(value, (six.binary_type, str)): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) # If a file already exists, delete it @@ -2038,12 +2036,7 @@ class ImageField(FileField): for att_name, att in extra_args.items(): value = None if isinstance(att, (tuple, list)): - if six.PY3: - value = dict( - itertools.zip_longest(params_size, att, fillvalue=None) - ) - else: - value = dict(map(None, params_size, att)) + value = dict(itertools.zip_longest(params_size, att, fillvalue=None)) setattr(self, att_name, value) @@ -2213,8 +2206,8 @@ class UUIDField(BaseField): if not self._binary: original_value = value try: - if not isinstance(value, six.string_types): - value = six.text_type(value) + if not isinstance(value, str): + value = str(value) return uuid.UUID(value) except (ValueError, TypeError, AttributeError): return original_value @@ -2222,8 +2215,8 @@ class UUIDField(BaseField): def to_mongo(self, value): if not self._binary: - return six.text_type(value) - elif isinstance(value, six.string_types): + return str(value) + elif isinstance(value, str): return uuid.UUID(value) return value @@ -2234,7 +2227,7 @@ class UUIDField(BaseField): def validate(self, value): if not isinstance(value, uuid.UUID): - if not isinstance(value, six.string_types): + if not isinstance(value, str): value = str(value) try: uuid.UUID(value) @@ -2433,7 +2426,7 @@ class LazyReferenceField(BaseField): document. Note this only work getting field (not setting or deleting). """ # XXX ValidationError raised outside of the "validate" method. - if not isinstance(document_type, six.string_types) and not issubclass( + if not isinstance(document_type, str) and not issubclass( document_type, Document ): self.error( @@ -2449,7 +2442,7 @@ class LazyReferenceField(BaseField): @property def document_type(self): - if isinstance(self.document_type_obj, six.string_types): + if isinstance(self.document_type_obj, str): if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: self.document_type_obj = self.owner_document else: diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index b7d0e0ab..8a068e2e 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -349,20 +349,20 @@ class BaseQuerySet(object): ) except pymongo.errors.DuplicateKeyError as err: message = "Could not save document (%s)" - raise NotUniqueError(message % six.text_type(err)) + raise NotUniqueError(message % err) except pymongo.errors.BulkWriteError as err: # inserting documents that already have an _id field will # give huge performance debt or raise message = u"Bulk write error: (%s)" - raise BulkWriteError(message % six.text_type(err.details)) + raise BulkWriteError(message % err.details) except pymongo.errors.OperationFailure as err: message = "Could not save document (%s)" - if re.match("^E1100[01] duplicate key", six.text_type(err)): + if re.match("^E1100[01] duplicate key", str(err)): # E11000 - duplicate key error index # E11001 - duplicate key on update message = u"Tried to save duplicate unique keys (%s)" - raise NotUniqueError(message % six.text_type(err)) - raise OperationError(message % six.text_type(err)) + raise NotUniqueError(message % err) + raise OperationError(message % err) # Apply inserted_ids to documents for doc, doc_id in zip(docs, ids): @@ -534,12 +534,12 @@ class BaseQuerySet(object): elif result.raw_result: return result.raw_result["n"] except pymongo.errors.DuplicateKeyError as err: - raise NotUniqueError(u"Update failed (%s)" % six.text_type(err)) + raise NotUniqueError("Update failed (%s)" % err) except pymongo.errors.OperationFailure as err: - if six.text_type(err) == u"multi not coded yet": - message = u"update() method requires MongoDB 1.1.3+" + if str(err) == "multi not coded yet": + message = "update() method requires MongoDB 1.1.3+" raise OperationError(message) - raise OperationError(u"Update failed (%s)" % six.text_type(err)) + raise OperationError("Update failed (%s)" % err) def upsert_one(self, write_concern=None, **update): """Overwrite or add the first document matched by the query. @@ -1348,13 +1348,13 @@ class BaseQuerySet(object): map_f_scope = {} if isinstance(map_f, Code): map_f_scope = map_f.scope - map_f = six.text_type(map_f) + map_f = str(map_f) map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) reduce_f_scope = {} if isinstance(reduce_f, Code): reduce_f_scope = reduce_f.scope - reduce_f = six.text_type(reduce_f) + reduce_f = str(reduce_f) reduce_f_code = queryset._sub_js_fields(reduce_f) reduce_f = Code(reduce_f_code, reduce_f_scope) @@ -1364,7 +1364,7 @@ class BaseQuerySet(object): finalize_f_scope = {} if isinstance(finalize_f, Code): finalize_f_scope = finalize_f.scope - finalize_f = six.text_type(finalize_f) + finalize_f = str(finalize_f) finalize_f_code = queryset._sub_js_fields(finalize_f) finalize_f = Code(finalize_f_code, finalize_f_scope) mr_args["finalize"] = finalize_f @@ -1380,7 +1380,7 @@ class BaseQuerySet(object): else: map_reduce_function = "map_reduce" - if isinstance(output, six.string_types): + if isinstance(output, str): mr_args["out"] = output elif isinstance(output, dict): @@ -1838,7 +1838,7 @@ class BaseQuerySet(object): field_parts = field.split(".") try: field = ".".join( - f if isinstance(f, six.string_types) else f.db_field + f if isinstance(f, str) else f.db_field for f in self._document._lookup_field(field_parts) ) db_field_paths.append(field) @@ -1850,7 +1850,7 @@ class BaseQuerySet(object): for subdoc in subclasses: try: subfield = ".".join( - f if isinstance(f, six.string_types) else f.db_field + f if isinstance(f, str) else f.db_field for f in subdoc._lookup_field(field_parts) ) db_field_paths.append(subfield) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index c179f541..efbdae4e 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -101,7 +101,7 @@ def query(_doc_cls=None, **kwargs): cleaned_fields = [] for field in fields: append_field = True - if isinstance(field, six.string_types): + if isinstance(field, str): parts.append(field) append_field = False # is last and CachedReferenceField @@ -281,7 +281,7 @@ def update(_doc_cls=None, **update): appended_sub_field = False for field in fields: append_field = True - if isinstance(field, six.string_types): + if isinstance(field, str): # Convert the S operator to $ if field == "S": field = "$" diff --git a/tests/fields/test_date_field.py b/tests/fields/test_date_field.py index e94ed0ce..5e1bfaa4 100644 --- a/tests/fields/test_date_field.py +++ b/tests/fields/test_date_field.py @@ -89,17 +89,6 @@ class TestDateField(MongoDBTestCase): assert log.date == d1.date() assert log.date == d2.date() - if not six.PY3: - # Pre UTC dates microseconds below 1000 are dropped - # This does not seem to be true in PY3 - d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) - d2 = datetime.datetime(1969, 12, 31, 23, 59, 59) - log.date = d1 - log.save() - log.reload() - assert log.date == d1.date() - assert log.date == d2.date() - def test_regular_usage(self): """Tests for regular datetime fields""" diff --git a/tests/fields/test_datetime_field.py b/tests/fields/test_datetime_field.py index 70debac5..21ab523b 100644 --- a/tests/fields/test_datetime_field.py +++ b/tests/fields/test_datetime_field.py @@ -98,17 +98,6 @@ class TestDateTimeField(MongoDBTestCase): assert log.date != d1 assert log.date == d2 - if not six.PY3: - # Pre UTC dates microseconds below 1000 are dropped - # This does not seem to be true in PY3 - d1 = dt.datetime(1969, 12, 31, 23, 59, 59, 999) - d2 = dt.datetime(1969, 12, 31, 23, 59, 59) - log.date = d1 - log.save() - log.reload() - assert log.date != d1 - assert log.date == d2 - def test_regular_usage(self): """Tests for regular datetime fields""" @@ -213,7 +202,7 @@ class TestDateTimeField(MongoDBTestCase): # make sure that passing a parsable datetime works dtd = DTDoc() dtd.date = date_str - assert isinstance(dtd.date, six.string_types) + assert isinstance(dtd.date, str) dtd.save() dtd.reload() diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 7dea6dd5..cf4cd0dc 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4445,24 +4445,14 @@ class TestQueryset(unittest.TestCase): "A0" == "%s" % self.Person.objects.order_by("name").scalar("name").first() ) assert "A0" == "%s" % self.Person.objects.scalar("name").order_by("name")[0] - if six.PY3: - assert ( - "['A1', 'A2']" - == "%s" % self.Person.objects.order_by("age").scalar("name")[1:3] - ) - assert ( - "['A51', 'A52']" - == "%s" % self.Person.objects.order_by("age").scalar("name")[51:53] - ) - else: - assert ( - "[u'A1', u'A2']" - == "%s" % self.Person.objects.order_by("age").scalar("name")[1:3] - ) - assert ( - "[u'A51', u'A52']" - == "%s" % self.Person.objects.order_by("age").scalar("name")[51:53] - ) + assert ( + "['A1', 'A2']" + == "%s" % self.Person.objects.order_by("age").scalar("name")[1:3] + ) + assert ( + "['A51', 'A52']" + == "%s" % self.Person.objects.order_by("age").scalar("name")[51:53] + ) # with_id and in_bulk person = self.Person.objects.order_by("name").first() From 8086576677d21bb1508bcf1c4c78efb0091d8f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 11 Mar 2020 23:07:03 +0100 Subject: [PATCH 41/79] get rid of six --- README.rst | 1 - docs/upgrade.rst | 2 +- mongoengine/base/datastructures.py | 5 ++-- mongoengine/base/document.py | 18 ++++++------- mongoengine/base/fields.py | 9 +++---- mongoengine/base/metaclasses.py | 16 +++++------ mongoengine/context_managers.py | 3 +-- mongoengine/dereference.py | 16 +++++------ mongoengine/document.py | 12 ++++----- mongoengine/errors.py | 9 +++---- mongoengine/fields.py | 16 +++++------ mongoengine/queryset/base.py | 12 ++++----- mongoengine/queryset/queryset.py | 10 +++---- mongoengine/queryset/transform.py | 4 +-- requirements.txt | 1 - setup.py | 2 +- tests/document/test_indexes.py | 25 +++++++++--------- tests/document/test_inheritance.py | 3 +-- tests/document/test_instance.py | 5 ++-- tests/fields/test_binary_field.py | 17 ++++++------ tests/fields/test_date_field.py | 1 - tests/fields/test_datetime_field.py | 1 - tests/fields/test_file_field.py | 41 +++++++++++++++-------------- tests/fields/test_float_field.py | 6 ++--- tests/fields/test_long_field.py | 6 ++--- tests/queryset/test_queryset.py | 4 +-- tests/test_datastructures.py | 3 +-- tests/test_dereference.py | 26 +++++++++--------- 28 files changed, 118 insertions(+), 156 deletions(-) diff --git a/README.rst b/README.rst index 1ef1363f..619970af 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,6 @@ All of the dependencies can easily be installed via `pip ` At the very least, you'll need these two packages to use MongoEngine: - pymongo>=3.4 -- six>=1.10.0 If you utilize a ``DateTimeField``, you might also use a more flexible date parser: diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 250347bf..f25bab8f 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -153,7 +153,7 @@ inherited classes like so: :: # 4. Remove indexes info = collection.index_information() - indexes_to_drop = [key for key, value in info.iteritems() + indexes_to_drop = [key for key, value in info.items() if '_types' in dict(value['key'])] for index in indexes_to_drop: collection.drop_index(index) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 2a2c7e7d..bb70089e 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,7 +1,6 @@ import weakref from bson import DBRef -from six import iteritems from mongoengine.common import _import_class from mongoengine.errors import DoesNotExist, MultipleObjectsReturned @@ -360,7 +359,7 @@ class StrictDict(object): _classes = {} def __init__(self, **kwargs): - for k, v in iteritems(kwargs): + for k, v in kwargs.items(): setattr(self, k, v) def __getitem__(self, key): @@ -408,7 +407,7 @@ class StrictDict(object): return (key for key in self.__slots__ if hasattr(self, key)) def __len__(self): - return len(list(iteritems(self))) + return len(list(self.items())) def __eq__(self, other): return list(self.items()) == list(other.items()) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 1254d042..dff759a7 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -5,8 +5,6 @@ from functools import partial from bson import DBRef, ObjectId, SON, json_util import pymongo -import six -from six import iteritems from mongoengine import signals from mongoengine.base.common import get_document @@ -110,7 +108,7 @@ class BaseDocument(object): # Assign default values to the instance. # We set default values only for fields loaded from DB. See # https://github.com/mongoengine/mongoengine/issues/399 for more info. - for key, field in iteritems(self._fields): + for key, field in self._fields.items(): if self._db_field_map.get(key, key) in __only_fields: continue value = getattr(self, key, None) @@ -122,14 +120,14 @@ class BaseDocument(object): # Set passed values after initialisation if self._dynamic: dynamic_data = {} - for key, value in iteritems(values): + for key, value in values.items(): if key in self._fields or key == "_id": setattr(self, key, value) else: dynamic_data[key] = value else: FileField = _import_class("FileField") - for key, value in iteritems(values): + for key, value in values.items(): key = self._reverse_db_field_map.get(key, key) if key in self._fields or key in ("id", "pk", "_cls"): if __auto_convert and value is not None: @@ -145,7 +143,7 @@ class BaseDocument(object): if self._dynamic: self._dynamic_lock = False - for key, value in iteritems(dynamic_data): + for key, value in dynamic_data.items(): setattr(self, key, value) # Flag initialised @@ -575,7 +573,7 @@ class BaseDocument(object): if not hasattr(data, "items"): iterator = enumerate(data) else: - iterator = iteritems(data) + iterator = data.items() for index_or_key, value in iterator: item_key = "%s%s." % (base_key, index_or_key) @@ -741,7 +739,7 @@ class BaseDocument(object): # Convert SON to a data dict, making sure each key is a string and # corresponds to the right db field. data = {} - for key, value in iteritems(son): + for key, value in son.items(): key = str(key) key = cls._db_field_map.get(key, key) data[key] = value @@ -756,7 +754,7 @@ class BaseDocument(object): if not _auto_dereference: fields = copy.deepcopy(fields) - for field_name, field in iteritems(fields): + for field_name, field in fields.items(): field._auto_dereference = _auto_dereference if field.db_field in data: value = data[field.db_field] @@ -781,7 +779,7 @@ class BaseDocument(object): # In STRICT documents, remove any keys that aren't in cls._fields if cls.STRICT: - data = {k: v for k, v in iteritems(data) if k in cls._fields} + data = {k: v for k, v in data.items() if k in cls._fields} obj = cls( __auto_convert=False, _created=created, __only_fields=only_fields, **data diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 3c88149b..ac894d91 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -4,7 +4,6 @@ import weakref from bson import DBRef, ObjectId, SON import pymongo -from six import iteritems from mongoengine.base.common import UPDATE_OPERATORS from mongoengine.base.datastructures import BaseDict, BaseList, EmbeddedDocumentList @@ -418,11 +417,11 @@ class ComplexBaseField(BaseField): if self.field: value_dict = { key: self.field._to_mongo_safe_call(item, use_db_field, fields) - for key, item in iteritems(value) + for key, item in value.items() } else: value_dict = {} - for k, v in iteritems(value): + for k, v in value.items(): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: @@ -461,8 +460,8 @@ class ComplexBaseField(BaseField): """If field is provided ensure the value is valid.""" errors = {} if self.field: - if hasattr(value, "iteritems") or hasattr(value, "items"): - sequence = iteritems(value) + if hasattr(value, "items"): + sequence = value.items() else: sequence = enumerate(value) for k, v in sequence: diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 0d6e08be..30a6fbab 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -1,8 +1,6 @@ import itertools import warnings -from six import iteritems, itervalues - from mongoengine.base.common import _document_registry from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField from mongoengine.common import _import_class @@ -68,7 +66,7 @@ class DocumentMetaclass(type): # Standard object mixin - merge in any Fields if not hasattr(base, "_meta"): base_fields = {} - for attr_name, attr_value in iteritems(base.__dict__): + for attr_name, attr_value in base.__dict__.items(): if not isinstance(attr_value, BaseField): continue attr_value.name = attr_name @@ -80,7 +78,7 @@ class DocumentMetaclass(type): # Discover any document fields field_names = {} - for attr_name, attr_value in iteritems(attrs): + for attr_name, attr_value in attrs.items(): if not isinstance(attr_value, BaseField): continue attr_value.name = attr_name @@ -110,9 +108,7 @@ class DocumentMetaclass(type): attrs["_fields_ordered"] = tuple( i[1] - for i in sorted( - (v.creation_counter, v.name) for v in itervalues(doc_fields) - ) + for i in sorted((v.creation_counter, v.name) for v in doc_fields.values()) ) # @@ -190,7 +186,7 @@ class DocumentMetaclass(type): # f.__dict__.update({"im_self": getattr(f, "__self__")}) # Handle delete rules - for field in itervalues(new_class._fields): + for field in new_class._fields.values(): f = field if f.owner_document is None: f.owner_document = new_class @@ -399,7 +395,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class.objects = QuerySetManager() # Validate the fields and set primary key if needed - for field_name, field in iteritems(new_class._fields): + for field_name, field in new_class._fields.items(): if field.primary_key: # Ensure only one primary key is set current_pk = new_class._meta.get("id_field") @@ -476,7 +472,7 @@ class MetaDict(dict): _merge_options = ("indexes",) def merge(self, new_options): - for k, v in iteritems(new_options): + for k, v in new_options.items(): if k in self._merge_options: self[k] = self.get(k, []) + v else: diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 1592ceef..5e9a6e8b 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -1,7 +1,6 @@ from contextlib import contextmanager from pymongo.write_concern import WriteConcern -from six import iteritems from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db @@ -123,7 +122,7 @@ class no_dereference(object): self.deref_fields = [ k - for k, v in iteritems(self.cls._fields) + for k, v in self.cls._fields.items() if isinstance(v, (ReferenceField, GenericReferenceField, ComplexBaseField)) ] diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 01cd5f36..d0e6c527 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,6 +1,4 @@ from bson import DBRef, SON -import six -from six import iteritems from mongoengine.base import ( BaseDict, @@ -79,7 +77,7 @@ class DeReference(object): def _get_items_from_dict(items): new_items = {} - for k, v in iteritems(items): + for k, v in items.items(): value = v if isinstance(v, list): value = _get_items_from_list(v) @@ -120,7 +118,7 @@ class DeReference(object): depth += 1 for item in iterator: if isinstance(item, (Document, EmbeddedDocument)): - for field_name, field in iteritems(item._fields): + for field_name, field in item._fields.items(): v = item._data.get(field_name, None) if isinstance(v, LazyReference): # LazyReference inherits DBRef but should not be dereferenced here ! @@ -136,7 +134,7 @@ class DeReference(object): getattr(field, "field", None), "document_type", None ) references = self._find_references(v, depth) - for key, refs in iteritems(references): + for key, refs in references.items(): if isinstance( field_cls, (Document, TopLevelDocumentMetaclass) ): @@ -153,7 +151,7 @@ class DeReference(object): ) elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth: references = self._find_references(item, depth - 1) - for key, refs in iteritems(references): + for key, refs in references.items(): reference_map.setdefault(key, set()).update(refs) return reference_map @@ -162,7 +160,7 @@ class DeReference(object): """Fetch all references and convert to their document objects """ object_map = {} - for collection, dbrefs in iteritems(self.reference_map): + for collection, dbrefs in self.reference_map.items(): # we use getattr instead of hasattr because hasattr swallows any exception under python2 # so it could hide nasty things without raising exceptions (cfr bug #1688)) @@ -174,7 +172,7 @@ class DeReference(object): dbref for dbref in dbrefs if (col_name, dbref) not in object_map ] references = collection.objects.in_bulk(refs) - for key, doc in iteritems(references): + for key, doc in references.items(): object_map[(col_name, key)] = doc else: # Generic reference: use the refs data to convert to document if isinstance(doc_type, (ListField, DictField, MapField)): @@ -250,7 +248,7 @@ class DeReference(object): data = [] else: is_list = False - iterator = iteritems(items) + iterator = items.items() data = {} depth += 1 diff --git a/mongoengine/document.py b/mongoengine/document.py index 13541da8..4ac979d1 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -4,8 +4,6 @@ import warnings from bson.dbref import DBRef import pymongo from pymongo.read_preferences import ReadPreference -import six -from six import iteritems from mongoengine import signals from mongoengine.base import ( @@ -55,7 +53,7 @@ class InvalidCollectionError(Exception): pass -class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): +class EmbeddedDocument(BaseDocument, metaclass=DocumentMetaclass): """A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the @@ -103,7 +101,7 @@ class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): return data -class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): +class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): """The base class used for defining the structure and properties of collections of documents stored in MongoDB. Inherit from this class, and add fields as class attributes to define a document's structure. @@ -632,7 +630,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): # Delete FileFields separately FileField = _import_class("FileField") - for name, field in iteritems(self._fields): + for name, field in self._fields.items(): if isinstance(field, FileField): getattr(self, name).delete() @@ -1029,7 +1027,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): return {"missing": missing, "extra": extra} -class DynamicDocument(six.with_metaclass(TopLevelDocumentMetaclass, Document)): +class DynamicDocument(Document, metaclass=TopLevelDocumentMetaclass): """A Dynamic Document class allowing flexible, expandable and uncontrolled schemas. As a :class:`~mongoengine.Document` subclass, acts in the same way as an ordinary document but has expanded style properties. Any data @@ -1060,7 +1058,7 @@ class DynamicDocument(six.with_metaclass(TopLevelDocumentMetaclass, Document)): super(DynamicDocument, self).__delattr__(*args, **kwargs) -class DynamicEmbeddedDocument(six.with_metaclass(DocumentMetaclass, EmbeddedDocument)): +class DynamicEmbeddedDocument(EmbeddedDocument, metaclass=DocumentMetaclass): """A Dynamic Embedded Document class allowing flexible, expandable and uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more information about dynamic documents. diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 1ac01257..7045145c 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -1,6 +1,5 @@ from collections import defaultdict -from six import iteritems __all__ = ( "NotRegistered", @@ -125,7 +124,7 @@ class ValidationError(AssertionError): def build_dict(source): errors_dict = {} if isinstance(source, dict): - for field_name, error in iteritems(source): + for field_name, error in source.items(): errors_dict[field_name] = build_dict(error) elif isinstance(source, ValidationError) and source.errors: return build_dict(source.errors) @@ -146,15 +145,15 @@ class ValidationError(AssertionError): if isinstance(value, list): value = " ".join([generate_key(k) for k in value]) elif isinstance(value, dict): - value = " ".join([generate_key(v, k) for k, v in iteritems(value)]) + value = " ".join([generate_key(v, k) for k, v in value.items()]) results = "%s.%s" % (prefix, value) if prefix else value return results error_dict = defaultdict(list) - for k, v in iteritems(self.to_dict()): + for k, v in self.to_dict().items(): error_dict[generate_key(v)].append(k) - return " ".join(["%s: %s" % (k, v) for k, v in iteritems(error_dict)]) + return " ".join(["%s: %s" % (k, v) for k, v in error_dict.items()]) class DeprecatedError(Exception): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b6ddd566..391ad37b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -13,8 +13,6 @@ from bson.int64 import Int64 import gridfs import pymongo from pymongo import ReturnDocument -import six -from six import iteritems try: import dateutil @@ -205,7 +203,7 @@ class EmailField(StringField): ) UTF8_USER_REGEX = LazyRegexCompiler( - six.u( + ( # RFC 6531 Section 3.3 extends `atext` (used by dot-atom) to # include `UTF8-non-ascii`. r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z\u0080-\U0010FFFF]+)*\Z" @@ -387,7 +385,7 @@ class FloatField(BaseField): return value def validate(self, value): - if isinstance(value, six.integer_types): + if isinstance(value, int): try: value = float(value) except OverflowError: @@ -868,12 +866,12 @@ class DynamicField(BaseField): value = {k: v for k, v in enumerate(value)} data = {} - for k, v in iteritems(value): + for k, v in value.items(): data[k] = self.to_mongo(v, use_db_field, fields) value = data if is_list: # Convert back to a list - value = [v for k, v in sorted(iteritems(data), key=itemgetter(0))] + value = [v for k, v in sorted(data.items(), key=itemgetter(0))] return value def to_python(self, value): @@ -1607,10 +1605,10 @@ class BinaryField(BaseField): return Binary(value) def validate(self, value): - if not isinstance(value, (six.binary_type, Binary)): + if not isinstance(value, (bytes, Binary)): self.error( "BinaryField only accepts instances of " - "(%s, %s, Binary)" % (six.binary_type.__name__, Binary.__name__) + "(%s, %s, Binary)" % (bytes.__name__, Binary.__name__) ) if self.max_bytes is not None and len(value) > self.max_bytes: @@ -1829,7 +1827,7 @@ class FileField(BaseField): key = self.name if ( hasattr(value, "read") and not isinstance(value, GridFSProxy) - ) or isinstance(value, (six.binary_type, str)): + ) or isinstance(value, (bytes, str)): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) # If a file already exists, delete it diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 8a068e2e..e6901100 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -9,8 +9,6 @@ import pymongo import pymongo.errors from pymongo.collection import ReturnDocument from pymongo.common import validate_read_preference -import six -from six import iteritems from mongoengine import signals from mongoengine.base import get_document @@ -252,12 +250,12 @@ class BaseQuerySet(object): queryset = queryset.filter(*q_objs, **query) try: - result = six.next(queryset) + result = next(queryset) except StopIteration: msg = "%s matching query does not exist." % queryset._document._class_name raise queryset._document.DoesNotExist(msg) try: - six.next(queryset) + next(queryset) except StopIteration: return result @@ -1567,7 +1565,7 @@ class BaseQuerySet(object): if self._limit == 0 or self._none: raise StopIteration - raw_doc = six.next(self._cursor) + raw_doc = next(self._cursor) if self._as_pymongo: return raw_doc @@ -1812,13 +1810,13 @@ class BaseQuerySet(object): } """ total, data, types = self.exec_js(freq_func, field) - values = {types.get(k): int(v) for k, v in iteritems(data)} + values = {types.get(k): int(v) for k, v in data.items()} if normalize: values = {k: float(v) / total for k, v in values.items()} frequencies = {} - for k, v in iteritems(values): + for k, v in values.items(): if isinstance(k, float): if int(k) == k: k = int(k) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4ba62d46..39b09c9d 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,5 +1,3 @@ -import six - from mongoengine.errors import OperationError from mongoengine.queryset.base import ( BaseQuerySet, @@ -127,8 +125,8 @@ class QuerySet(BaseQuerySet): # Pull in ITER_CHUNK_SIZE docs from the database and store them in # the result cache. try: - for _ in six.moves.range(ITER_CHUNK_SIZE): - self._result_cache.append(six.next(self)) + for _ in range(ITER_CHUNK_SIZE): + self._result_cache.append(next(self)) except StopIteration: # Getting this exception means there are no more docs in the # db cursor. Set _has_more to False so that we can use that @@ -180,9 +178,9 @@ class QuerySetNoCache(BaseQuerySet): return ".. queryset mid-iteration .." data = [] - for _ in six.moves.range(REPR_OUTPUT_SIZE + 1): + for _ in range(REPR_OUTPUT_SIZE + 1): try: - data.append(six.next(self)) + data.append(next(self)) except StopIteration: break diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index efbdae4e..1202ec45 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -3,8 +3,6 @@ from collections import defaultdict from bson import ObjectId, SON from bson.dbref import DBRef import pymongo -import six -from six import iteritems from mongoengine.base import UPDATE_OPERATORS from mongoengine.common import _import_class @@ -180,7 +178,7 @@ def query(_doc_cls=None, **kwargs): "$near" in value_dict or "$nearSphere" in value_dict ): value_son = SON() - for k, v in iteritems(value_dict): + for k, v in value_dict.items(): if k == "$maxDistance" or k == "$minDistance": continue value_son[k] = v diff --git a/requirements.txt b/requirements.txt index 43e5261b..0ce39f74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ pymongo>=3.4 -six==1.10.0 Sphinx==1.5.5 sphinx-rtd-theme==0.2.4 diff --git a/setup.py b/setup.py index 6873284a..b60188c7 100644 --- a/setup.py +++ b/setup.py @@ -145,7 +145,7 @@ setup( platforms=["any"], classifiers=CLASSIFIERS, python_requires=">=3.5", - install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0"], + install_requires=["pymongo>=3.4, <4.0"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index be857b59..4179a026 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -5,7 +5,6 @@ from datetime import datetime from pymongo.collation import Collation from pymongo.errors import OperationFailure import pytest -from six import iteritems from mongoengine import * from mongoengine.connection import get_db @@ -59,7 +58,7 @@ class TestIndexes(unittest.TestCase): info = BlogPost.objects._collection.index_information() # _id, '-date', 'tags', ('cat', 'date') assert len(info) == 4 - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] for expected in expected_specs: assert expected["fields"] in info @@ -87,7 +86,7 @@ class TestIndexes(unittest.TestCase): # the indices on -date and tags will both contain # _cls as first element in the key assert len(info) == 4 - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] for expected in expected_specs: assert expected["fields"] in info @@ -102,7 +101,7 @@ class TestIndexes(unittest.TestCase): ExtendedBlogPost.ensure_indexes() info = ExtendedBlogPost.objects._collection.index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] for expected in expected_specs: assert expected["fields"] in info @@ -192,7 +191,7 @@ class TestIndexes(unittest.TestCase): # Indexes are lazy so use list() to perform query list(Person.objects) info = Person.objects._collection.index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("rank.title", 1)] in info def test_explicit_geo2d_index(self): @@ -207,7 +206,7 @@ class TestIndexes(unittest.TestCase): Place.ensure_indexes() info = Place._get_collection().index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("location.point", "2d")] in info def test_explicit_geo2d_index_embedded(self): @@ -227,7 +226,7 @@ class TestIndexes(unittest.TestCase): Place.ensure_indexes() info = Place._get_collection().index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("current.location.point", "2d")] in info def test_explicit_geosphere_index(self): @@ -244,7 +243,7 @@ class TestIndexes(unittest.TestCase): Place.ensure_indexes() info = Place._get_collection().index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("location.point", "2dsphere")] in info def test_explicit_geohaystack_index(self): @@ -266,7 +265,7 @@ class TestIndexes(unittest.TestCase): Place.ensure_indexes() info = Place._get_collection().index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("location.point", "geoHaystack")] in info def test_create_geohaystack_index(self): @@ -279,7 +278,7 @@ class TestIndexes(unittest.TestCase): Place.create_index({"fields": (")location.point", "name")}, bucketSize=10) info = Place._get_collection().index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("location.point", "geoHaystack"), ("name", 1)] in info def test_dictionary_indexes(self): @@ -308,7 +307,7 @@ class TestIndexes(unittest.TestCase): info = BlogPost.objects._collection.index_information() info = [ (value["key"], value.get("unique", False), value.get("sparse", False)) - for key, value in iteritems(info) + for key, value in info.items() ] assert ([("addDate", -1)], True, True) in info @@ -901,7 +900,7 @@ class TestIndexes(unittest.TestCase): self.fail("Unbound local error at index + pk definition") info = BlogPost.objects._collection.index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] index_item = [("_id", 1), ("comments.comment_id", 1)] assert index_item in info @@ -942,7 +941,7 @@ class TestIndexes(unittest.TestCase): meta = {"indexes": ["provider_ids.foo", "provider_ids.bar"]} info = MyDoc.objects._collection.index_information() - info = [value["key"] for key, value in iteritems(info)] + info = [value["key"] for key, value in info.items()] assert [("provider_ids.foo", 1)] in info assert [("provider_ids.bar", 1)] in info diff --git a/tests/document/test_inheritance.py b/tests/document/test_inheritance.py index 5072f841..0107c0a6 100644 --- a/tests/document/test_inheritance.py +++ b/tests/document/test_inheritance.py @@ -3,7 +3,6 @@ import unittest import warnings import pytest -from six import iteritems from mongoengine import ( BooleanField, @@ -550,7 +549,7 @@ class TestInheritance(MongoDBTestCase): class Human(Mammal): pass - for k, v in iteritems(defaults): + for k, v in defaults.items(): for cls in [Animal, Fish, Guppy]: assert cls._meta[k] == v diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index a5c21323..920bf392 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -10,7 +10,6 @@ import bson from bson import DBRef, ObjectId from pymongo.errors import DuplicateKeyError import pytest -from six import iteritems from mongoengine import * from mongoengine import signals @@ -3274,7 +3273,7 @@ class TestDocumentInstance(MongoDBTestCase): def expand(self): self.flattened_parameter = {} - for parameter_name, parameter in iteritems(self.parameters): + for parameter_name, parameter in self.parameters.items(): parameter.expand() class NodesSystem(Document): @@ -3282,7 +3281,7 @@ class TestDocumentInstance(MongoDBTestCase): nodes = MapField(ReferenceField(Node, dbref=False)) def save(self, *args, **kwargs): - for node_name, node in iteritems(self.nodes): + for node_name, node in self.nodes.items(): node.expand() node.save(*args, **kwargs) super(NodesSystem, self).save(*args, **kwargs) diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py index a653b961..a9c0c7e5 100644 --- a/tests/fields/test_binary_field.py +++ b/tests/fields/test_binary_field.py @@ -3,13 +3,12 @@ import uuid from bson import Binary import pytest -import six from mongoengine import * from tests.utils import MongoDBTestCase -BIN_VALUE = six.b( - "\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde\xe6\x87\xb1\x7f\xc6\xe6\xd9r\x18\xf5" +BIN_VALUE = "\xa9\xf3\x8d(\xd7\x03\x84\xb4k[\x0f\xe3\xa2\x19\x85p[J\xa3\xd2>\xde\xe6\x87\xb1\x7f\xc6\xe6\xd9r\x18\xf5".encode( + "latin-1" ) @@ -22,7 +21,7 @@ class TestBinaryField(MongoDBTestCase): content_type = StringField() blob = BinaryField() - BLOB = six.b("\xe6\x00\xc4\xff\x07") + BLOB = "\xe6\x00\xc4\xff\x07".encode("latin-1") MIME_TYPE = "application/octet-stream" Attachment.drop_collection() @@ -32,7 +31,7 @@ class TestBinaryField(MongoDBTestCase): attachment_1 = Attachment.objects().first() assert MIME_TYPE == attachment_1.content_type - assert BLOB == six.binary_type(attachment_1.blob) + assert BLOB == bytes(attachment_1.blob) def test_validation_succeeds(self): """Ensure that valid values can be assigned to binary fields. @@ -47,11 +46,11 @@ class TestBinaryField(MongoDBTestCase): attachment_required = AttachmentRequired() with pytest.raises(ValidationError): attachment_required.validate() - attachment_required.blob = Binary(six.b("\xe6\x00\xc4\xff\x07")) + attachment_required.blob = Binary("\xe6\x00\xc4\xff\x07".encode("latin-1")) attachment_required.validate() - _5_BYTES = six.b("\xe6\x00\xc4\xff\x07") - _4_BYTES = six.b("\xe6\x00\xc4\xff") + _5_BYTES = "\xe6\x00\xc4\xff\x07".encode("latin-1") + _4_BYTES = "\xe6\x00\xc4\xff".encode("latin-1") with pytest.raises(ValidationError): AttachmentSizeLimit(blob=_5_BYTES).validate() AttachmentSizeLimit(blob=_4_BYTES).validate() @@ -133,7 +132,7 @@ class TestBinaryField(MongoDBTestCase): MyDocument.drop_collection() - bin_data = six.b("\xe6\x00\xc4\xff\x07") + bin_data = "\xe6\x00\xc4\xff\x07".encode("latin-1") doc = MyDocument(bin_field=bin_data).save() n_updated = MyDocument.objects(bin_field=bin_data).update_one( diff --git a/tests/fields/test_date_field.py b/tests/fields/test_date_field.py index 5e1bfaa4..42a4b7f1 100644 --- a/tests/fields/test_date_field.py +++ b/tests/fields/test_date_field.py @@ -2,7 +2,6 @@ import datetime import pytest -import six try: import dateutil diff --git a/tests/fields/test_datetime_field.py b/tests/fields/test_datetime_field.py index 21ab523b..48936af7 100644 --- a/tests/fields/test_datetime_field.py +++ b/tests/fields/test_datetime_field.py @@ -2,7 +2,6 @@ import datetime as dt import pytest -import six try: import dateutil diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py index 5ab6c93f..cbac9b69 100644 --- a/tests/fields/test_file_field.py +++ b/tests/fields/test_file_field.py @@ -7,7 +7,6 @@ from io import BytesIO import gridfs import pytest -import six from mongoengine import * from mongoengine.connection import get_db @@ -58,7 +57,7 @@ class TestFileField(MongoDBTestCase): PutFile.drop_collection() - text = six.b("Hello, World!") + text = "Hello, World!".encode("latin-1") content_type = "text/plain" putfile = PutFile() @@ -101,8 +100,8 @@ class TestFileField(MongoDBTestCase): StreamFile.drop_collection() - text = six.b("Hello, World!") - more_text = six.b("Foo Bar") + text = "Hello, World!".encode("latin-1") + more_text = "Foo Bar".encode("latin-1") content_type = "text/plain" streamfile = StreamFile() @@ -137,8 +136,8 @@ class TestFileField(MongoDBTestCase): StreamFile.drop_collection() - text = six.b("Hello, World!") - more_text = six.b("Foo Bar") + text = "Hello, World!".encode("latin-1") + more_text = "Foo Bar".encode("latin-1") streamfile = StreamFile() streamfile.save() @@ -167,8 +166,8 @@ class TestFileField(MongoDBTestCase): class SetFile(Document): the_file = FileField() - text = six.b("Hello, World!") - more_text = six.b("Foo Bar") + text = "Hello, World!".encode("latin-1") + more_text = "Foo Bar".encode("latin-1") SetFile.drop_collection() @@ -196,7 +195,7 @@ class TestFileField(MongoDBTestCase): GridDocument.drop_collection() with tempfile.TemporaryFile() as f: - f.write(six.b("Hello World!")) + f.write("Hello World!".encode("latin-1")) f.flush() # Test without default @@ -213,7 +212,7 @@ class TestFileField(MongoDBTestCase): assert doc_b.the_file.grid_id == doc_c.the_file.grid_id # Test with default - doc_d = GridDocument(the_file=six.b("")) + doc_d = GridDocument(the_file="".encode("latin-1")) doc_d.save() doc_e = GridDocument.objects.with_id(doc_d.id) @@ -240,7 +239,7 @@ class TestFileField(MongoDBTestCase): # First instance test_file = TestFile() test_file.name = "Hello, World!" - test_file.the_file.put(six.b("Hello, World!")) + test_file.the_file.put("Hello, World!".encode("latin-1")) test_file.save() # Second instance @@ -297,7 +296,9 @@ class TestFileField(MongoDBTestCase): test_file = TestFile() assert not bool(test_file.the_file) - test_file.the_file.put(six.b("Hello, World!"), content_type="text/plain") + test_file.the_file.put( + "Hello, World!".encode("latin-1"), content_type="text/plain" + ) test_file.save() assert bool(test_file.the_file) @@ -319,7 +320,7 @@ class TestFileField(MongoDBTestCase): class TestFile(Document): the_file = FileField() - text = six.b("Hello, World!") + text = "Hello, World!".encode("latin-1") content_type = "text/plain" testfile = TestFile() @@ -363,7 +364,7 @@ class TestFileField(MongoDBTestCase): testfile.the_file.put(text, content_type=content_type, filename="hello") testfile.save() - text = six.b("Bonjour, World!") + text = "Bonjour, World!".encode("latin-1") testfile.the_file.replace(text, content_type=content_type, filename="hello") testfile.save() @@ -387,7 +388,7 @@ class TestFileField(MongoDBTestCase): TestImage.drop_collection() with tempfile.TemporaryFile() as f: - f.write(six.b("Hello World!")) + f.write("Hello World!".encode("latin-1")) f.flush() t = TestImage() @@ -503,21 +504,21 @@ class TestFileField(MongoDBTestCase): # First instance test_file = TestFile() test_file.name = "Hello, World!" - test_file.the_file.put(six.b("Hello, World!"), name="hello.txt") + test_file.the_file.put("Hello, World!".encode("latin-1"), name="hello.txt") test_file.save() data = get_db("test_files").macumba.files.find_one() assert data.get("name") == "hello.txt" test_file = TestFile.objects.first() - assert test_file.the_file.read() == six.b("Hello, World!") + assert test_file.the_file.read() == "Hello, World!".encode("latin-1") test_file = TestFile.objects.first() - test_file.the_file = six.b("HELLO, WORLD!") + test_file.the_file = "Hello, World!".encode("latin-1") test_file.save() test_file = TestFile.objects.first() - assert test_file.the_file.read() == six.b("HELLO, WORLD!") + assert test_file.the_file.read() == "Hello, World!".encode("latin-1") def test_copyable(self): class PutFile(Document): @@ -525,7 +526,7 @@ class TestFileField(MongoDBTestCase): PutFile.drop_collection() - text = six.b("Hello, World!") + text = "Hello, World!".encode("latin-1") content_type = "text/plain" putfile = PutFile() diff --git a/tests/fields/test_float_field.py b/tests/fields/test_float_field.py index a1cd7a0a..839494a9 100644 --- a/tests/fields/test_float_field.py +++ b/tests/fields/test_float_field.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import pytest -import six from mongoengine import * @@ -52,9 +51,8 @@ class TestFloatField(MongoDBTestCase): big_person = BigPerson() - for value, value_type in enumerate(six.integer_types): - big_person.height = value_type(value) - big_person.validate() + big_person.height = int(0) + big_person.validate() big_person.height = 2 ** 500 big_person.validate() diff --git a/tests/fields/test_long_field.py b/tests/fields/test_long_field.py index b39a714c..330051c3 100644 --- a/tests/fields/test_long_field.py +++ b/tests/fields/test_long_field.py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- -import pytest from bson.int64 import Int64 -import six +import pytest from mongoengine import * from mongoengine.connection import get_db @@ -24,7 +22,7 @@ class TestLongField(MongoDBTestCase): assert isinstance( db.test_long_field_considered_as_int64.find()[0]["some_long"], Int64 ) - assert isinstance(doc.some_long, six.integer_types) + assert isinstance(doc.some_long, int) def test_long_validation(self): """Ensure that invalid values cannot be assigned to long fields. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index cf4cd0dc..fac6dfeb 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -10,8 +10,6 @@ import pymongo from pymongo.read_preferences import ReadPreference from pymongo.results import UpdateResult import pytest -import six -from six import iteritems from mongoengine import * from mongoengine.connection import get_db @@ -4093,7 +4091,7 @@ class TestQueryset(unittest.TestCase): info = Comment.objects._collection.index_information() info = [ (value["key"], value.get("unique", False), value.get("sparse", False)) - for key, value in iteritems(info) + for key, value in info.items() ] assert ([("_cls", 1), ("message", 1)], False, False) in info diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py index 734061ed..6d432e32 100644 --- a/tests/test_datastructures.py +++ b/tests/test_datastructures.py @@ -1,7 +1,6 @@ import unittest import pytest -from six import iterkeys from mongoengine import Document from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict @@ -372,7 +371,7 @@ class TestStrictDict(unittest.TestCase): def test_iterkeys(self): d = self.dtype(a=1) - assert list(iterkeys(d)) == ["a"] + assert list(d.keys()) == ["a"] def test_len(self): d = self.dtype(a=1) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index b9d92883..0f9f412c 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -2,10 +2,8 @@ import unittest from bson import DBRef, ObjectId -from six import iteritems from mongoengine import * -from mongoengine.connection import get_db from mongoengine.context_managers import query_counter @@ -739,7 +737,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, User) # Document select_related @@ -752,7 +750,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, User) # Queryset select_related @@ -766,7 +764,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, User) User.drop_collection() @@ -820,7 +818,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ # Document select_related @@ -836,7 +834,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ # Queryset select_related @@ -853,7 +851,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ Group.objects.delete() @@ -910,7 +908,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, UserA) # Document select_related @@ -926,7 +924,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, UserA) # Queryset select_related @@ -943,7 +941,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 2 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert isinstance(m, UserA) UserA.drop_collection() @@ -997,7 +995,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ # Document select_related @@ -1013,7 +1011,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ # Queryset select_related @@ -1030,7 +1028,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] assert q == 4 - for k, m in iteritems(group_obj.members): + for k, m in group_obj.members.items(): assert "User" in m.__class__.__name__ Group.objects.delete() From b234aa48e4c17ac056afde75e43f06ef8f8200a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 11 Mar 2020 23:21:38 +0100 Subject: [PATCH 42/79] run pyupgrade --- mongoengine/base/datastructures.py | 18 ++++++++++-------- mongoengine/base/document.py | 26 +++++++++++++------------- mongoengine/base/fields.py | 6 ++++-- mongoengine/base/metaclasses.py | 21 ++------------------- mongoengine/connection.py | 2 +- mongoengine/dereference.py | 4 ++-- mongoengine/document.py | 2 +- mongoengine/errors.py | 8 ++++---- mongoengine/fields.py | 20 ++++++++++++-------- mongoengine/queryset/field_list.py | 2 +- mongoengine/queryset/transform.py | 4 +++- setup.py | 6 +----- 12 files changed, 54 insertions(+), 65 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index bb70089e..86fde15e 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -67,11 +67,11 @@ class BaseDict(dict): if isinstance(value, EmbeddedDocument) and value._instance is None: value._instance = self._instance elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, None, "%s.%s" % (self._name, key)) + value = BaseDict(value, None, "{}.{}".format(self._name, key)) super(BaseDict, self).__setitem__(key, value) value._instance = self._instance elif isinstance(value, list) and not isinstance(value, BaseList): - value = BaseList(value, None, "%s.%s" % (self._name, key)) + value = BaseList(value, None, "{}.{}".format(self._name, key)) super(BaseDict, self).__setitem__(key, value) value._instance = self._instance return value @@ -97,7 +97,7 @@ class BaseDict(dict): def _mark_as_changed(self, key=None): if hasattr(self._instance, "_mark_as_changed"): if key: - self._instance._mark_as_changed("%s.%s" % (self._name, key)) + self._instance._mark_as_changed("{}.{}".format(self._name, key)) else: self._instance._mark_as_changed(self._name) @@ -133,12 +133,12 @@ class BaseList(list): value._instance = self._instance elif isinstance(value, dict) and not isinstance(value, BaseDict): # Replace dict by BaseDict - value = BaseDict(value, None, "%s.%s" % (self._name, key)) + value = BaseDict(value, None, "{}.{}".format(self._name, key)) super(BaseList, self).__setitem__(key, value) value._instance = self._instance elif isinstance(value, list) and not isinstance(value, BaseList): # Replace list by BaseList - value = BaseList(value, None, "%s.%s" % (self._name, key)) + value = BaseList(value, None, "{}.{}".format(self._name, key)) super(BaseList, self).__setitem__(key, value) value._instance = self._instance return value @@ -181,7 +181,9 @@ class BaseList(list): def _mark_as_changed(self, key=None): if hasattr(self._instance, "_mark_as_changed"): if key: - self._instance._mark_as_changed("%s.%s" % (self._name, key % len(self))) + self._instance._mark_as_changed( + "{}.{}".format(self._name, key % len(self)) + ) else: self._instance._mark_as_changed(self._name) @@ -428,7 +430,7 @@ class StrictDict(object): def __repr__(self): return "{%s}" % ", ".join( - '"{0!s}": {1!r}'.format(k, v) for k, v in self.items() + '"{!s}": {!r}'.format(k, v) for k, v in self.items() ) cls._classes[allowed_keys] = SpecificStrictDict @@ -473,4 +475,4 @@ class LazyReference(DBRef): raise AttributeError() def __repr__(self): - return "" % (self.document_type, self.pk) + return "".format(self.document_type, self.pk) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index dff759a7..83c12a96 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -93,7 +93,7 @@ class BaseDocument(object): list(self._fields.keys()) + ["id", "pk", "_cls", "_text_score"] ) if _undefined_fields: - msg = ('The fields "{0}" do not exist on the document "{1}"').format( + msg = ('The fields "{}" do not exist on the document "{}"').format( _undefined_fields, self._class_name ) raise FieldDoesNotExist(msg) @@ -286,7 +286,7 @@ class BaseDocument(object): except (UnicodeEncodeError, UnicodeDecodeError): u = "[Bad Unicode data]" repr_type = str if u is None else type(u) - return repr_type("<%s: %s>" % (self.__class__.__name__, u)) + return repr_type("<{}: {}>".format(self.__class__.__name__, u)) def __str__(self): # TODO this could be simpler? @@ -441,7 +441,7 @@ class BaseDocument(object): pk = self.pk elif self._instance and hasattr(self._instance, "pk"): pk = self._instance.pk - message = "ValidationError (%s:%s) " % (self._class_name, pk) + message = "ValidationError ({}:{}) ".format(self._class_name, pk) raise ValidationError(message, errors=errors) def to_json(self, *args, **kwargs): @@ -514,7 +514,7 @@ class BaseDocument(object): if "." in key: key, rest = key.split(".", 1) key = self._db_field_map.get(key, key) - key = "%s.%s" % (key, rest) + key = "{}.{}".format(key, rest) else: key = self._db_field_map.get(key, key) @@ -576,7 +576,7 @@ class BaseDocument(object): iterator = data.items() for index_or_key, value in iterator: - item_key = "%s%s." % (base_key, index_or_key) + item_key = "{}{}.".format(base_key, index_or_key) # don't check anything lower if this key is already marked # as changed. if item_key[:-1] in changed_fields: @@ -584,7 +584,7 @@ class BaseDocument(object): if hasattr(value, "_get_changed_fields"): changed = value._get_changed_fields() - changed_fields += ["%s%s" % (item_key, k) for k in changed if k] + changed_fields += ["{}{}".format(item_key, k) for k in changed if k] elif isinstance(value, (list, tuple, dict)): self._nestable_types_changed_fields(changed_fields, item_key, value) @@ -615,7 +615,7 @@ class BaseDocument(object): if isinstance(data, EmbeddedDocument): # Find all embedded fields that have been changed changed = data._get_changed_fields() - changed_fields += ["%s%s" % (key, k) for k in changed if k] + changed_fields += ["{}{}".format(key, k) for k in changed if k] elif isinstance(data, (list, tuple, dict)): if hasattr(field, "field") and isinstance( field.field, (ReferenceField, GenericReferenceField) @@ -769,11 +769,10 @@ class BaseDocument(object): if errors_dict: errors = "\n".join( - ["Field '%s' - %s" % (k, v) for k, v in errors_dict.items()] + ["Field '{}' - {}".format(k, v) for k, v in errors_dict.items()] ) - msg = "Invalid data to create a `%s` instance.\n%s" % ( - cls._class_name, - errors, + msg = "Invalid data to create a `{}` instance.\n{}".format( + cls._class_name, errors, ) raise InvalidDocumentError(msg) @@ -944,7 +943,8 @@ class BaseDocument(object): # Add the new index to the list fields = [ - ("%s%s" % (namespace, f), pymongo.ASCENDING) for f in unique_fields + ("{}{}".format(namespace, f), pymongo.ASCENDING) + for f in unique_fields ] index = {"fields": fields, "unique": True, "sparse": sparse} unique_indexes.append(index) @@ -1001,7 +1001,7 @@ class BaseDocument(object): elif field._geo_index: field_name = field.db_field if parent_field: - field_name = "%s.%s" % (parent_field, field_name) + field_name = "{}.{}".format(parent_field, field_name) geo_indices.append({"fields": [(field_name, field._geo_index)]}) return geo_indices diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index ac894d91..2fadcfe1 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -474,7 +474,9 @@ class ComplexBaseField(BaseField): if errors: field_class = self.field.__class__.__name__ - self.error("Invalid %s item (%s)" % (field_class, value), errors=errors) + self.error( + "Invalid {} item ({})".format(field_class, value), errors=errors + ) # Don't allow empty values if required if self.required and not value: self.error("Field is required and cannot be empty") @@ -546,7 +548,7 @@ class GeoJsonBaseField(BaseField): if isinstance(value, dict): if set(value.keys()) == {"type", "coordinates"}: if value["type"] != self._type: - self.error('%s type must be "%s"' % (self._name, self._type)) + self.error('{} type must be "{}"'.format(self._name, self._type)) return self.validate(value["coordinates"]) else: self.error( diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 30a6fbab..473e6b18 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -168,23 +168,6 @@ class DocumentMetaclass(type): # Add class to the _document_registry _document_registry[new_class._class_name] = new_class - # In Python 2, User-defined methods objects have special read-only - # attributes 'im_func' and 'im_self' which contain the function obj - # and class instance object respectively. With Python 3 these special - # attributes have been replaced by __func__ and __self__. The Blinker - # module continues to use im_func and im_self, so the code below - # copies __func__ into im_func and __self__ into im_self for - # classmethod objects in Document derived classes. - # - # Relates to https://github.com/MongoEngine/mongoengine/issues/1107 - # for val in new_class.__dict__.values(): - # if isinstance(val, classmethod): - # f = val.__get__(new_class) - # if hasattr(f, "__func__") and not hasattr(f, "im_func"): - # f.__dict__.update({"im_func": getattr(f, "__func__")}) - # if hasattr(f, "__self__") and not hasattr(f, "im_self"): - # f.__dict__.update({"im_self": getattr(f, "__self__")}) - # Handle delete rules for field in new_class._fields.values(): f = field @@ -458,8 +441,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_basename, id_db_basename, i = ("auto_id", "_auto_id", 0) for i in itertools.count(): - id_name = "{0}_{1}".format(id_basename, i) - id_db_name = "{0}_{1}".format(id_db_basename, i) + id_name = "{}_{}".format(id_basename, i) + id_db_name = "{}_{}".format(id_db_basename, i) if id_name not in existing_fields and id_db_name not in existing_db_fields: return id_name, id_db_name diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 3f754619..b03e0b1d 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -317,7 +317,7 @@ def _create_connection(alias, connection_class, **connection_settings): try: return connection_class(**connection_settings) except Exception as e: - raise ConnectionFailure("Cannot connect to database %s :\n%s" % (alias, e)) + raise ConnectionFailure("Cannot connect to database {} :\n{}".format(alias, e)) def _find_existing_connection(connection_settings): diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index d0e6c527..3756d84a 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -272,12 +272,12 @@ class DeReference(object): (v["_ref"].collection, v["_ref"].id), v ) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: - item_name = "{0}.{1}.{2}".format(name, k, field_name) + item_name = "{}.{}.{}".format(name, k, field_name) data[k]._data[field_name] = self._attach_objects( v, depth, instance=instance, name=item_name ) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: - item_name = "%s.%s" % (name, k) if name else name + item_name = "{}.{}".format(name, k) if name else name data[k] = self._attach_objects( v, depth - 1, instance=instance, name=item_name ) diff --git a/mongoengine/document.py b/mongoengine/document.py index 4ac979d1..e13918c2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -555,7 +555,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): if not getattr(ref, "_changed_fields", True): continue - ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) + ref_id = "{},{}".format(ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: _refs.append(ref_id) kwargs["_refs"] = _refs diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 7045145c..76c25773 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -94,7 +94,7 @@ class ValidationError(AssertionError): return str(self.message) def __repr__(self): - return "%s(%s,)" % (self.__class__.__name__, self.message) + return "{}({},)".format(self.__class__.__name__, self.message) def __getattribute__(self, name): message = super(ValidationError, self).__getattribute__(name) @@ -102,7 +102,7 @@ class ValidationError(AssertionError): if self.field_name: message = "%s" % message if self.errors: - message = "%s(%s)" % (message, self._format_errors()) + message = "{}({})".format(message, self._format_errors()) return message def _get_message(self): @@ -147,13 +147,13 @@ class ValidationError(AssertionError): elif isinstance(value, dict): value = " ".join([generate_key(v, k) for k, v in value.items()]) - results = "%s.%s" % (prefix, value) if prefix else value + results = "{}.{}".format(prefix, value) if prefix else value return results error_dict = defaultdict(list) for k, v in self.to_dict().items(): error_dict[generate_key(v)].append(k) - return " ".join(["%s: %s" % (k, v) for k, v in error_dict.items()]) + return " ".join(["{}: {}".format(k, v) for k, v in error_dict.items()]) class DeprecatedError(Exception): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 391ad37b..91f85d26 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -292,12 +292,16 @@ class EmailField(StringField): domain_part = domain_part.encode("idna").decode("ascii") except UnicodeError: self.error( - "%s %s" % (self.error_msg % value, "(domain failed IDN encoding)") + "{} {}".format( + self.error_msg % value, "(domain failed IDN encoding)" + ) ) else: if not self.validate_domain_part(domain_part): self.error( - "%s %s" % (self.error_msg % value, "(domain validation failed)") + "{} {}".format( + self.error_msg % value, "(domain validation failed)" + ) ) @@ -1344,7 +1348,7 @@ class CachedReferenceField(BaseField): return None update_kwargs = { - "set__%s__%s" % (self.name, key): val + "set__{}__{}".format(self.name, key): val for key, val in document._delta()[0].items() if key in self.fields } @@ -1688,12 +1692,12 @@ class GridFSProxy(object): return self.__copy__() def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.grid_id) + return "<{}: {}>".format(self.__class__.__name__, self.grid_id) def __str__(self): gridout = self.get() filename = getattr(gridout, "filename") if gridout else "" - return "<%s: %s (%s)>" % (self.__class__.__name__, filename, self.grid_id) + return "<{}: {} ({})>".format(self.__class__.__name__, filename, self.grid_id) def __eq__(self, other): if isinstance(other, GridFSProxy): @@ -2097,7 +2101,7 @@ class SequenceField(BaseField): Generate and Increment the counter """ sequence_name = self.get_sequence_name() - sequence_id = "%s.%s" % (sequence_name, self.name) + sequence_id = "{}.{}".format(sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_one_and_update( @@ -2111,7 +2115,7 @@ class SequenceField(BaseField): def set_next_value(self, value): """Helper method to set the next sequence value""" sequence_name = self.get_sequence_name() - sequence_id = "%s.%s" % (sequence_name, self.name) + sequence_id = "{}.{}".format(sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_one_and_update( filter={"_id": sequence_id}, @@ -2128,7 +2132,7 @@ class SequenceField(BaseField): as it is only fixed on set. """ sequence_name = self.get_sequence_name() - sequence_id = "%s.%s" % (sequence_name, self.name) + sequence_id = "{}.{}".format(sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] data = collection.find_one({"_id": sequence_id}) diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index c2618ebd..e0d8e322 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -78,7 +78,7 @@ class QueryFieldList(object): return field_list def reset(self): - self.fields = set([]) + self.fields = set() self.slice = {} self.value = self.ONLY diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 1202ec45..3f1db8fa 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -433,7 +433,9 @@ def _geo_operator(field, op, value): value = {"$near": _infer_geometry(value)} else: raise NotImplementedError( - 'Geo method "%s" has not been implemented for a %s ' % (op, field._name) + 'Geo method "{}" has not been implemented for a {} '.format( + op, field._name + ) ) return value diff --git a/setup.py b/setup.py index b60188c7..fe3253ae 100644 --- a/setup.py +++ b/setup.py @@ -108,9 +108,6 @@ CLASSIFIERS = [ "Topic :: Software Development :: Libraries :: Python Modules", ] -PYTHON_VERSION = sys.version_info[0] -PY3 = PYTHON_VERSION == 3 - extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), "tests_require": [ @@ -118,8 +115,7 @@ extra_opts = { "pytest-cov", "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", - "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support - "zipp<2.0.0", # (dependency of pytest) dropped python2 support + "Pillow>=2.0.0", ], } From 1e110a2c41d2e4675d0195fafa00e49dcd331675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 11 Mar 2020 23:26:37 +0100 Subject: [PATCH 43/79] run pyupgrade --py3-plus --- mongoengine/base/datastructures.py | 29 +++--- mongoengine/base/document.py | 8 +- mongoengine/base/fields.py | 6 +- mongoengine/base/metaclasses.py | 7 +- mongoengine/base/utils.py | 2 +- mongoengine/connection.py | 4 +- mongoengine/context_managers.py | 12 +-- mongoengine/dereference.py | 2 +- mongoengine/document.py | 26 +++--- mongoengine/errors.py | 4 +- mongoengine/fields.py | 144 ++++++++++++++--------------- mongoengine/queryset/base.py | 16 ++-- mongoengine/queryset/field_list.py | 2 +- mongoengine/queryset/manager.py | 2 +- mongoengine/queryset/queryset.py | 4 +- mongoengine/queryset/visitor.py | 4 +- mongoengine/signals.py | 4 +- setup.py | 2 +- 18 files changed, 134 insertions(+), 144 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 86fde15e..d3bff2b3 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -51,7 +51,7 @@ class BaseDict(dict): if isinstance(instance, BaseDocument): self._instance = weakref.proxy(instance) self._name = name - super(BaseDict, self).__init__(dict_items) + super().__init__(dict_items) def get(self, key, default=None): # get does not use __getitem__ by default so we must override it as well @@ -61,18 +61,18 @@ class BaseDict(dict): return default def __getitem__(self, key): - value = super(BaseDict, self).__getitem__(key) + value = super().__getitem__(key) EmbeddedDocument = _import_class("EmbeddedDocument") if isinstance(value, EmbeddedDocument) and value._instance is None: value._instance = self._instance elif isinstance(value, dict) and not isinstance(value, BaseDict): value = BaseDict(value, None, "{}.{}".format(self._name, key)) - super(BaseDict, self).__setitem__(key, value) + super().__setitem__(key, value) value._instance = self._instance elif isinstance(value, list) and not isinstance(value, BaseList): value = BaseList(value, None, "{}.{}".format(self._name, key)) - super(BaseDict, self).__setitem__(key, value) + super().__setitem__(key, value) value._instance = self._instance return value @@ -115,13 +115,13 @@ class BaseList(list): if isinstance(instance, BaseDocument): self._instance = weakref.proxy(instance) self._name = name - super(BaseList, self).__init__(list_items) + super().__init__(list_items) def __getitem__(self, key): # change index to positive value because MongoDB does not support negative one if isinstance(key, int) and key < 0: key = len(self) + key - value = super(BaseList, self).__getitem__(key) + value = super().__getitem__(key) if isinstance(key, slice): # When receiving a slice operator, we don't convert the structure and bind @@ -134,18 +134,17 @@ class BaseList(list): elif isinstance(value, dict) and not isinstance(value, BaseDict): # Replace dict by BaseDict value = BaseDict(value, None, "{}.{}".format(self._name, key)) - super(BaseList, self).__setitem__(key, value) + super().__setitem__(key, value) value._instance = self._instance elif isinstance(value, list) and not isinstance(value, BaseList): # Replace list by BaseList value = BaseList(value, None, "{}.{}".format(self._name, key)) - super(BaseList, self).__setitem__(key, value) + super().__setitem__(key, value) value._instance = self._instance return value def __iter__(self): - for v in super(BaseList, self).__iter__(): - yield v + yield from super().__iter__() def __getstate__(self): self.instance = None @@ -163,7 +162,7 @@ class BaseList(list): # instead, we simply marks the whole list as changed changed_key = None - result = super(BaseList, self).__setitem__(key, value) + result = super().__setitem__(key, value) self._mark_as_changed(changed_key) return result @@ -190,7 +189,7 @@ class BaseList(list): class EmbeddedDocumentList(BaseList): def __init__(self, list_items, instance, name): - super(EmbeddedDocumentList, self).__init__(list_items, instance, name) + super().__init__(list_items, instance, name) self._instance = instance @classmethod @@ -355,7 +354,7 @@ class EmbeddedDocumentList(BaseList): return len(values) -class StrictDict(object): +class StrictDict: __slots__ = () _special_fields = {"get", "pop", "iteritems", "items", "keys", "create"} _classes = {} @@ -455,9 +454,7 @@ class LazyReference(DBRef): self.document_type = document_type self._cached_doc = cached_doc self.passthrough = passthrough - super(LazyReference, self).__init__( - self.document_type._get_collection_name(), pk - ) + super().__init__(self.document_type._get_collection_name(), pk) def __getitem__(self, name): if not self.passthrough: diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 83c12a96..e697fe40 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -30,7 +30,7 @@ __all__ = ("BaseDocument", "NON_FIELD_ERRORS") NON_FIELD_ERRORS = "__all__" -class BaseDocument(object): +class BaseDocument: # TODO simplify how `_changed_fields` is used. # Currently, handling of `_changed_fields` seems unnecessarily convoluted: # 1. `BaseDocument` defines `_changed_fields` in its `__slots__`, yet it's @@ -161,7 +161,7 @@ class BaseDocument(object): default = default() setattr(self, field_name, default) else: - super(BaseDocument, self).__delattr__(*args, **kwargs) + super().__delattr__(*args, **kwargs) def __setattr__(self, name, value): # Handle dynamic data only if an initialised dynamic document @@ -208,9 +208,9 @@ class BaseDocument(object): and self__created and name == self._meta.get("id_field") ): - super(BaseDocument, self).__setattr__("_created", False) + super().__setattr__("_created", False) - super(BaseDocument, self).__setattr__(name, value) + super().__setattr__(name, value) def __getstate__(self): data = {} diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 2fadcfe1..e44b5744 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -13,7 +13,7 @@ from mongoengine.errors import DeprecatedError, ValidationError __all__ = ("BaseField", "ComplexBaseField", "ObjectIdField", "GeoJsonBaseField") -class BaseField(object): +class BaseField: """A base class for fields in a MongoDB document. Instances of this class may be added to subclasses of `Document` to define a document's schema. @@ -310,7 +310,7 @@ class ComplexBaseField(BaseField): if hasattr(instance._data[self.name], "_dereferenced"): instance._data[self.name]._dereferenced = True - value = super(ComplexBaseField, self).__get__(instance, owner) + value = super().__get__(instance, owner) # Convert lists / values so we can watch for any changes on them if isinstance(value, (list, tuple)): @@ -541,7 +541,7 @@ class GeoJsonBaseField(BaseField): self._name = "%sField" % self._type if not auto_index: self._geo_index = False - super(GeoJsonBaseField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def validate(self, value): """Validate the GeoJson object based on its type.""" diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 473e6b18..ce24ff58 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -22,7 +22,7 @@ class DocumentMetaclass(type): # TODO lower complexity of this method def __new__(mcs, name, bases, attrs): flattened_bases = mcs._get_bases(bases) - super_new = super(DocumentMetaclass, mcs).__new__ + super_new = super().__new__ # If a base class just call super metaclass = attrs.get("my_metaclass") @@ -231,8 +231,7 @@ class DocumentMetaclass(type): if base is object: continue yield base - for child_base in mcs.__get_bases(base.__bases__): - yield child_base + yield from mcs.__get_bases(base.__bases__) @classmethod def _import_classes(mcs): @@ -250,7 +249,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): def __new__(mcs, name, bases, attrs): flattened_bases = mcs._get_bases(bases) - super_new = super(TopLevelDocumentMetaclass, mcs).__new__ + super_new = super().__new__ # Set default _meta data if base class, otherwise get user defined meta if attrs.get("my_metaclass") == TopLevelDocumentMetaclass: diff --git a/mongoengine/base/utils.py b/mongoengine/base/utils.py index 8f27ee14..7753ad50 100644 --- a/mongoengine/base/utils.py +++ b/mongoengine/base/utils.py @@ -1,7 +1,7 @@ import re -class LazyRegexCompiler(object): +class LazyRegexCompiler: """Descriptor to allow lazy compilation of regex""" def __init__(self, pattern, flags=0): diff --git a/mongoengine/connection.py b/mongoengine/connection.py index b03e0b1d..13d170ec 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -395,8 +395,8 @@ def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs): if new_conn_settings != prev_conn_setting: err_msg = ( - u"A different connection with alias `{}` was already " - u"registered. Use disconnect() first" + "A different connection with alias `{}` was already " + "registered. Use disconnect() first" ).format(alias) raise ConnectionFailure(err_msg) else: diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 5e9a6e8b..8bfb902b 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -16,7 +16,7 @@ __all__ = ( ) -class switch_db(object): +class switch_db: """switch_db alias context manager. Example :: @@ -57,7 +57,7 @@ class switch_db(object): self.cls._collection = self.collection -class switch_collection(object): +class switch_collection: """switch_collection alias context manager. Example :: @@ -99,7 +99,7 @@ class switch_collection(object): self.cls._get_collection_name = self.ori_get_collection_name -class no_dereference(object): +class no_dereference: """no_dereference context manager. Turns off all dereferencing in Documents for the duration of the context @@ -139,7 +139,7 @@ class no_dereference(object): return self.cls -class no_sub_classes(object): +class no_sub_classes: """no_sub_classes context manager. Only returns instances of this class and no sub (inherited) classes:: @@ -167,7 +167,7 @@ class no_sub_classes(object): self.cls._subclasses = self.cls_initial_subclasses -class query_counter(object): +class query_counter: """Query_counter context manager to get the number of queries. This works by updating the `profiling_level` of the database so that all queries get logged, resetting the db.system.profile collection at the beginning of the context and counting the new entries. @@ -234,7 +234,7 @@ class query_counter(object): def __repr__(self): """repr query_counter as the number of queries.""" - return u"%s" % self._get_count() + return "%s" % self._get_count() def _get_count(self): """Get the number of queries by counting the current number of entries in db.system.profile diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 3756d84a..ff608a3b 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -14,7 +14,7 @@ from mongoengine.fields import DictField, ListField, MapField, ReferenceField from mongoengine.queryset import QuerySet -class DeReference(object): +class DeReference: def __call__(self, items, max_depth=1, instance=None, name=None): """ Cheaply dereferences the items to a set depth. diff --git a/mongoengine/document.py b/mongoengine/document.py index e13918c2..9166a959 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -79,7 +79,7 @@ class EmbeddedDocument(BaseDocument, metaclass=DocumentMetaclass): __hash__ = None def __init__(self, *args, **kwargs): - super(EmbeddedDocument, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._instance = None self._changed_fields = [] @@ -92,7 +92,7 @@ class EmbeddedDocument(BaseDocument, metaclass=DocumentMetaclass): return not self.__eq__(other) def to_mongo(self, *args, **kwargs): - data = super(EmbeddedDocument, self).to_mongo(*args, **kwargs) + data = super().to_mongo(*args, **kwargs) # remove _id from the SON if it's in it and it's None if "_id" in data and data["_id"] is None: @@ -256,7 +256,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): return db.create_collection(collection_name, **opts) def to_mongo(self, *args, **kwargs): - data = super(Document, self).to_mongo(*args, **kwargs) + data = super().to_mongo(*args, **kwargs) # If '_id' is None, try and set it from self._data. If that # doesn't exist either, remove '_id' from the SON completely. @@ -427,14 +427,14 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): self.cascade_save(**kwargs) except pymongo.errors.DuplicateKeyError as err: - message = u"Tried to save duplicate unique keys (%s)" + message = "Tried to save duplicate unique keys (%s)" raise NotUniqueError(message % err) except pymongo.errors.OperationFailure as err: message = "Could not save document (%s)" if re.match("^E1100[01] duplicate key", str(err)): # E11000 - duplicate key error index # E11001 - duplicate key on update - message = u"Tried to save duplicate unique keys (%s)" + message = "Tried to save duplicate unique keys (%s)" raise NotUniqueError(message % err) raise OperationError(message % err) @@ -639,7 +639,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): write_concern=write_concern, _from_doc_delete=True ) except pymongo.errors.OperationFailure as err: - message = u"Could not delete document (%s)" % err.message + message = "Could not delete document (%s)" % err.message raise OperationError(message) signals.post_delete.send(self.__class__, document=self, **signal_kwargs) @@ -988,10 +988,10 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): indexes.append(index) # finish up by appending { '_id': 1 } and { '_cls': 1 }, if needed - if [(u"_id", 1)] not in indexes: - indexes.append([(u"_id", 1)]) + if [("_id", 1)] not in indexes: + indexes.append([("_id", 1)]) if cls._meta.get("index_cls", True) and cls._meta.get("allow_inheritance"): - indexes.append([(u"_cls", 1)]) + indexes.append([("_cls", 1)]) return indexes @@ -1015,14 +1015,14 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): extra = [index for index in existing if index not in required] # if { _cls: 1 } is missing, make sure it's *really* necessary - if [(u"_cls", 1)] in missing: + if [("_cls", 1)] in missing: cls_obsolete = False for index in existing: if includes_cls(index) and index not in extra: cls_obsolete = True break if cls_obsolete: - missing.remove([(u"_cls", 1)]) + missing.remove([("_cls", 1)]) return {"missing": missing, "extra": extra} @@ -1055,7 +1055,7 @@ class DynamicDocument(Document, metaclass=TopLevelDocumentMetaclass): setattr(self, field_name, None) self._dynamic_fields[field_name].null = False else: - super(DynamicDocument, self).__delattr__(*args, **kwargs) + super().__delattr__(*args, **kwargs) class DynamicEmbeddedDocument(EmbeddedDocument, metaclass=DocumentMetaclass): @@ -1083,7 +1083,7 @@ class DynamicEmbeddedDocument(EmbeddedDocument, metaclass=DocumentMetaclass): setattr(self, field_name, None) -class MapReduceDocument(object): +class MapReduceDocument: """A document returned from a map/reduce query. :param collection: An instance of :class:`~pymongo.Collection` diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 76c25773..95564ff9 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -85,7 +85,7 @@ class ValidationError(AssertionError): _message = None def __init__(self, message="", **kwargs): - super(ValidationError, self).__init__(message) + super().__init__(message) self.errors = kwargs.get("errors", {}) self.field_name = kwargs.get("field_name") self.message = message @@ -97,7 +97,7 @@ class ValidationError(AssertionError): return "{}({},)".format(self.__class__.__name__, self.message) def __getattribute__(self, name): - message = super(ValidationError, self).__getattribute__(name) + message = super().__getattribute__(name) if name == "message": if self.field_name: message = "%s" % message diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 91f85d26..b4cf4d25 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -103,7 +103,7 @@ class StringField(BaseField): self.regex = re.compile(regex) if regex else None self.max_length = max_length self.min_length = min_length - super(StringField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): if isinstance(value, str): @@ -151,7 +151,7 @@ class StringField(BaseField): # escape unsafe characters which could lead to a re.error value = re.escape(value) value = re.compile(regex % value, flags) - return super(StringField, self).prepare_query_value(op, value) + return super().prepare_query_value(op, value) class URLField(StringField): @@ -175,17 +175,17 @@ class URLField(StringField): def __init__(self, url_regex=None, schemes=None, **kwargs): self.url_regex = url_regex or self._URL_REGEX self.schemes = schemes or self._URL_SCHEMES - super(URLField, self).__init__(**kwargs) + super().__init__(**kwargs) def validate(self, value): # Check first if the scheme is valid scheme = value.split("://")[0].lower() if scheme not in self.schemes: - self.error(u"Invalid scheme {} in URL: {}".format(scheme, value)) + self.error("Invalid scheme {} in URL: {}".format(scheme, value)) # Then check full URL if not self.url_regex.match(value): - self.error(u"Invalid URL: {}".format(value)) + self.error("Invalid URL: {}".format(value)) class EmailField(StringField): @@ -218,7 +218,7 @@ class EmailField(StringField): re.IGNORECASE, ) - error_msg = u"Invalid email address: %s" + error_msg = "Invalid email address: %s" def __init__( self, @@ -242,7 +242,7 @@ class EmailField(StringField): self.domain_whitelist = domain_whitelist or [] self.allow_utf8_user = allow_utf8_user self.allow_ip_domain = allow_ip_domain - super(EmailField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def validate_user_part(self, user_part): """Validate the user part of the email address. Return True if @@ -269,13 +269,13 @@ class EmailField(StringField): try: socket.inet_pton(addr_family, domain_part[1:-1]) return True - except (socket.error, UnicodeEncodeError): + except (OSError, UnicodeEncodeError): pass return False def validate(self, value): - super(EmailField, self).validate(value) + super().validate(value) if "@" not in value: self.error(self.error_msg % value) @@ -310,7 +310,7 @@ class IntField(BaseField): def __init__(self, min_value=None, max_value=None, **kwargs): self.min_value, self.max_value = min_value, max_value - super(IntField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): try: @@ -335,7 +335,7 @@ class IntField(BaseField): if value is None: return value - return super(IntField, self).prepare_query_value(op, int(value)) + return super().prepare_query_value(op, int(value)) class LongField(BaseField): @@ -343,7 +343,7 @@ class LongField(BaseField): def __init__(self, min_value=None, max_value=None, **kwargs): self.min_value, self.max_value = min_value, max_value - super(LongField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): try: @@ -371,7 +371,7 @@ class LongField(BaseField): if value is None: return value - return super(LongField, self).prepare_query_value(op, int(value)) + return super().prepare_query_value(op, int(value)) class FloatField(BaseField): @@ -379,7 +379,7 @@ class FloatField(BaseField): def __init__(self, min_value=None, max_value=None, **kwargs): self.min_value, self.max_value = min_value, max_value - super(FloatField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): try: @@ -408,7 +408,7 @@ class FloatField(BaseField): if value is None: return value - return super(FloatField, self).prepare_query_value(op, float(value)) + return super().prepare_query_value(op, float(value)) class DecimalField(BaseField): @@ -455,7 +455,7 @@ class DecimalField(BaseField): self.precision = precision self.rounding = rounding - super(DecimalField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): if value is None: @@ -493,7 +493,7 @@ class DecimalField(BaseField): self.error("Decimal value is too large") def prepare_query_value(self, op, value): - return super(DecimalField, self).prepare_query_value(op, self.to_mongo(value)) + return super().prepare_query_value(op, self.to_mongo(value)) class BooleanField(BaseField): @@ -533,7 +533,7 @@ class DateTimeField(BaseField): def validate(self, value): new_value = self.to_mongo(value) if not isinstance(new_value, (datetime.datetime, datetime.date)): - self.error(u'cannot parse date "%s"' % value) + self.error('cannot parse date "%s"' % value) def to_mongo(self, value): if value is None: @@ -590,19 +590,19 @@ class DateTimeField(BaseField): return None def prepare_query_value(self, op, value): - return super(DateTimeField, self).prepare_query_value(op, self.to_mongo(value)) + return super().prepare_query_value(op, self.to_mongo(value)) class DateField(DateTimeField): def to_mongo(self, value): - value = super(DateField, self).to_mongo(value) + value = super().to_mongo(value) # drop hours, minutes, seconds if isinstance(value, datetime.datetime): value = datetime.datetime(value.year, value.month, value.day) return value def to_python(self, value): - value = super(DateField, self).to_python(value) + value = super().to_python(value) # convert datetime to date if isinstance(value, datetime.datetime): value = datetime.date(value.year, value.month, value.day) @@ -636,7 +636,7 @@ class ComplexDateTimeField(StringField): """ self.separator = separator self.format = separator.join(["%Y", "%m", "%d", "%H", "%M", "%S", "%f"]) - super(ComplexDateTimeField, self).__init__(**kwargs) + super().__init__(**kwargs) def _convert_from_datetime(self, val): """ @@ -667,14 +667,14 @@ class ComplexDateTimeField(StringField): if instance is None: return self - data = super(ComplexDateTimeField, self).__get__(instance, owner) + data = super().__get__(instance, owner) if isinstance(data, datetime.datetime) or data is None: return data return self._convert_from_string(data) def __set__(self, instance, value): - super(ComplexDateTimeField, self).__set__(instance, value) + super().__set__(instance, value) value = instance._data[self.name] if value is not None: instance._data[self.name] = self._convert_from_datetime(value) @@ -696,9 +696,7 @@ class ComplexDateTimeField(StringField): return self._convert_from_datetime(value) def prepare_query_value(self, op, value): - return super(ComplexDateTimeField, self).prepare_query_value( - op, self._convert_from_datetime(value) - ) + return super().prepare_query_value(op, self._convert_from_datetime(value)) class EmbeddedDocumentField(BaseField): @@ -718,7 +716,7 @@ class EmbeddedDocumentField(BaseField): ) self.document_type_obj = document_type - super(EmbeddedDocumentField, self).__init__(**kwargs) + super().__init__(**kwargs) @property def document_type(self): @@ -779,7 +777,7 @@ class EmbeddedDocumentField(BaseField): "Querying the embedded document '%s' failed, due to an invalid query value" % (self.document_type._class_name,) ) - super(EmbeddedDocumentField, self).prepare_query_value(op, value) + super().prepare_query_value(op, value) return self.to_mongo(value) @@ -795,9 +793,7 @@ class GenericEmbeddedDocumentField(BaseField): """ def prepare_query_value(self, op, value): - return super(GenericEmbeddedDocumentField, self).prepare_query_value( - op, self.to_mongo(value) - ) + return super().prepare_query_value(op, self.to_mongo(value)) def to_python(self, value): if isinstance(value, dict): @@ -885,7 +881,7 @@ class DynamicField(BaseField): value = doc_cls._get_db().dereference(value["_ref"]) return doc_cls._from_son(value) - return super(DynamicField, self).to_python(value) + return super().to_python(value) def lookup_member(self, member_name): return member_name @@ -893,7 +889,7 @@ class DynamicField(BaseField): def prepare_query_value(self, op, value): if isinstance(value, str): return StringField().prepare_query_value(op, value) - return super(DynamicField, self).prepare_query_value(op, self.to_mongo(value)) + return super().prepare_query_value(op, self.to_mongo(value)) def validate(self, value, clean=True): if hasattr(value, "validate"): @@ -914,7 +910,7 @@ class ListField(ComplexBaseField): self.field = field self.max_length = max_length kwargs.setdefault("default", lambda: []) - super(ListField, self).__init__(**kwargs) + super().__init__(**kwargs) def __get__(self, instance, owner): if instance is None: @@ -928,7 +924,7 @@ class ListField(ComplexBaseField): and value ): instance._data[self.name] = [self.field.build_lazyref(x) for x in value] - return super(ListField, self).__get__(instance, owner) + return super().__get__(instance, owner) def validate(self, value): """Make sure that a list of valid fields is being used.""" @@ -942,7 +938,7 @@ class ListField(ComplexBaseField): if self.max_length is not None and len(value) > self.max_length: self.error("List is too long") - super(ListField, self).validate(value) + super().validate(value) def prepare_query_value(self, op, value): # Validate that the `set` operator doesn't contain more items than `max_length`. @@ -963,7 +959,7 @@ class ListField(ComplexBaseField): return self.field.prepare_query_value(op, value) - return super(ListField, self).prepare_query_value(op, value) + return super().prepare_query_value(op, value) class EmbeddedDocumentListField(ListField): @@ -984,9 +980,7 @@ class EmbeddedDocumentListField(ListField): :param kwargs: Keyword arguments passed directly into the parent :class:`~mongoengine.ListField`. """ - super(EmbeddedDocumentListField, self).__init__( - field=EmbeddedDocumentField(document_type), **kwargs - ) + super().__init__(field=EmbeddedDocumentField(document_type), **kwargs) class SortedListField(ListField): @@ -1012,10 +1006,10 @@ class SortedListField(ListField): self._ordering = kwargs.pop("ordering") if "reverse" in kwargs.keys(): self._order_reverse = kwargs.pop("reverse") - super(SortedListField, self).__init__(field, **kwargs) + super().__init__(field, **kwargs) def to_mongo(self, value, use_db_field=True, fields=None): - value = super(SortedListField, self).to_mongo(value, use_db_field, fields) + value = super().to_mongo(value, use_db_field, fields) if self._ordering is not None: return sorted( value, key=itemgetter(self._ordering), reverse=self._order_reverse @@ -1068,7 +1062,7 @@ class DictField(ComplexBaseField): self._auto_dereference = False kwargs.setdefault("default", lambda: {}) - super(DictField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def validate(self, value): """Make sure that a list of valid fields is being used.""" @@ -1090,7 +1084,7 @@ class DictField(ComplexBaseField): self.error( 'Invalid dictionary key name - keys may not startswith "$" characters' ) - super(DictField, self).validate(value) + super().validate(value) def lookup_member(self, member_name): return DictField(db_field=member_name) @@ -1119,7 +1113,7 @@ class DictField(ComplexBaseField): } return self.field.prepare_query_value(op, value) - return super(DictField, self).prepare_query_value(op, value) + return super().prepare_query_value(op, value) class MapField(DictField): @@ -1134,7 +1128,7 @@ class MapField(DictField): # XXX ValidationError raised outside of the "validate" method. if not isinstance(field, BaseField): self.error("Argument to MapField constructor must be a valid field") - super(MapField, self).__init__(field=field, *args, **kwargs) + super().__init__(field=field, *args, **kwargs) class ReferenceField(BaseField): @@ -1205,7 +1199,7 @@ class ReferenceField(BaseField): self.dbref = dbref self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule - super(ReferenceField, self).__init__(**kwargs) + super().__init__(**kwargs) @property def document_type(self): @@ -1238,7 +1232,7 @@ class ReferenceField(BaseField): else: instance._data[self.name] = cls._from_son(dereferenced) - return super(ReferenceField, self).__get__(instance, owner) + return super().__get__(instance, owner) def to_mongo(self, document): if isinstance(document, DBRef): @@ -1289,7 +1283,7 @@ class ReferenceField(BaseField): def prepare_query_value(self, op, value): if value is None: return None - super(ReferenceField, self).prepare_query_value(op, value) + super().prepare_query_value(op, value) return self.to_mongo(value) def validate(self, value): @@ -1336,7 +1330,7 @@ class CachedReferenceField(BaseField): self.auto_sync = auto_sync self.document_type_obj = document_type self.fields = fields - super(CachedReferenceField, self).__init__(**kwargs) + super().__init__(**kwargs) def start_listener(self): from mongoengine import signals @@ -1394,7 +1388,7 @@ class CachedReferenceField(BaseField): else: instance._data[self.name] = self.document_type._from_son(dereferenced) - return super(CachedReferenceField, self).__get__(instance, owner) + return super().__get__(instance, owner) def to_mongo(self, document, use_db_field=True, fields=None): id_field_name = self.document_type._meta["id_field"] @@ -1493,7 +1487,7 @@ class GenericReferenceField(BaseField): def __init__(self, *args, **kwargs): choices = kwargs.pop("choices", None) - super(GenericReferenceField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.choices = [] # Keep the choices as a list of allowed Document class names if choices: @@ -1517,7 +1511,7 @@ class GenericReferenceField(BaseField): value = value.get("_cls") elif isinstance(value, Document): value = value._class_name - super(GenericReferenceField, self)._validate_choices(value) + super()._validate_choices(value) def __get__(self, instance, owner): if instance is None: @@ -1533,7 +1527,7 @@ class GenericReferenceField(BaseField): else: instance._data[self.name] = dereferenced - return super(GenericReferenceField, self).__get__(instance, owner) + return super().__get__(instance, owner) def validate(self, value): if not isinstance(value, (Document, DBRef, dict, SON)): @@ -1597,13 +1591,13 @@ class BinaryField(BaseField): def __init__(self, max_bytes=None, **kwargs): self.max_bytes = max_bytes - super(BinaryField, self).__init__(**kwargs) + super().__init__(**kwargs) def __set__(self, instance, value): """Handle bytearrays in python 3.1""" if isinstance(value, bytearray): value = bytes(value) - return super(BinaryField, self).__set__(instance, value) + return super().__set__(instance, value) def to_mongo(self, value): return Binary(value) @@ -1621,14 +1615,14 @@ class BinaryField(BaseField): def prepare_query_value(self, op, value): if value is None: return value - return super(BinaryField, self).prepare_query_value(op, self.to_mongo(value)) + return super().prepare_query_value(op, self.to_mongo(value)) class GridFSError(Exception): pass -class GridFSProxy(object): +class GridFSProxy: """Proxy object to handle writing and reading of files to and from GridFS .. versionadded:: 0.4 @@ -1808,7 +1802,7 @@ class FileField(BaseField): def __init__( self, db_alias=DEFAULT_CONNECTION_NAME, collection_name="fs", **kwargs ): - super(FileField, self).__init__(**kwargs) + super().__init__(**kwargs) self.collection_name = collection_name self.db_alias = db_alias @@ -1953,7 +1947,7 @@ class ImageGridFsProxy(GridFSProxy): img.save(io, img_format, progressive=progressive) io.seek(0) - return super(ImageGridFsProxy, self).put( + return super().put( io, width=w, height=h, format=img_format, thumbnail_id=thumb_id, **kwargs ) @@ -1963,7 +1957,7 @@ class ImageGridFsProxy(GridFSProxy): if out and out.thumbnail_id: self.fs.delete(out.thumbnail_id) - return super(ImageGridFsProxy, self).delete() + return super().delete() def _put_thumbnail(self, thumbnail, format, progressive, **kwargs): w, h = thumbnail.size @@ -2042,7 +2036,7 @@ class ImageField(FileField): setattr(self, att_name, value) - super(ImageField, self).__init__(collection_name=collection_name, **kwargs) + super().__init__(collection_name=collection_name, **kwargs) class SequenceField(BaseField): @@ -2094,7 +2088,7 @@ class SequenceField(BaseField): self.value_decorator = ( value_decorator if callable(value_decorator) else self.VALUE_DECORATOR ) - super(SequenceField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def generate(self): """ @@ -2155,7 +2149,7 @@ class SequenceField(BaseField): ) def __get__(self, instance, owner): - value = super(SequenceField, self).__get__(instance, owner) + value = super().__get__(instance, owner) if value is None and instance._initialised: value = self.generate() instance._data[self.name] = value @@ -2168,7 +2162,7 @@ class SequenceField(BaseField): if value is None and instance._initialised: value = self.generate() - return super(SequenceField, self).__set__(instance, value) + return super().__set__(instance, value) def prepare_query_value(self, op, value): """ @@ -2202,7 +2196,7 @@ class UUIDField(BaseField): .. versionchanged:: 0.6.19 """ self._binary = binary - super(UUIDField, self).__init__(**kwargs) + super().__init__(**kwargs) def to_python(self, value): if not self._binary: @@ -2440,7 +2434,7 @@ class LazyReferenceField(BaseField): self.passthrough = passthrough self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule - super(LazyReferenceField, self).__init__(**kwargs) + super().__init__(**kwargs) @property def document_type(self): @@ -2483,7 +2477,7 @@ class LazyReferenceField(BaseField): if value: instance._data[self.name] = value - return super(LazyReferenceField, self).__get__(instance, owner) + return super().__get__(instance, owner) def to_mongo(self, value): if isinstance(value, LazyReference): @@ -2547,7 +2541,7 @@ class LazyReferenceField(BaseField): def prepare_query_value(self, op, value): if value is None: return None - super(LazyReferenceField, self).prepare_query_value(op, value) + super().prepare_query_value(op, value) return self.to_mongo(value) def lookup_member(self, member_name): @@ -2574,12 +2568,12 @@ class GenericLazyReferenceField(GenericReferenceField): def __init__(self, *args, **kwargs): self.passthrough = kwargs.pop("passthrough", False) - super(GenericLazyReferenceField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _validate_choices(self, value): if isinstance(value, LazyReference): value = value.document_type._class_name - super(GenericLazyReferenceField, self)._validate_choices(value) + super()._validate_choices(value) def build_lazyref(self, value): if isinstance(value, LazyReference): @@ -2608,7 +2602,7 @@ class GenericLazyReferenceField(GenericReferenceField): if value: instance._data[self.name] = value - return super(GenericLazyReferenceField, self).__get__(instance, owner) + return super().__get__(instance, owner) def validate(self, value): if isinstance(value, LazyReference) and value.pk is None: @@ -2616,7 +2610,7 @@ class GenericLazyReferenceField(GenericReferenceField): "You can only reference documents once they have been" " saved to the database" ) - return super(GenericLazyReferenceField, self).validate(value) + return super().validate(value) def to_mongo(self, document): if document is None: @@ -2635,4 +2629,4 @@ class GenericLazyReferenceField(GenericReferenceField): ) ) else: - return super(GenericLazyReferenceField, self).to_mongo(document) + return super().to_mongo(document) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index e6901100..4dbf7d47 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -37,7 +37,7 @@ DENY = 3 PULL = 4 -class BaseQuerySet(object): +class BaseQuerySet: """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. """ @@ -262,7 +262,7 @@ class BaseQuerySet(object): # If we were able to retrieve the 2nd doc, rewind the cursor and # raise the MultipleObjectsReturned exception. queryset.rewind() - message = u"%d items returned, instead of 1" % queryset.count() + message = "%d items returned, instead of 1" % queryset.count() raise queryset._document.MultipleObjectsReturned(message) def create(self, **kwargs): @@ -351,14 +351,14 @@ class BaseQuerySet(object): except pymongo.errors.BulkWriteError as err: # inserting documents that already have an _id field will # give huge performance debt or raise - message = u"Bulk write error: (%s)" + message = "Bulk write error: (%s)" raise BulkWriteError(message % err.details) except pymongo.errors.OperationFailure as err: message = "Could not save document (%s)" if re.match("^E1100[01] duplicate key", str(err)): # E11000 - duplicate key error index # E11001 - duplicate key on update - message = u"Tried to save duplicate unique keys (%s)" + message = "Tried to save duplicate unique keys (%s)" raise NotUniqueError(message % err) raise OperationError(message % err) @@ -655,9 +655,9 @@ class BaseQuerySet(object): **self._cursor_args ) except pymongo.errors.DuplicateKeyError as err: - raise NotUniqueError(u"Update failed (%s)" % err) + raise NotUniqueError("Update failed (%s)" % err) except pymongo.errors.OperationFailure as err: - raise OperationError(u"Update failed (%s)" % err) + raise OperationError("Update failed (%s)" % err) if full_response: if result["value"] is not None: @@ -686,7 +686,7 @@ class BaseQuerySet(object): return queryset.filter(pk=object_id).first() def in_bulk(self, object_ids): - """Retrieve a set of documents by their ids. + """"Retrieve a set of documents by their ids. :param object_ids: a list or tuple of ``ObjectId``\ s :rtype: dict of ObjectIds as keys and collection-specific @@ -1922,7 +1922,7 @@ class BaseQuerySet(object): field_name = match.group(1).split(".") fields = self._document._lookup_field(field_name) # Substitute the correct name for the field into the javascript - return u'["%s"]' % fields[-1].db_field + return '["%s"]' % fields[-1].db_field def field_path_sub(match): # Extract just the field name, and look up the field objects diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index e0d8e322..443c895c 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -1,7 +1,7 @@ __all__ = ("QueryFieldList",) -class QueryFieldList(object): +class QueryFieldList: """Object that handles combinations of .only() and .exclude() calls""" ONLY = 1 diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 5067ffbf..699526fd 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -4,7 +4,7 @@ from mongoengine.queryset.queryset import QuerySet __all__ = ("queryset_manager", "QuerySetManager") -class QuerySetManager(object): +class QuerySetManager: """ The default QuerySet Manager. diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 39b09c9d..8b5872f8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -141,10 +141,10 @@ class QuerySet(BaseQuerySet): getting the count """ if with_limit_and_skip is False: - return super(QuerySet, self).count(with_limit_and_skip) + return super().count(with_limit_and_skip) if self._len is None: - self._len = super(QuerySet, self).count(with_limit_and_skip) + self._len = super().count(with_limit_and_skip) return self._len diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 72e36ac0..0eacc2ef 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -7,7 +7,7 @@ from mongoengine.queryset import transform __all__ = ("Q", "QNode") -class QNodeVisitor(object): +class QNodeVisitor: """Base visitor class for visiting Q-object nodes in a query tree. """ @@ -79,7 +79,7 @@ class QueryCompilerVisitor(QNodeVisitor): return transform.query(self.document, **query.query) -class QNode(object): +class QNode: """Base class for nodes in query trees.""" AND = 0 diff --git a/mongoengine/signals.py b/mongoengine/signals.py index 0db63604..582b533d 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -15,11 +15,11 @@ try: signals_available = True except ImportError: - class Namespace(object): + class Namespace: def signal(self, name, doc=None): return _FakeSignal(name, doc) - class _FakeSignal(object): + class _FakeSignal: """If blinker is unavailable, create a fake class with the same interface that allows sending of signals but will fail with an error on anything else. Instead of doing anything on send, it diff --git a/setup.py b/setup.py index fe3253ae..b6c8ea64 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ extra_opts = { "pytest-cov", "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", - "Pillow>=2.0.0", + "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support ], } From b57946ec989defa457b70a2b18bc0543a54bae40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 14 Mar 2020 21:39:47 +0100 Subject: [PATCH 44/79] remove virtualenv installation in travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d34f8a36..5d04571a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,8 +60,7 @@ install: - pip install --upgrade pip - pip install coveralls - pip install flake8 flake8-import-order - - pip install tox # tox 3.11.0 has requirement virtualenv>=14.0.0 - - pip install virtualenv # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32) + - pip install tox # tox dryrun to setup the tox venv (we run a mock test). - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder" # Install black for Python v3.7 only. From aa4a6ae0234be23f237f68c3061f9db28c274741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 21:02:44 +0100 Subject: [PATCH 45/79] Fix invalid escape seq in codebase --- mongoengine/document.py | 2 +- mongoengine/queryset/base.py | 6 +++--- tests/fields/test_complex_datetime_field.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 5e812510..3cc0046e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class InvalidCollectionError(Exception): class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): - """A :class:`~mongoengine.Document` that isn't stored in its own + r"""A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37ac..7941e970 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -694,8 +694,8 @@ class BaseQuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - :param object_ids: a list or tuple of ``ObjectId``\ s - :rtype: dict of ObjectIds as keys and collection-specific + :param object_ids: a list or tuple of ObjectId's + :rtype: dict of ObjectId's as keys and collection-specific Document subclasses as values. .. versionadded:: 0.3 @@ -1140,7 +1140,7 @@ class BaseQuerySet(object): def explain(self): """Return an explain plan record for the - :class:`~mongoengine.queryset.QuerySet`\ 's cursor. + :class:`~mongoengine.queryset.QuerySet` cursor. """ return self._cursor.explain() diff --git a/tests/fields/test_complex_datetime_field.py b/tests/fields/test_complex_datetime_field.py index f0a6b96e..5bd6c56b 100644 --- a/tests/fields/test_complex_datetime_field.py +++ b/tests/fields/test_complex_datetime_field.py @@ -65,7 +65,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond): stored = LogEntry(date=datetime.datetime(*values)).to_mongo()["date"] assert ( - re.match("^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) + re.match(r"^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) is not None ) @@ -74,7 +74,7 @@ class ComplexDateTimeFieldTest(MongoDBTestCase): "date_with_dots" ] assert ( - re.match("^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None + re.match(r"^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None ) def test_complexdatetime_usage(self): From c0c0efce188b32f4e59da89ead845a17b824b236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 22:14:26 +0100 Subject: [PATCH 46/79] improve docstring related to #2267 and document the change in the changelog --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dcea62a..41ff8c85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,7 @@ Development - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 +- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 Changes in 0.19.1 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 9671a3dc..fbf0a1ba 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1026,9 +1026,11 @@ class BaseQuerySet(object): posts = BlogPost.objects(...).fields(comments=0) - To retrieve a subrange of array elements: + To retrieve a subrange or sublist of array elements, + support exist for both the `slice` and `elemMatch` projection operator: posts = BlogPost.objects(...).fields(slice__comments=5) + posts = BlogPost.objects(...).fields(elemMatch__comments="test") :param kwargs: A set of keyword arguments identifying what to include, exclude, or slice. From a3f9016ae938a91871657373ade2970a0fc157d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 15 Mar 2020 22:27:19 +0100 Subject: [PATCH 47/79] reformat import for flake8 --- mongoengine/context_managers.py | 1 + mongoengine/queryset/base.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 0c58b57c..6891e21d 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -14,6 +14,7 @@ __all__ = ( "no_sub_classes", "query_counter", "set_write_concern", + "set_read_write_concern", ) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0788f563..d8a4b96f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -20,8 +20,8 @@ from mongoengine.base import get_document from mongoengine.common import _import_class from mongoengine.connection import get_db from mongoengine.context_managers import ( - set_write_concern, set_read_write_concern, + set_write_concern, switch_db, ) from mongoengine.errors import ( From ad0669a32605bb7d9984b1830349d82c208b3c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Mon, 16 Mar 2020 22:39:31 +0100 Subject: [PATCH 48/79] update changelog --- docs/changelog.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7f68bbe5..1bfb190d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,13 +8,9 @@ Development - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 -- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - -Changes in 0.19.2 -================= +- Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 - Changes in 0.19.1 ================= - Requires Pillow < 7.0.0 as it dropped Python2 support From aadc6262edc5627bafbef41a7c814fedbe66fdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:10:52 +0100 Subject: [PATCH 49/79] remove qs.slave_okay() that is deprecated since pymongo3 --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1bfb190d..b2090645 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 +- Remove method queryset.slave_okay() that was deprecated a while ago and disappeared since pymongo3 Changes in 0.19.1 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index fbf0a1ba..95606f2a 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -60,7 +60,6 @@ class BaseQuerySet(object): self._ordering = None self._snapshot = False self._timeout = True - self._slave_okay = False self._read_preference = None self._iter = False self._scalar = [] @@ -775,7 +774,6 @@ class BaseQuerySet(object): "_ordering", "_snapshot", "_timeout", - "_slave_okay", "_read_preference", "_iter", "_scalar", @@ -1172,20 +1170,6 @@ class BaseQuerySet(object): queryset._timeout = enabled return queryset - # DEPRECATED. Has no more impact on PyMongo 3+ - def slave_okay(self, enabled): - """Enable or disable the slave_okay when querying. - - :param enabled: whether or not the slave_okay is enabled - - .. deprecated:: Ignored with PyMongo 3+ - """ - msg = "slave_okay is deprecated as it has no impact when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) - queryset = self.clone() - queryset._slave_okay = enabled - return queryset - def read_preference(self, read_preference): """Change the read_preference when querying. From 8eb51790b50858d351068c3f37316d16143d3b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:26:41 +0100 Subject: [PATCH 50/79] Remove Field(name='...') which was deprecated when db_field was introduced a while ago --- docs/changelog.rst | 1 + mongoengine/base/fields.py | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1bfb190d..32cfe647 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 +- Remove name parameter in Field constructor e.g `StringField(name="...")`, it was deprecated a while ago in favor of db_field Changes in 0.19.1 ================= diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index cd1039cb..379098e5 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -36,7 +36,6 @@ class BaseField(object): def __init__( self, db_field=None, - name=None, required=False, default=None, unique=False, @@ -51,7 +50,6 @@ class BaseField(object): """ :param db_field: The database field to store this field in (defaults to the name of the field) - :param name: Deprecated - use db_field :param required: If the field is required. Whether it has to have a value or not. Defaults to False. :param default: (optional) The default value for this field if no value @@ -75,11 +73,8 @@ class BaseField(object): existing attributes. Common metadata includes `verbose_name` and `help_text`. """ - self.db_field = (db_field or name) if not primary_key else "_id" + self.db_field = db_field if not primary_key else "_id" - if name: - msg = 'Field\'s "name" attribute deprecated in favour of "db_field"' - warnings.warn(msg, DeprecationWarning) self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) From ee2d50b2d1b3549887512bfaefdb1abff3b831fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 21:38:50 +0100 Subject: [PATCH 51/79] remove drop_dups index option, deprecated with MongoDB3 --- docs/changelog.rst | 6 ++++-- docs/guide/defining-documents.rst | 6 ------ mongoengine/base/metaclasses.py | 1 - mongoengine/document.py | 15 +-------------- tests/document/test_indexes.py | 16 ---------------- tests/document/test_inheritance.py | 1 - 6 files changed, 5 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0784050..1037ca13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,8 +10,10 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 -- Remove name parameter in Field constructor e.g `StringField(name="...")`, it was deprecated a while ago in favor of db_field -- Remove method queryset.slave_okay() that was deprecated a while ago and disappeared since pymongo3 +- Remove methods deprecated years ago: + - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field + - Queryset.slave_okay() was deprecated since pymongo3 + - dropDups was dropped with MongoDB3 Changes in 0.19.1 ================= diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index bd2b43e2..6dc35c30 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -555,7 +555,6 @@ There are a few top level defaults for all indexes that can be set:: 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'index_drop_dups': True, } @@ -574,11 +573,6 @@ There are a few top level defaults for all indexes that can be set:: in systems where indexes are managed separately. Disabling this will improve performance. -:attr:`index_drop_dups` (Optional) - Set the default value for if an index should drop duplicates - Since MongoDB 3.0 drop_dups is not supported anymore. Raises a Warning - and has no effect - Compound Indexes and Indexing sub documents ------------------------------------------- diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index e4d26811..3bba796b 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -284,7 +284,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): "indexes": [], # indexes to be ensured at runtime "id_field": None, "index_background": False, - "index_drop_dups": False, "index_opts": None, "delete_rules": None, # allow_inheritance can be True, False, and None. True means diff --git a/mongoengine/document.py b/mongoengine/document.py index 3cc0046e..c8710fb5 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -851,17 +851,13 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): index_spec = cls._build_index_spec(keys) index_spec = index_spec.copy() fields = index_spec.pop("fields") - drop_dups = kwargs.get("drop_dups", False) - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) index_spec["background"] = background index_spec.update(kwargs) return cls._get_collection().create_index(fields, **index_spec) @classmethod - def ensure_index(cls, key_or_list, drop_dups=False, background=False, **kwargs): + def ensure_index(cls, key_or_list, background=False, **kwargs): """Ensure that the given indexes are in place. Deprecated in favour of create_index. @@ -869,12 +865,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering :param background: Allows index creation in the background - :param drop_dups: Was removed/ignored with MongoDB >2.7.5. The value - will be removed if PyMongo3+ is used """ - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) return cls.create_index(key_or_list, background=background, **kwargs) @classmethod @@ -887,12 +878,8 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): `auto_create_index` to False in the documents meta data """ background = cls._meta.get("index_background", False) - drop_dups = cls._meta.get("index_drop_dups", False) index_opts = cls._meta.get("index_opts") or {} index_cls = cls._meta.get("index_cls", True) - if drop_dups: - msg = "drop_dups is deprecated and is removed when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) collection = cls._get_collection() # 746: when connection is via mongos, the read preference is not necessarily an indication that diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index be857b59..b08306a0 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -806,18 +806,6 @@ class TestIndexes(unittest.TestCase): info = Log.objects._collection.index_information() assert 3600 == info["created_1"]["expireAfterSeconds"] - def test_index_drop_dups_silently_ignored(self): - class Customer(Document): - cust_id = IntField(unique=True, required=True) - meta = { - "indexes": ["cust_id"], - "index_drop_dups": True, - "allow_inheritance": False, - } - - Customer.drop_collection() - Customer.objects.first() - def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by meta.indexes. @@ -1058,10 +1046,6 @@ class TestIndexes(unittest.TestCase): del index_info[key][ "ns" ] # drop the index namespace - we don't care about that here, MongoDB 3+ - if "dropDups" in index_info[key]: - del index_info[key][ - "dropDups" - ] # drop the index dropDups - it is deprecated in MongoDB 3+ assert index_info == { "txt_1": {"key": [("txt", 1)], "background": False}, diff --git a/tests/document/test_inheritance.py b/tests/document/test_inheritance.py index 5072f841..d7bd0632 100644 --- a/tests/document/test_inheritance.py +++ b/tests/document/test_inheritance.py @@ -523,7 +523,6 @@ class TestInheritance(MongoDBTestCase): defaults = { "index_background": True, - "index_drop_dups": True, "index_opts": {"hello": "world"}, "allow_inheritance": True, "queryset_class": "QuerySet", From 3d80637fa414354b2d05b0ca6ac3afb71222d8f1 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 19:08:43 -0300 Subject: [PATCH 52/79] Refactor set_read_write_concern so read_conern is consistent with the write_concerns argument. --- mongoengine/context_managers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 6891e21d..a87c9bb3 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -261,10 +261,14 @@ def set_write_concern(collection, write_concerns): @contextmanager -def set_read_write_concern(collection, write_concerns, read_concern): +def set_read_write_concern(collection, write_concerns, read_concerns): combined_write_concerns = dict(collection.write_concern.document.items()) combined_write_concerns.update(write_concerns) + combined_read_concerns = dict(collection.read_concern.document.items()) + combined_read_concerns.update(read_concerns) + yield collection.with_options( - write_concern=WriteConcern(**combined_write_concerns), read_concern=read_concern + write_concern=WriteConcern(**combined_write_concerns), + read_concern=ReadConcern(**combined_read_concerns) ) From 4c62a060f0c460b1d68ebba861eb7a3fd9f9e14c Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 19:37:21 -0300 Subject: [PATCH 53/79] Add tests for set_write_conern and set_read_write_concern --- tests/test_context_managers.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index fa3f5960..f445cf57 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -10,11 +10,59 @@ from mongoengine.context_managers import ( query_counter, switch_collection, switch_db, + set_write_concern, + set_read_write_concern, ) from mongoengine.pymongo_support import count_documents class TestContextManagers: + def test_set_write_concern(self): + connect("mongoenginetest") + + class User(Document): + name = StringField() + + collection = User._get_collection() + original_write_concern = collection.write_concern + + with set_write_concern( + collection, {"w": "majority", "j": True, "wtimeout": 1234} + ) as updated_collection: + assert updated_collection.write_concern.document == { + "w": "majority", + "j": True, + "wtimeout": 1234, + } + + assert original_write_concern.document == collection.write_concern.document + + def test_set_read_write_concern(self): + connect("mongoenginetest") + + class User(Document): + name = StringField() + + collection = User._get_collection() + + original_read_concern = collection.read_concern + original_write_concern = collection.write_concern + + with set_read_write_concern( + collection, + {"w": "majority", "j": True, "wtimeout": 1234}, + {"level": "local"}, + ) as update_collection: + assert update_collection.read_concern.document == {"level": "local"} + assert update_collection.write_concern.document == { + "w": "majority", + "j": True, + "wtimeout": 1234, + } + + assert original_read_concern.document == collection.read_concern.document + assert original_write_concern.document == collection.write_concern.document + def test_switch_db_context_manager(self): connect("mongoenginetest") register_connection("testdb-1", "mongoenginetest2") From 476b07af6ebe26e0418d1ab8da9e8179794ddc4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 17 Mar 2020 23:59:54 +0100 Subject: [PATCH 54/79] reformat changelog --- docs/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a844761..9b864b02 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,8 +7,6 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis -- BREAKING CHANGE: Removed ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes`` that were deprecated in 2013. - ``Document.ensure_indexes`` still exists and is the right method to use - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 @@ -16,6 +14,7 @@ Development - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field - Queryset.slave_okay() was deprecated since pymongo3 - dropDups was dropped with MongoDB3 + - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes`` Changes in 0.19.1 ================= From af35b25d155e050cba3e4925bf76529f7ae52137 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 21:28:08 -0300 Subject: [PATCH 55/79] Refactor read_preference to accept a dictionary instead of a ReadPreference instance. --- mongoengine/queryset/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index d8a4b96f..f8671f8f 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -5,6 +5,8 @@ import itertools import re import warnings +from collections.abc import Mapping + from bson import SON, json_util from bson.code import Code import pymongo @@ -1221,11 +1223,11 @@ class BaseQuerySet(object): :param read_concern: override ReplicaSetConnection-level preference. """ - if read_concern is not None and not isinstance(read_concern, ReadConcern): - raise TypeError("%r is not a read concern." % (read_concern,)) + if read_concern is not None and not isinstance(read_concern, Mapping): + raise TypeError("%r is not a valid read concern." % (read_concern,)) queryset = self.clone() - queryset._read_concern = read_concern + queryset._read_concern = ReadConcern(**read_concern) queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_concern return queryset From 8913a74a86e01c3a96aaebf3372b1898c4efe060 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 21:31:05 -0300 Subject: [PATCH 56/79] Allow setting the read concern to None --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index f8671f8f..76427c89 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1227,7 +1227,7 @@ class BaseQuerySet(object): raise TypeError("%r is not a valid read concern." % (read_concern,)) queryset = self.clone() - queryset._read_concern = ReadConcern(**read_concern) + queryset._read_concern = ReadConcern(**read_concern) if read_concern is not None else None queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_concern return queryset From bc77322c2f166e28390e1492a5a67440f90af021 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 21:49:17 -0300 Subject: [PATCH 57/79] Update unit tests --- tests/queryset/test_queryset.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 8098b137..a8954526 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4739,33 +4739,33 @@ class TestQueryset(unittest.TestCase): bars = list(Bar.objects.read_concern(None)) assert bars == [bar] - bars = Bar.objects.read_concern(ReadConcern(level="local")) - assert bars._read_concern == ReadConcern(level="local") - assert bars._cursor.collection.read_concern == ReadConcern(level="local") + bars = Bar.objects.read_concern({"level": "local"}) + assert bars._read_concern.document == {"level": "local"} + assert bars._cursor.collection.read_concern.document == {"level": "local"} - # Make sure that `.read_concern(...)` does accept string values. + # Make sure that `.read_concern(...)` does not accept string values. with pytest.raises(TypeError): Bar.objects.read_concern("local") def assert_read_concern(qs, expected_read_concern): - assert qs._read_concern == expected_read_concern - assert qs._cursor.collection.read_concern == expected_read_concern + assert qs._read_concern.document == expected_read_concern + assert qs._cursor.collection.read_concern.document == expected_read_concern # Make sure read concern is respected after a `.skip(...)`. - bars = Bar.objects.skip(1).read_concern(ReadConcern("majority")) - assert_read_concern(bars, ReadConcern("majority")) + bars = Bar.objects.skip(1).read_concern({"level": "local"}) + assert_read_concern(bars, {"level": "local"}) # Make sure read concern is respected after a `.limit(...)`. - bars = Bar.objects.limit(1).read_concern(ReadConcern("majority")) - assert_read_concern(bars, ReadConcern("majority")) + bars = Bar.objects.limit(1).read_concern({"level": "local"}) + assert_read_concern(bars, {"level": "local"}) # Make sure read concern is respected after an `.order_by(...)`. - bars = Bar.objects.order_by("txt").read_concern(ReadConcern("majority")) - assert_read_concern(bars, ReadConcern("majority")) + bars = Bar.objects.order_by("txt").read_concern({"level": "local"}) + assert_read_concern(bars, {"level": "local"}) # Make sure read concern is respected after a `.hint(...)`. - bars = Bar.objects.hint([("txt", 1)]).read_concern(ReadConcern("majority")) - assert_read_concern(bars, ReadConcern("majority")) + bars = Bar.objects.hint([("txt", 1)]).read_concern({"level": "majority"}) + assert_read_concern(bars, {"level": "majority"}) def test_json_simple(self): class Embedded(EmbeddedDocument): From 7cc964c7d86d9e6dccf4f819ad291a4150b1d1ee Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Tue, 17 Mar 2020 21:58:36 -0300 Subject: [PATCH 58/79] Add missing import --- mongoengine/context_managers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index a87c9bb3..9319abae 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from pymongo.write_concern import WriteConcern +from pymongo.read_concern import ReadConcern from six import iteritems from mongoengine.common import _import_class @@ -263,10 +264,14 @@ def set_write_concern(collection, write_concerns): @contextmanager def set_read_write_concern(collection, write_concerns, read_concerns): combined_write_concerns = dict(collection.write_concern.document.items()) - combined_write_concerns.update(write_concerns) + + if write_concerns is not None: + combined_write_concerns.update(write_concerns) combined_read_concerns = dict(collection.read_concern.document.items()) - combined_read_concerns.update(read_concerns) + + if read_concerns is not None: + combined_read_concerns.update(read_concerns) yield collection.with_options( write_concern=WriteConcern(**combined_write_concerns), From 2b0157aecddb4a77d25a7fbb724578be3c1e097c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 22 Mar 2020 14:05:04 +0100 Subject: [PATCH 59/79] Improve Queryset.get to avoid confusing message in case multiple match are found --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 8 +++++--- tests/queryset/test_queryset.py | 33 ++++++++++++++++++++++++--------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9b864b02..ec0209f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +- Improve Queryset.get to avoid confusing MultipleObjectsReturned message in case multiple match are found #630 - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index eed30413..15c58481 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -259,16 +259,18 @@ class BaseQuerySet(object): except StopIteration: msg = "%s matching query does not exist." % queryset._document._class_name raise queryset._document.DoesNotExist(msg) + try: + # Check if there is another match six.next(queryset) except StopIteration: return result # If we were able to retrieve the 2nd doc, rewind the cursor and # raise the MultipleObjectsReturned exception. - queryset.rewind() - message = u"%d items returned, instead of 1" % queryset.count() - raise queryset._document.MultipleObjectsReturned(message) + raise queryset._document.MultipleObjectsReturned( + u"2 or more items returned, instead of 1" + ) def create(self, **kwargs): """Create new object. Returns the saved object instance. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index f6d1a916..f15b9748 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -274,32 +274,47 @@ class TestQueryset(unittest.TestCase): with pytest.raises(InvalidQueryError): self.Person.objects(name="User A").with_id(person1.id) - def test_find_only_one(self): - """Ensure that a query using ``get`` returns at most one result. - """ + def test_get_no_document_exists_raises_doesnotexist(self): + assert self.Person.objects.count() == 0 # Try retrieving when no objects exists with pytest.raises(DoesNotExist): self.Person.objects.get() with pytest.raises(self.Person.DoesNotExist): self.Person.objects.get() + def test_get_multiple_match_raises_multipleobjectsreturned(self): + """Ensure that a query using ``get`` returns at most one result. + """ + assert self.Person.objects().count() == 0 + person1 = self.Person(name="User A", age=20) person1.save() - person2 = self.Person(name="User B", age=30) + + p = self.Person.objects.get() + assert p == person1 + + person2 = self.Person(name="User B", age=20) person2.save() - # Retrieve the first person from the database + person3 = self.Person(name="User C", age=30) + person3.save() + + # .get called without argument with pytest.raises(MultipleObjectsReturned): self.Person.objects.get() with pytest.raises(self.Person.MultipleObjectsReturned): self.Person.objects.get() + # check filtering + with pytest.raises(MultipleObjectsReturned): + self.Person.objects.get(age__lt=30) + with pytest.raises(MultipleObjectsReturned) as exc_info: + self.Person.objects(age__lt=30).get() + assert "2 or more items returned, instead of 1" == str(exc_info.value) + # Use a query to filter the people found to just person2 person = self.Person.objects.get(age=30) - assert person.name == "User B" - - person = self.Person.objects.get(age__lt=30) - assert person.name == "User A" + assert person == person3 def test_find_array_position(self): """Ensure that query by array position works. From ce74978b1ece68710b33295524b7563271cd44d4 Mon Sep 17 00:00:00 2001 From: Kes Date: Thu, 26 Mar 2020 17:23:16 +0100 Subject: [PATCH 60/79] Correct and improve docstring on EmbeddedDocumentList.create() --- mongoengine/base/datastructures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index dcc1f092..0572e22e 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -303,11 +303,11 @@ class EmbeddedDocumentList(BaseList): def create(self, **values): """ - Creates a new embedded document and saves it to the database. + Creates a new instance of the EmbeddedDocument and appends it to this EmbeddedDocumentList. .. note:: - The embedded document changes are not automatically saved - to the database after calling this method. + the instance of the EmbeddedDocument is not automatically saved to the database. + You still need to call save() o this EmbeddedDocumentList. :param values: A dictionary of values for the embedded document. :return: The new embedded document instance. From d73f0bb1af7fa63424137174da76f89dca762010 Mon Sep 17 00:00:00 2001 From: Kes Date: Thu, 26 Mar 2020 17:25:25 +0100 Subject: [PATCH 61/79] fix typo --- mongoengine/base/datastructures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 0572e22e..fd93c1db 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -307,7 +307,7 @@ class EmbeddedDocumentList(BaseList): .. note:: the instance of the EmbeddedDocument is not automatically saved to the database. - You still need to call save() o this EmbeddedDocumentList. + You still need to call save() on this EmbeddedDocumentList. :param values: A dictionary of values for the embedded document. :return: The new embedded document instance. From 707923e3f5d7682df58652a598edf2127a4ff95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Wed, 8 Apr 2020 21:43:50 +0200 Subject: [PATCH 62/79] fix subdependencies that dropped Py2 support --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 5cba5d9e..11185fd2 100644 --- a/setup.py +++ b/setup.py @@ -121,6 +121,8 @@ extra_opts = { "blinker", "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support "zipp<2.0.0", # (dependency of pytest) dropped python2 support + "pyparsing<3", # sub-dependency that dropped py2 support + "configparser<5", # sub-dependency that dropped py2 support ], } if PY3: From 43724e40b2871d8ca97b437a97ff6d2db01d8001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 25 Apr 2020 14:16:56 +0200 Subject: [PATCH 63/79] improve doc related to dropping Py2 support --- CONTRIBUTING.rst | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d939e2ee..4afcd69e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -20,7 +20,7 @@ post to the `user group ` Supported Interpreters ---------------------- -MongoEngine supports CPython 3.7 and newer as well as Pypy3. +MongoEngine supports CPython 3.5 and newer as well as Pypy3. Language features not supported by all interpreters can not be used. Python3 codebase diff --git a/setup.py b/setup.py index b6c8ea64..4d63b5b2 100644 --- a/setup.py +++ b/setup.py @@ -97,11 +97,11 @@ CLASSIFIERS = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database", From f4d7e72426f7fe26c913bc5872c6a12711d85123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 25 Apr 2020 20:43:55 +0200 Subject: [PATCH 64/79] improve .gitignore --- .gitignore | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 048a2d19..16633bae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,15 @@ -.* !.gitignore *~ *.py[co] .*.sw[po] +.cache/ +.coverage +.coveragerc +.env +.idea/ +.pytest_cache/ +.tox/ +.eggs/ *.egg docs/.build docs/_build @@ -13,8 +20,6 @@ env/ .settings .project .pydevproject -tests/test_bugfix.py htmlcov/ venv venv3 -scratchpad From ef7da36ac681e572461971c87549e3d5f7e108b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 25 Apr 2020 21:36:07 +0200 Subject: [PATCH 65/79] Update pre-commit and fix existing flake8 warnings once for all --- .pre-commit-config.yaml | 14 +++++++------- .travis.yml | 8 +++----- tests/document/test_instance.py | 18 +++++++++--------- tests/fields/test_dict_field.py | 2 +- tests/fields/test_lazy_reference_field.py | 6 +++--- tests/queryset/test_queryset.py | 20 ++++++++++---------- tests/test_connection.py | 6 +++--- tests/test_context_managers.py | 12 ++++++------ tests/test_signals.py | 2 +- 9 files changed, 43 insertions(+), 45 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cac25e41..8b794103 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ +fail_fast: false repos: - repo: https://github.com/ambv/black - rev: 19.3b0 + rev: 19.10b0 hooks: - id: black - language_version: python3 - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 +# language_version: python3 + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.0a2 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - id: flake8 + additional_dependencies: + - flake8-import-order diff --git a/.travis.yml b/.travis.yml index 5d04571a..a99f1117 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,18 +59,16 @@ install: # Install Python dependencies. - pip install --upgrade pip - pip install coveralls - - pip install flake8 flake8-import-order + - pip install pre-commit - pip install tox # tox dryrun to setup the tox venv (we run a mock test). - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder" - # Install black for Python v3.7 only. - - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pip install black; fi before_script: - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork - - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then flake8 .; else echo "flake8 only runs on py37"; fi - - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then black --check .; else echo "black only runs on py37"; fi + # Run pre-commit hooks (black, flake8, etc) on entire codebase + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi - mongo --eval 'db.version();' # Make sure mongo is awake script: diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 920bf392..993cc161 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -1414,7 +1414,7 @@ class TestDocumentInstance(MongoDBTestCase): assert raw_doc["first_name"] == "John" def test_inserts_if_you_set_the_pk(self): - p1 = self.Person(name="p1", id=bson.ObjectId()).save() + _ = self.Person(name="p1", id=bson.ObjectId()).save() p2 = self.Person(name="p2") p2.id = bson.ObjectId() p2.save() @@ -2195,7 +2195,7 @@ class TestDocumentInstance(MongoDBTestCase): user = User(name="Mike").save() reviewer = User(name="John").save() - book = Book(author=user, reviewer=reviewer).save() + _ = Book(author=user, reviewer=reviewer).save() reviewer.delete() assert Book.objects.count() == 1 @@ -2221,7 +2221,7 @@ class TestDocumentInstance(MongoDBTestCase): user_1 = User(id=1).save() user_2 = User(id=2).save() - book_1 = Book(id=1, author=user_2).save() + _ = Book(id=1, author=user_2).save() book_2 = Book(id=2, author=user_1).save() user_2.delete() @@ -2230,7 +2230,7 @@ class TestDocumentInstance(MongoDBTestCase): assert Book.objects.get() == book_2 user_3 = User(id=3).save() - book_3 = Book(id=3, author=user_3).save() + _ = Book(id=3, author=user_3).save() user_3.delete() # Deleting user_3 should also delete book_3 @@ -3204,7 +3204,7 @@ class TestDocumentInstance(MongoDBTestCase): def test_positional_creation(self): """Document cannot be instantiated using positional arguments.""" with pytest.raises(TypeError) as exc_info: - person = self.Person("Test User", 42) + self.Person("Test User", 42) expected_msg = ( "Instantiating a document with positional arguments is not " @@ -3606,13 +3606,13 @@ class TestDocumentInstance(MongoDBTestCase): v = StringField() class A(Document): - l = ListField(EmbeddedDocumentField(B)) + array = ListField(EmbeddedDocumentField(B)) A.objects.delete() - A(l=[B(v="1"), B(v="2"), B(v="3")]).save() + A(array=[B(v="1"), B(v="2"), B(v="3")]).save() a = A.objects.get() - assert a.l._instance == a - for idx, b in enumerate(a.l): + assert a.array._instance == a + for idx, b in enumerate(a.array): assert b._instance == a assert idx == 2 diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index 6850cd58..f423bf8b 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import pytest from bson import InvalidDocument +import pytest from mongoengine import * from mongoengine.base import BaseDict diff --git a/tests/fields/test_lazy_reference_field.py b/tests/fields/test_lazy_reference_field.py index b5b8690e..50e60262 100644 --- a/tests/fields/test_lazy_reference_field.py +++ b/tests/fields/test_lazy_reference_field.py @@ -152,7 +152,7 @@ class TestLazyReferenceField(MongoDBTestCase): LazyReference(BadDoc, animal.pk), ): with pytest.raises(ValidationError): - p = Ocurrence(person="test", animal=bad).save() + Ocurrence(person="test", animal=bad).save() def test_lazy_reference_query_conversion(self): """Ensure that LazyReferenceFields can be queried using objects and values @@ -386,7 +386,7 @@ class TestGenericLazyReferenceField(MongoDBTestCase): mineral = Mineral(name="Granite").save() occ_animal = Ocurrence(living_thing=animal, thing=animal).save() - occ_vegetal = Ocurrence(living_thing=vegetal, thing=vegetal).save() + _ = Ocurrence(living_thing=vegetal, thing=vegetal).save() with pytest.raises(ValidationError): Ocurrence(living_thing=mineral).save() @@ -458,7 +458,7 @@ class TestGenericLazyReferenceField(MongoDBTestCase): baddoc = BadDoc().save() for bad in (42, "foo", baddoc, LazyReference(BadDoc, animal.pk)): with pytest.raises(ValidationError): - p = Ocurrence(person="test", animal=bad).save() + Ocurrence(person="test", animal=bad).save() def test_generic_lazy_reference_query_conversion(self): class Member(Document): diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 984e2bd1..cb8e7bba 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -116,7 +116,7 @@ class TestQueryset(unittest.TestCase): def test_limit(self): """Ensure that QuerySet.limit works as expected.""" user_a = self.Person.objects.create(name="User A", age=20) - user_b = self.Person.objects.create(name="User B", age=30) + _ = self.Person.objects.create(name="User B", age=30) # Test limit on a new queryset people = list(self.Person.objects.limit(1)) @@ -148,6 +148,11 @@ class TestQueryset(unittest.TestCase): user_b = self.Person.objects.create(name="User B", age=30) # Test skip on a new queryset + people = list(self.Person.objects.skip(0)) + assert len(people) == 2 + assert people[0] == user_a + assert people[1] == user_b + people = list(self.Person.objects.skip(1)) assert len(people) == 1 assert people[0] == user_b @@ -2586,13 +2591,8 @@ class TestQueryset(unittest.TestCase): age = IntField() with db_ops_tracker() as q: - adult1 = ( - User.objects.filter(age__gte=18).comment("looking for an adult").first() - ) - - adult2 = ( - User.objects.comment("looking for an adult").filter(age__gte=18).first() - ) + User.objects.filter(age__gte=18).comment("looking for an adult").first() + User.objects.comment("looking for an adult").filter(age__gte=18).first() ops = q.get_ops() assert len(ops) == 2 @@ -4518,7 +4518,7 @@ class TestQueryset(unittest.TestCase): foos_without_y = list(Foo.objects.order_by("y").fields(y=0)) - assert all(o.y is None for o in foos_with_x) + assert all(o.y is None for o in foos_without_y) foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1)) @@ -5595,7 +5595,7 @@ class TestQueryset(unittest.TestCase): self.Person.objects.create(name="Baz") assert self.Person.objects.count(with_limit_and_skip=True) == 3 - newPerson = self.Person.objects.create(name="Foo_1") + self.Person.objects.create(name="Foo_1") assert self.Person.objects.count(with_limit_and_skip=True) == 4 def test_no_cursor_timeout(self): diff --git a/tests/test_connection.py b/tests/test_connection.py index e40a6994..56bc22cd 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -348,7 +348,7 @@ class ConnectionTest(unittest.TestCase): def test_disconnect_cleans_cached_collection_attribute_in_document(self): """Ensure that the disconnect() method works properly""" - conn1 = connect("mongoenginetest") + connect("mongoenginetest") class History(Document): pass @@ -518,7 +518,7 @@ class ConnectionTest(unittest.TestCase): """Ensure connect() uses the username & password params if the URI doesn't explicitly specify them. """ - c = connect( + connect( host="mongodb://localhost/mongoenginetest", username="user", password="pass" ) @@ -632,7 +632,7 @@ class ConnectionTest(unittest.TestCase): """Ensure connect() works when specifying a replicaSet via the MongoDB URI. """ - c = connect(host="mongodb://localhost/test?replicaSet=local-rs") + connect(host="mongodb://localhost/test?replicaSet=local-rs") db = get_db() assert isinstance(db, pymongo.database.Database) assert db.name == "test" diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index fa3f5960..4410fa90 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -216,7 +216,7 @@ class TestContextManagers: def test_query_counter_does_not_swallow_exception(self): with pytest.raises(TypeError): - with query_counter() as q: + with query_counter(): raise TypeError() def test_query_counter_temporarily_modifies_profiling_level(self): @@ -226,12 +226,12 @@ class TestContextManagers: initial_profiling_level = db.profiling_level() try: - NEW_LEVEL = 1 - db.set_profiling_level(NEW_LEVEL) - assert db.profiling_level() == NEW_LEVEL - with query_counter() as q: + new_level = 1 + db.set_profiling_level(new_level) + assert db.profiling_level() == new_level + with query_counter(): assert db.profiling_level() == 2 - assert db.profiling_level() == NEW_LEVEL + assert db.profiling_level() == new_level except Exception: db.set_profiling_level( initial_profiling_level diff --git a/tests/test_signals.py b/tests/test_signals.py index 451e01ff..64976e25 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -267,7 +267,7 @@ class TestSignal(unittest.TestCase): a = self.Author(name="Bill Shakespeare") a.save() self.get_signal_output(lambda: None) # eliminate signal output - a1 = self.Author.objects(name="Bill Shakespeare")[0] + _ = self.Author.objects(name="Bill Shakespeare")[0] assert self.get_signal_output(create_author) == [ "pre_init signal, Author", From 394da67cf12f2339452e3730cce36bec6b60f7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 25 Apr 2020 21:47:37 +0200 Subject: [PATCH 66/79] fix travis env var --- .pre-commit-config.yaml | 1 - .travis.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b794103..e11640b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ repos: rev: 19.10b0 hooks: - id: black -# language_version: python3 - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.0a2 hooks: diff --git a/.travis.yml b/.travis.yml index a99f1117..2316124a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ env: - PYMONGO_3_9=3.9 - PYMONGO_3_10=3.10 - - MAIN_PYTHON_VERSION = "3.7" + - MAIN_PYTHON_VERSION=3.7 matrix: - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} From fd0095b73f9130bd537a07dc5b13c329ef16a871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 26 Apr 2020 22:28:14 +0200 Subject: [PATCH 67/79] improve recent docstring of EmbeddedDocumentList.create --- mongoengine/base/datastructures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 85f05c9b..d08d4930 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -293,7 +293,7 @@ class EmbeddedDocumentList(BaseList): .. note:: the instance of the EmbeddedDocument is not automatically saved to the database. - You still need to call save() on this EmbeddedDocumentList. + You still need to call .save() on the parent Document. :param values: A dictionary of values for the embedded document. :return: The new embedded document instance. From 78c9e9745d82dff40c0c4e0c5931c4ae557f6374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sun, 26 Apr 2020 22:51:56 +0200 Subject: [PATCH 68/79] fix linting + update changelog & contributors --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/context_managers.py | 6 +++--- mongoengine/queryset/base.py | 4 +++- tests/test_context_managers.py | 4 ++-- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7d3000ce..02e43955 100644 --- a/AUTHORS +++ b/AUTHORS @@ -256,3 +256,4 @@ that much better: * Eric Timmons (https://github.com/daewok) * Matthew Simpson (https://github.com/mcsimps2) * Leonardo Domingues (https://github.com/leodmgs) + * Agustin Barto (https://github.com/abarto) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76545559..625526a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ Development - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes`` - Added pre-commit #2212 - Renamed requirements-lint.txt to requirements-dev.txt #2212 +- Support for setting ReadConcern #2255 Changes in 0.19.1 ================= diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 77e6b55c..5f2b5229 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -1,7 +1,7 @@ from contextlib import contextmanager -from pymongo.write_concern import WriteConcern from pymongo.read_concern import ReadConcern +from pymongo.write_concern import WriteConcern from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db @@ -268,11 +268,11 @@ def set_read_write_concern(collection, write_concerns, read_concerns): combined_write_concerns.update(write_concerns) combined_read_concerns = dict(collection.read_concern.document.items()) - + if read_concerns is not None: combined_read_concerns.update(read_concerns) yield collection.with_options( write_concern=WriteConcern(**combined_write_concerns), - read_concern=ReadConcern(**combined_read_concerns) + read_concern=ReadConcern(**combined_read_concerns), ) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 317ec698..39c44b29 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1206,7 +1206,9 @@ class BaseQuerySet: raise TypeError("%r is not a valid read concern." % (read_concern,)) queryset = self.clone() - queryset._read_concern = ReadConcern(**read_concern) if read_concern is not None else None + queryset._read_concern = ( + ReadConcern(**read_concern) if read_concern is not None else None + ) queryset._cursor_obj = None # we need to re-create the cursor object whenever we apply read_concern return queryset diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 8f3dd555..a4864c40 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -8,10 +8,10 @@ from mongoengine.context_managers import ( no_dereference, no_sub_classes, query_counter, + set_read_write_concern, + set_write_concern, switch_collection, switch_db, - set_write_concern, - set_read_write_concern, ) from mongoengine.pymongo_support import count_documents From 3fbe9c3cdda00fc7307dfc971df25e4573344d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Fri, 1 May 2020 13:13:30 +0200 Subject: [PATCH 69/79] Bump version to 0.20.0 and update CHANGELOG accordingly --- docs/changelog.rst | 7 +++++-- mongoengine/__init__.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76545559..73f27efe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). + +Changes in 0.20.0 +================= - ATTENTION: Drop support for Python2 - Add Mongo 4.0 to Travis - Fix error when setting a string as a ComplexDateTimeField #2253 @@ -14,12 +17,12 @@ Development - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 - Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 - DictField validate failed without default connection (bug introduced in 0.19.0) #2239 -- Remove methods deprecated years ago: +- Remove methods that were deprecated years ago: - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field - Queryset.slave_okay() was deprecated since pymongo3 - dropDups was dropped with MongoDB3 - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes`` -- Added pre-commit #2212 +- Added pre-commit for development/CI #2212 - Renamed requirements-lint.txt to requirements-dev.txt #2212 Changes in 0.19.1 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index e45dfc2b..dbd88a68 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 19, 1) +VERSION = (0, 20, 0) def get_version(): From d8657be320ca5fdebb1257b70c201d97f7ba8d14 Mon Sep 17 00:00:00 2001 From: Terence Honles Date: Tue, 19 May 2020 10:23:07 -0700 Subject: [PATCH 70/79] Fix requirement Pillow < 7 to mention it is for tests only --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2dd5bcd5..71f9ac5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,7 +28,7 @@ Changes in 0.20.0 Changes in 0.19.1 ================= -- Requires Pillow < 7.0.0 as it dropped Python2 support +- Tests require Pillow < 7.0.0 as it dropped Python2 support - DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 From 22bff8566d35dc0c0d598f0ca391b0ac5f6ed50e Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Tue, 19 May 2020 11:00:30 -0700 Subject: [PATCH 71/79] fix self inflicted deprecation warnings in QNode --- mongoengine/queryset/visitor.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 0eacc2ef..a2448f28 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -7,6 +7,11 @@ from mongoengine.queryset import transform __all__ = ("Q", "QNode") +def warn_empty_is_deprecated(): + msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + + class QNodeVisitor: """Base visitor class for visiting Q-object nodes in a query tree. """ @@ -98,19 +103,18 @@ class QNode: object. """ # If the other Q() is empty, ignore it and just use `self`. - if getattr(other, "empty", True): + if not bool(other): return self # Or if this Q is empty, ignore it and just use `other`. - if self.empty: + if not bool(self): return other return QCombination(operation, [self, other]) @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" - warnings.warn(msg, DeprecationWarning) + warn_empty_is_deprecated() return False def __or__(self, other): @@ -152,8 +156,7 @@ class QCombination(QNode): @property def empty(self): - msg = "'empty' property is deprecated in favour of using 'not bool(filter)'" - warnings.warn(msg, DeprecationWarning) + warn_empty_is_deprecated() return not bool(self.children) def __eq__(self, other): @@ -186,4 +189,5 @@ class Q(QNode): @property def empty(self): + warn_empty_is_deprecated() return not bool(self.query) From 1698f398eb11a85633a1aa3def93386322d648f3 Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Wed, 20 May 2020 18:56:13 -0300 Subject: [PATCH 72/79] Add _read_concern to copied properties. Add read_concern to aggregate. Add test to check the read_concern and read_preference values are kept after cloning. --- mongoengine/queryset/base.py | 6 ++++-- tests/queryset/test_queryset.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 39c44b29..b70fae64 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -789,6 +789,7 @@ class BaseQuerySet: "_snapshot", "_timeout", "_read_preference", + "_read_concern", "_iter", "_scalar", "_as_pymongo", @@ -1311,10 +1312,11 @@ class BaseQuerySet: final_pipeline = initial_pipeline + user_pipeline collection = self._collection - if self._read_preference is not None: + if self._read_preference is not None or self._read_concern is not None: collection = self._collection.with_options( - read_preference=self._read_preference + read_preference=self._read_preference, read_concern=self._read_concern ) + return collection.aggregate(final_pipeline, cursor={}, **kwargs) # JS functionality diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 6b6000c9..88f51104 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4021,6 +4021,32 @@ class TestQueryset(unittest.TestCase): Number.drop_collection() + def test_clone_retains_settings(self): + """Ensure that cloning retains the read_preference and read_concern + """ + + class Number(Document): + n = IntField() + + Number.drop_collection() + + qs = Number.objects + qs_clone = qs.clone() + assert qs._read_preference == qs_clone._read_preference + assert qs._read_concern == qs_clone._read_concern + + qs = Number.objects.read_preference(ReadPreference.PRIMARY_PREFERRED) + qs_clone = qs.clone() + assert qs._read_preference == ReadPreference.PRIMARY_PREFERRED + assert qs._read_preference == qs_clone._read_preference + + qs = Number.objects.read_concern({'level': 'majority'}) + qs_clone = qs.clone() + assert qs._read_concern.document == {'level': 'majority'} + assert qs._read_concern == qs_clone._read_concern + + Number.drop_collection() + def test_using(self): """Ensure that switching databases for a queryset is possible """ From 31498bd7dd1f7316a766aae7f511cd5e4ac4accd Mon Sep 17 00:00:00 2001 From: Agustin Barto Date: Wed, 20 May 2020 18:58:18 -0300 Subject: [PATCH 73/79] Update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2dd5bcd5..acf20195 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). +- Fixed a bug that made the queryset drop the read_preference after clone(). Changes in 0.20.0 ================= From 49f9bca23b081c69ba6a8e0c26102f8cd677d778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 23 May 2020 23:08:56 +0200 Subject: [PATCH 74/79] fix black formatting --- tests/queryset/test_queryset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 88f51104..4900f3a3 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4040,9 +4040,9 @@ class TestQueryset(unittest.TestCase): assert qs._read_preference == ReadPreference.PRIMARY_PREFERRED assert qs._read_preference == qs_clone._read_preference - qs = Number.objects.read_concern({'level': 'majority'}) + qs = Number.objects.read_concern({"level": "majority"}) qs_clone = qs.clone() - assert qs._read_concern.document == {'level': 'majority'} + assert qs._read_concern.document == {"level": "majority"} assert qs._read_concern == qs_clone._read_concern Number.drop_collection() From adb5f74ddbad4478444af24431bf6b4cc8206670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 26 May 2020 23:37:55 +0200 Subject: [PATCH 75/79] Fix a bug in limit0 #2311 --- mongoengine/queryset/base.py | 21 ++++++++++++++----- tests/queryset/test_queryset.py | 36 ++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 39c44b29..f755a05c 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -83,6 +83,7 @@ class BaseQuerySet: self._cursor_obj = None self._limit = None self._skip = None + self._empty = False self._hint = -1 # Using -1 as None is a valid value for hint self._collation = None self._batch_size = None @@ -162,6 +163,7 @@ class BaseQuerySet: [, ] """ queryset = self.clone() + queryset._empty = False # Handle a slice if isinstance(key, slice): @@ -169,6 +171,8 @@ class BaseQuerySet: queryset._skip, queryset._limit = key.start, key.stop if key.start and key.stop: queryset._limit = key.stop - key.start + if queryset._limit == 0: + queryset._empty = True # Allow further QuerySet modifications to be performed return queryset @@ -394,7 +398,12 @@ class BaseQuerySet: :meth:`skip` that has been applied to this cursor into account when getting the count """ - if self._limit == 0 and with_limit_and_skip is False or self._none: + if ( + self._limit == 0 + and with_limit_and_skip is False + or self._none + or self._empty + ): return 0 count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) self._cursor_obj = None @@ -735,7 +744,9 @@ class BaseQuerySet: return doc_map def none(self): - """Helper that just returns a list""" + """Returns a queryset that never returns any objects and no query will be executed when accessing the results + inspired by django none() https://docs.djangoproject.com/en/dev/ref/models/querysets/#none + """ queryset = self.clone() queryset._none = True return queryset @@ -794,6 +805,7 @@ class BaseQuerySet: "_as_pymongo", "_limit", "_skip", + "_empty", "_hint", "_collation", "_auto_dereference", @@ -834,6 +846,7 @@ class BaseQuerySet: """ queryset = self.clone() queryset._limit = n + queryset._empty = False # cancels the effect of empty # If a cursor object has already been created, apply the limit to it. if queryset._cursor_obj: @@ -1584,7 +1597,7 @@ class BaseQuerySet: def __next__(self): """Wrap the result in a :class:`~mongoengine.Document` object. """ - if self._limit == 0 or self._none: + if self._none or self._empty: raise StopIteration raw_doc = next(self._cursor) @@ -1603,8 +1616,6 @@ class BaseQuerySet: return doc - next = __next__ # For Python2 support - def rewind(self): """Rewind the cursor to its unevaluated state. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 6b6000c9..aece69bf 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -114,6 +114,38 @@ class TestQueryset(unittest.TestCase): assert person.name == "User A" assert person.age == 20 + def test_slicing_sets_empty_limit_skip(self): + self.Person.objects.insert( + [self.Person(name="User {}".format(i), age=i) for i in range(5)], + load_bulk=False, + ) + + self.Person.objects.create(name="User B", age=30) + self.Person.objects.create(name="User C", age=40) + + qs = self.Person.objects()[1:2] + assert (qs._empty, qs._skip, qs._limit) == (False, 1, 1) + assert len(list(qs)) == 1 + + # Test edge case of [1:1] which should return nothing + # and require a hack so that it doesn't clash with limit(0) + qs = self.Person.objects()[1:1] + assert (qs._empty, qs._skip, qs._limit) == (True, 1, 0) + assert len(list(qs)) == 0 + + qs2 = qs[1:5] # Make sure that further slicing resets _empty + assert (qs2._empty, qs2._skip, qs2._limit) == (False, 1, 4) + assert len(list(qs2)) == 4 + + def test_limit_0_returns_all_documents(self): + self.Person.objects.create(name="User A", age=20) + self.Person.objects.create(name="User B", age=30) + + n_docs = self.Person.objects().count() + + persons = list(self.Person.objects().limit(0)) + assert len(persons) == 2 == n_docs + def test_limit(self): """Ensure that QuerySet.limit works as expected.""" user_a = self.Person.objects.create(name="User A", age=20) @@ -4442,7 +4474,9 @@ class TestQueryset(unittest.TestCase): assert len(people) == 1 assert people[0] == "User B" - people = list(self.Person.objects[1:1].scalar("name")) + # people = list(self.Person.objects[1:1].scalar("name")) + people = self.Person.objects[1:1] + people = people.scalar("name") assert len(people) == 0 # Test slice out of range From 194b0cac88073a4118a9827b4fbd5ecf9b28dccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 26 May 2020 23:45:35 +0200 Subject: [PATCH 76/79] improve doc + changelog --- docs/changelog.rst | 3 ++- mongoengine/queryset/base.py | 9 ++++++++- tests/queryset/test_queryset.py | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d543b169..819f4e0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,8 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). -- Fixed a bug that made the queryset drop the read_preference after clone(). +- Fix a bug that made the queryset drop the read_preference after clone(). +- Fix the behavior of Doc.objects.limit(0) which should return all documents (similar to mongodb) #2311 Changes in 0.20.0 ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 875cbfda..6ad08617 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -83,7 +83,7 @@ class BaseQuerySet: self._cursor_obj = None self._limit = None self._skip = None - self._empty = False + self._hint = -1 # Using -1 as None is a valid value for hint self._collation = None self._batch_size = None @@ -91,6 +91,13 @@ class BaseQuerySet: self._max_time_ms = None self._comment = None + # Hack - As people expect cursor[5:5] to return + # an empty result set. It's hard to do that right, though, because the + # server uses limit(0) to mean 'no limit'. So we set _empty + # in that case and check for it when iterating. We also unset + # it anytime we change _limit. Inspired by how it is done in pymongo.Cursor + self._empty = False + def __call__(self, q_obj=None, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 36da5d74..73c419b3 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -409,6 +409,9 @@ class TestQueryset(unittest.TestCase): assert list(A.objects.none()) == [] assert list(A.objects.none().all()) == [] + assert list(A.objects.none().limit(1)) == [] + assert list(A.objects.none().skip(1)) == [] + assert list(A.objects.none()[:5]) == [] def test_chaining(self): class A(Document): From e431e27cb27d06290e451b7fbc62e811b264eeac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Sat, 1 Aug 2020 15:09:10 +0200 Subject: [PATCH 77/79] #2360 fix py3 incompatible code --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index db64054a..4a57d511 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -639,7 +639,7 @@ class Document(BaseDocument, metaclass=TopLevelDocumentMetaclass): write_concern=write_concern, _from_doc_delete=True ) except pymongo.errors.OperationFailure as err: - message = "Could not delete document (%s)" % err.message + message = "Could not delete document (%s)" % err.args raise OperationError(message) signals.post_delete.send(self.__class__, document=self, **signal_kwargs) From 3e1c83f8fab28c37ab2e25eab2d7dcc6121f50cc Mon Sep 17 00:00:00 2001 From: Johnny Chang Date: Tue, 4 Aug 2020 00:30:15 +0800 Subject: [PATCH 78/79] Fix query transformation regarding special operators --- mongoengine/fields.py | 3 +++ tests/queryset/test_transform.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b05e726a..f50e6045 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -773,6 +773,9 @@ class EmbeddedDocumentField(BaseField): def prepare_query_value(self, op, value): if value is not None and not isinstance(value, self.document_type): + # Short circuit for special operators, returning them as is + if isinstance(value, dict) and all(k.startswith('$') for k in value.keys()): + return value try: value = self.document_type._from_son(value) except ValueError: diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 8d6c2d06..0fba3975 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -344,6 +344,36 @@ class TestTransform(unittest.TestCase): ) assert update == {"$pull": {"content.text": {"word": {"$nin": ["foo", "bar"]}}}} + def test_transform_embedded_document_list_fields(self): + """ + Test added to check filtering + EmbeddedDocumentListField which is inside a EmbeddedDocumentField + """ + + class Drink(EmbeddedDocument): + id = StringField() + meta = { + 'strict': False + } + + class Shop(Document): + drinks = EmbeddedDocumentListField(Drink) + + Shop.drop_collection() + drinks = [Drink(id='drink_1'), Drink(id='drink_2')] + Shop.objects.create(drinks=drinks) + q_obj = transform.query( + Shop, + drinks__all=[{'$elemMatch': {'_id': x.id}} for x in drinks] + ) + assert q_obj == { + 'drinks': { + '$all': [{'$elemMatch': {'_id': x.id}} for x in drinks] + } + } + + Shop.drop_collection() + if __name__ == "__main__": unittest.main() From 7116dec74a394900d2ffd9bcca5b3318d03598aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastien=20G=C3=A9rard?= Date: Tue, 11 Aug 2020 21:55:22 +0200 Subject: [PATCH 79/79] run black to please ci --- mongoengine/fields.py | 2 +- tests/queryset/test_transform.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f50e6045..8fdeae15 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -774,7 +774,7 @@ class EmbeddedDocumentField(BaseField): def prepare_query_value(self, op, value): if value is not None and not isinstance(value, self.document_type): # Short circuit for special operators, returning them as is - if isinstance(value, dict) and all(k.startswith('$') for k in value.keys()): + if isinstance(value, dict) and all(k.startswith("$") for k in value.keys()): return value try: value = self.document_type._from_son(value) diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 0fba3975..f5d248af 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -352,24 +352,19 @@ class TestTransform(unittest.TestCase): class Drink(EmbeddedDocument): id = StringField() - meta = { - 'strict': False - } + meta = {"strict": False} class Shop(Document): drinks = EmbeddedDocumentListField(Drink) Shop.drop_collection() - drinks = [Drink(id='drink_1'), Drink(id='drink_2')] + drinks = [Drink(id="drink_1"), Drink(id="drink_2")] Shop.objects.create(drinks=drinks) q_obj = transform.query( - Shop, - drinks__all=[{'$elemMatch': {'_id': x.id}} for x in drinks] + Shop, drinks__all=[{"$elemMatch": {"_id": x.id}} for x in drinks] ) assert q_obj == { - 'drinks': { - '$all': [{'$elemMatch': {'_id': x.id}} for x in drinks] - } + "drinks": {"$all": [{"$elemMatch": {"_id": x.id}} for x in drinks]} } Shop.drop_collection()