diff --git a/.travis.yml b/.travis.yml index ad659fca..d34f8a36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ # For full coverage, we'd have to test all supported Python, MongoDB, and # PyMongo combinations. However, that would result in an overly long build # with a very large number of jobs, hence we only test a subset of all the -# combinations: -# * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, v3.7, v3.8, PyPy and PyPy3. -# * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo -# combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. -# * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. -# +# combinations. +# * Python3.7, MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, +# Other combinations are tested. See below for the details or check the travis jobs + # We should periodically check MongoDB Server versions supported by MongoDB # Inc., add newly released versions to the test matrix, and remove versions # which have reached their End of Life. See: @@ -16,21 +13,15 @@ # # Reminder: Update README.rst if you change MongoDB versions we test. - language: python +dist: xenial python: -- 2.7 - 3.5 - 3.6 - 3.7 - 3.8 -- pypy - pypy3 -dist: xenial - -dist: xenial - env: global: - MONGODB_3_4=3.4.17 @@ -41,6 +32,8 @@ env: - PYMONGO_3_6=3.6 - PYMONGO_3_9=3.9 - PYMONGO_3_10=3.10 + + - MAIN_PYTHON_VERSION = "3.7" matrix: - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} @@ -49,8 +42,6 @@ matrix: fast_finish: true include: - - python: 2.7 - env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 @@ -74,20 +65,20 @@ install: # tox dryrun to setup the tox venv (we run a mock test). - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder" # Install black for Python v3.7 only. - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then pip install black; fi + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pip install black; fi before_script: - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then flake8 .; else echo "flake8 only runs on py37"; fi # Run flake8 for Python 3.7 only - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then black --check .; else echo "black only runs on py37"; fi # Run black for Python 3.7 only + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then flake8 .; else echo "flake8 only runs on py37"; fi + - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then black --check .; else echo "black only runs on py37"; fi - mongo --eval 'db.version();' # Make sure mongo is awake script: - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine" after_success: -- coveralls --verbose +- - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi notifications: irc: irc.freenode.org#mongoengine @@ -109,11 +100,11 @@ deploy: distributions: "sdist bdist_wheel" # Only deploy on tagged commits (aka GitHub releases) and only for the parent - # repo's builds running Python v2.7 along with PyMongo v3.x and MongoDB v3.4. + # repo's builds running Python v3.7 along with PyMongo v3.x and MongoDB v3.4. # We run Travis against many different Python, PyMongo, and MongoDB versions # and we don't want the deploy to occur multiple times). on: tags: true repo: MongoEngine/mongoengine condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4}) - python: 2.7 + python: 3.7 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b04ae968..0deda0fc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -20,23 +20,23 @@ post to the `user group ` Supported Interpreters ---------------------- -MongoEngine supports CPython 2.7 and newer. Language -features not supported by all interpreters can not be used. +MongoEngine supports CPython 3.7 and newer as well as Pypy3. +Language features not supported by all interpreters can not be used. -Python 2/3 compatibility +Python3 codebase ---------------------- -The codebase is written in a compatible manner for python 2 & 3 so it -is important that this is taken into account when it comes to discrepencies -between the two versions (see https://python-future.org/compatible_idioms.html). -Travis runs the tests against different Python versions as a safety net. +Since 0.20, the codebase is exclusively Python 3. + +Earlier versions were exclusively Python2, and was relying on 2to3 to support Python3 installs. +Travis runs the tests against the main Python 3.x versions. Style Guide ----------- -MongoEngine uses `black `_ for code -formatting. +MongoEngine uses `black `_ for code formatting. +Black runs as part of the CI so it will fail in case the code is not formatted properly. Testing ------- diff --git a/benchmarks/test_inserts.py b/benchmarks/test_inserts.py index af6399f7..4ecd48de 100644 --- a/benchmarks/test_inserts.py +++ b/benchmarks/test_inserts.py @@ -3,8 +3,6 @@ import timeit def main(): setup = """ -from builtins import range - from pymongo import MongoClient connection = MongoClient() @@ -59,8 +57,6 @@ myNoddys = noddy.find() print("{}s".format(t.timeit(1))) setup = """ -from builtins import range - from pymongo import MongoClient connection = MongoClient() diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 9f78fec0..f2d64c33 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,7 +1,6 @@ import weakref from bson import DBRef -from future.utils import listitems import six from six import iteritems @@ -181,19 +180,6 @@ class BaseList(list): __iadd__ = mark_as_changed_wrapper(list.__iadd__) __imul__ = mark_as_changed_wrapper(list.__imul__) - if six.PY2: - # Under py3 __setslice__, __delslice__ and __getslice__ - # are replaced by __setitem__, __delitem__ and __getitem__ with a slice as parameter - # so we mimic this under python 2 - def __setslice__(self, i, j, sequence): - return self.__setitem__(slice(i, j), sequence) - - def __delslice__(self, i, j): - return self.__delitem__(slice(i, j)) - - def __getslice__(self, i, j): - return self.__getitem__(slice(i, j)) - def _mark_as_changed(self, key=None): if hasattr(self._instance, "_mark_as_changed"): if key: @@ -426,7 +412,7 @@ class StrictDict(object): return len(list(iteritems(self))) def __eq__(self, other): - return listitems(self) == listitems(other) + return list(self.items()) == list(other.items()) def __ne__(self, other): return not (self == other) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 4e4df92f..73c88774 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,9 +1,9 @@ import copy + import numbers from functools import partial from bson import DBRef, ObjectId, SON, json_util -from future.utils import listitems import pymongo import six from six import iteritems @@ -26,7 +26,6 @@ from mongoengine.errors import ( OperationError, ValidationError, ) -from mongoengine.python_support import Hashable __all__ = ("BaseDocument", "NON_FIELD_ERRORS") @@ -294,10 +293,7 @@ class BaseDocument(object): def __str__(self): # TODO this could be simpler? if hasattr(self, "__unicode__"): - if six.PY3: - return self.__unicode__() - else: - return six.text_type(self).encode("utf-8") + return self.__unicode__() return six.text_type("%s object" % self.__class__.__name__) def __eq__(self, other): @@ -671,7 +667,7 @@ class BaseDocument(object): del set_data["_id"] # Determine if any changed items were actually unset. - for path, value in listitems(set_data): + for path, value in list(set_data.items()): if value or isinstance( value, (numbers.Number, bool) ): # Account for 0 and True that are truthy diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9d7d1d04..5cbf3ae0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -5,12 +5,12 @@ import re import socket import time import uuid +from io import BytesIO from operator import itemgetter from bson import Binary, DBRef, ObjectId, SON from bson.int64 import Int64 import gridfs -from past.builtins import long import pymongo from pymongo import ReturnDocument import six @@ -39,7 +39,6 @@ from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.document import Document, EmbeddedDocument from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version -from mongoengine.python_support import StringIO from mongoengine.queryset import DO_NOTHING from mongoengine.queryset.base import BaseQuerySet from mongoengine.queryset.transform import STRING_OPERATORS @@ -338,7 +337,7 @@ class IntField(BaseField): class LongField(BaseField): - """64-bit integer field.""" + """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)""" def __init__(self, min_value=None, max_value=None, **kwargs): self.min_value, self.max_value = min_value, max_value @@ -346,7 +345,7 @@ class LongField(BaseField): def to_python(self, value): try: - value = long(value) + value = int(value) except (TypeError, ValueError): pass return value @@ -356,7 +355,7 @@ class LongField(BaseField): def validate(self, value): try: - value = long(value) + value = int(value) except (TypeError, ValueError): self.error("%s could not be converted to long" % value) @@ -370,7 +369,7 @@ class LongField(BaseField): if value is None: return value - return super(LongField, self).prepare_query_value(op, long(value)) + return super(LongField, self).prepare_query_value(op, int(value)) class FloatField(BaseField): @@ -1679,8 +1678,6 @@ class GridFSProxy(object): def __bool__(self): return bool(self.grid_id) - __nonzero__ = __bool__ # For Py2 support - def __getstate__(self): self_dict = self.__dict__ self_dict["_fs"] = None @@ -1952,7 +1949,7 @@ class ImageGridFsProxy(GridFSProxy): w, h = img.size - io = StringIO() + io = BytesIO() img.save(io, img_format, progressive=progressive) io.seek(0) @@ -1971,7 +1968,7 @@ class ImageGridFsProxy(GridFSProxy): def _put_thumbnail(self, thumbnail, format, progressive, **kwargs): w, h = thumbnail.size - io = StringIO() + io = BytesIO() thumbnail.save(io, format, progressive=progressive) io.seek(0) diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py deleted file mode 100644 index 57e467db..00000000 --- a/mongoengine/python_support.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Helper functions, constants, and types to aid with Python v2.7 - v3.x support -""" -import six - -# six.BytesIO resolves to StringIO.StringIO in Py2 and io.BytesIO in Py3. -StringIO = six.BytesIO - -# Additionally for Py2, try to use the faster cStringIO, if available -if not six.PY3: - try: - import cStringIO - except ImportError: - pass - else: - StringIO = cStringIO.StringIO - - -if six.PY3: - from collections.abc import Hashable -else: - # raises DeprecationWarnings in Python >=3.7 - from collections import Hashable diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 9f76d92d..b7d0e0ab 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import copy import itertools import re @@ -204,8 +202,6 @@ class BaseQuerySet(object): """Avoid to open all records in an if stmt in Py3.""" return self._has_data() - __nonzero__ = __bool__ # For Py2 support - # Core functions def all(self): diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 5c3ff222..c2618ebd 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -69,8 +69,6 @@ class QueryFieldList(object): def __bool__(self): return bool(self.fields) - __nonzero__ = __bool__ # For Py2 support - def as_dict(self): field_list = {field: self.value for field in self.fields} if self.slice: diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 659a97e2..c179f541 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -10,7 +10,7 @@ from mongoengine.base import UPDATE_OPERATORS from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError -__all__ = ("query", "update") +__all__ = ("query", "update", "STRING_OPERATORS") COMPARISON_OPERATORS = ( "ne", diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 470839c1..72e36ac0 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -143,8 +143,6 @@ class QCombination(QNode): def __bool__(self): return bool(self.children) - __nonzero__ = __bool__ # For Py2 support - def accept(self, visitor): for i in range(len(self.children)): if isinstance(self.children[i], QNode): @@ -180,8 +178,6 @@ class Q(QNode): def __bool__(self): return bool(self.query) - __nonzero__ = __bool__ # For Py2 support - def __eq__(self, other): return self.__class__ == other.__class__ and self.query == other.query diff --git a/setup.py b/setup.py index 418ff7ad..6873284a 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,6 @@ CLASSIFIERS = [ PYTHON_VERSION = sys.version_info[0] PY3 = PYTHON_VERSION == 3 -PY2 = PYTHON_VERSION == 2 extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), @@ -124,14 +123,11 @@ extra_opts = { ], } -if PY3: - if "test" in sys.argv: - extra_opts["packages"] = find_packages() - extra_opts["package_data"] = { - "tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"] - } -else: - extra_opts["tests_require"] += ["python-dateutil"] +if "test" in sys.argv: + extra_opts["packages"] = find_packages() + extra_opts["package_data"] = { + "tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"] + } setup( name="mongoengine", @@ -148,7 +144,8 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0", "future"], + python_requires=">=3.5", + install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/fields/test_binary_field.py b/tests/fields/test_binary_field.py index e2a1b8d6..a653b961 100644 --- a/tests/fields/test_binary_field.py +++ b/tests/fields/test_binary_field.py @@ -123,10 +123,7 @@ class TestBinaryField(MongoDBTestCase): upsert=True, new=True, set__bin_field=BIN_VALUE ) assert doc.some_field == "test" - if six.PY3: - assert doc.bin_field == BIN_VALUE - else: - assert doc.bin_field == Binary(BIN_VALUE) + assert doc.bin_field == BIN_VALUE def test_update_one(self): """Ensures no regression of bug #1127""" @@ -144,7 +141,4 @@ class TestBinaryField(MongoDBTestCase): ) assert n_updated == 1 fetched = MyDocument.objects.with_id(doc.id) - if six.PY3: - assert fetched.bin_field == BIN_VALUE - else: - assert fetched.bin_field == Binary(BIN_VALUE) + assert fetched.bin_field == BIN_VALUE diff --git a/tests/fields/test_embedded_document_field.py b/tests/fields/test_embedded_document_field.py index b80f4d8c..13ca9c0b 100644 --- a/tests/fields/test_embedded_document_field.py +++ b/tests/fields/test_embedded_document_field.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from builtins import str - import pytest from mongoengine import ( diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py index b8ece1a9..5ab6c93f 100644 --- a/tests/fields/test_file_field.py +++ b/tests/fields/test_file_field.py @@ -3,6 +3,7 @@ import copy import os import tempfile import unittest +from io import BytesIO import gridfs import pytest @@ -10,7 +11,6 @@ import six from mongoengine import * from mongoengine.connection import get_db -from mongoengine.python_support import StringIO try: from PIL import Image @@ -30,7 +30,7 @@ TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), "mongodb_leaf.png") def get_file(path): """Use a BytesIO instead of a file to allow to have a one-liner and avoid that the file remains opened""" - bytes_io = StringIO() + bytes_io = BytesIO() with open(path, "rb") as f: bytes_io.write(f.read()) bytes_io.seek(0) @@ -80,7 +80,7 @@ class TestFileField(MongoDBTestCase): PutFile.drop_collection() putfile = PutFile() - putstring = StringIO() + putstring = BytesIO() putstring.write(text) putstring.seek(0) putfile.the_file.put(putstring, content_type=content_type) diff --git a/tests/fields/test_url_field.py b/tests/fields/test_url_field.py index 477bced7..c449e467 100644 --- a/tests/fields/test_url_field.py +++ b/tests/fields/test_url_field.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from builtins import str - import pytest from mongoengine import * diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 17bf7405..7dea6dd5 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4470,10 +4470,7 @@ class TestQueryset(unittest.TestCase): pks = self.Person.objects.order_by("age").scalar("pk")[1:3] names = self.Person.objects.scalar("name").in_bulk(list(pks)).values() - if six.PY3: - expected = "['A1', 'A2']" - else: - expected = "[u'A1', u'A2']" + expected = "['A1', 'A2']" assert expected == "%s" % sorted(names) def test_elem_match(self): diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py index 7b5d7d11..734061ed 100644 --- a/tests/test_datastructures.py +++ b/tests/test_datastructures.py @@ -287,7 +287,7 @@ class TestBaseList: base_list[:] = [ 0, 1, - ] # Will use __setslice__ under py2 and __setitem__ under py3 + ] assert base_list._instance._changed_fields == ["my_name"] assert base_list == [0, 1] @@ -296,13 +296,13 @@ class TestBaseList: base_list[0:2] = [ 1, 0, - ] # Will use __setslice__ under py2 and __setitem__ under py3 + ] assert base_list._instance._changed_fields == ["my_name"] assert base_list == [1, 0, 2] def test___setitem___calls_with_step_slice_mark_as_changed(self): base_list = self._get_baselist([0, 1, 2]) - base_list[0:3:2] = [-1, -2] # uses __setitem__ in both py2 & 3 + base_list[0:3:2] = [-1, -2] # uses __setitem__ assert base_list._instance._changed_fields == ["my_name"] assert base_list == [-1, 1, -2] diff --git a/tox.ini b/tox.ini index 396817ca..675b6d9a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310} +envlist = {py35,pypy3}-{mg34,mg36,mg39,mg310} [testenv] commands =