Merge pull request #2497 from bagerard/switch_to_isort

Switch to isort + add few pre-commit hooks

.github/workflows/github-actions.yml

@@ -140,4 +140,4 @@ jobs:
     - name: publish pypi
       uses: pypa/gh-action-pypi-publish@master
       with:
-        password: ${{ secrets.pypi_token }}
+        password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,17 +1,26 @@
 fail_fast: false
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
   - repo: https://github.com/ambv/black
     rev: 20.8b1
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
     hooks:
       - id: flake8
-        additional_dependencies:
-          - flake8-import-order
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.4
+    rev: v2.11.0
     hooks:
       - id: pyupgrade
         args: [--py36-plus]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.8.0
+    hooks:
+      - id: isort

CONTRIBUTING.rst

@@ -35,8 +35,8 @@ Travis runs the tests against the main Python 3.x versions.
 Style Guide
 -----------
 
-MongoEngine's codebase is formatted with `black <https://github.com/python/black>`_, other tools like
+MongoEngine's codebase is auto-formatted with `black <https://github.com/python/black>`_, imports are ordered with `isort <https://pycqa.github.io/isort/>`_
-flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
+and other tools like flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
 
 To install all development tools, simply run the following commands:
 
@@ -58,6 +58,10 @@ To enable ``pre-commit`` simply run:
 See the ``.pre-commit-config.yaml`` configuration file for more information
 on how it works.
 
+pre-commit will now run upon every commit and will reject anything that doesn't comply.
+
+You can also run all the checks with ``pre-commit run -a``, this is what is used in the CI.
+
 Testing
 -------
 

docs/guide/validation.rst

@@ -120,4 +120,3 @@ the validation and cleaning of a document when you call :meth:`~mongoengine.docu
     Person(age=1000).save(validate=False)
     person = Person.objects.first()
     assert person.age == 1000
-

mongoengine/__init__.py

@@ -1,10 +1,12 @@
 # Import submodules so that we can expose their __all__
-from mongoengine import connection
+from mongoengine import (
-from mongoengine import document
+    connection,
-from mongoengine import errors
+    document,
-from mongoengine import fields
+    errors,
-from mongoengine import queryset
+    fields,
-from mongoengine import signals
+    queryset,
+    signals,
+)
 
 # Import everything from each submodule so that it can be accessed via
 # mongoengine, e.g. instead of `from mongoengine.connection import connect`,
@@ -17,7 +19,6 @@ from mongoengine.fields import *  # noqa: F401
 from mongoengine.queryset import *  # noqa: F401
 from mongoengine.signals import *  # noqa: F401
 
-
 __all__ = (
     list(document.__all__)
     + list(fields.__all__)

mongoengine/base/document.py

@@ -1,10 +1,9 @@
 import copy
-
 import numbers
 from functools import partial
 
-from bson import DBRef, ObjectId, SON, json_util
 import pymongo
+from bson import SON, DBRef, ObjectId, json_util
 
 from mongoengine import signals
 from mongoengine.base.common import get_document

mongoengine/base/fields.py

@@ -1,11 +1,15 @@
 import operator
 import weakref
 
-from bson import DBRef, ObjectId, SON
 import pymongo
+from bson import SON, DBRef, ObjectId
 
 from mongoengine.base.common import UPDATE_OPERATORS
-from mongoengine.base.datastructures import BaseDict, BaseList, EmbeddedDocumentList
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    EmbeddedDocumentList,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import DeprecatedError, ValidationError
 

mongoengine/base/metaclasses.py

@@ -2,7 +2,11 @@ import itertools
 import warnings
 
 from mongoengine.base.common import _document_registry
-from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
+from mongoengine.base.fields import (
+    BaseField,
+    ComplexBaseField,
+    ObjectIdField,
+)
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidDocumentError
 from mongoengine.queryset import (
@@ -12,7 +16,6 @@ from mongoengine.queryset import (
     QuerySetManager,
 )
 
-
 __all__ = ("DocumentMetaclass", "TopLevelDocumentMetaclass")
 
 

mongoengine/connection.py

@@ -217,8 +217,8 @@ def register_connection(
 
 def disconnect(alias=DEFAULT_CONNECTION_NAME):
     """Close the connection with a given alias."""
-    from mongoengine.base.common import _get_documents_by_db
     from mongoengine import Document
+    from mongoengine.base.common import _get_documents_by_db
 
     if alias in _connections:
         get_connection(alias=alias).close()

mongoengine/dereference.py

@@ -1,4 +1,4 @@
-from bson import DBRef, SON
+from bson import SON, DBRef
 
 from mongoengine.base import (
     BaseDict,
@@ -10,7 +10,12 @@ from mongoengine.base import (
 from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.fields import DictField, ListField, MapField, ReferenceField
+from mongoengine.fields import (
+    DictField,
+    ListField,
+    MapField,
+    ReferenceField,
+)
 from mongoengine.queryset import QuerySet
 
 

mongoengine/document.py

@@ -1,7 +1,7 @@
 import re
 
-from bson.dbref import DBRef
 import pymongo
+from bson.dbref import DBRef
 from pymongo.read_preferences import ReadPreference
 
 from mongoengine import signals
@@ -16,14 +16,23 @@ from mongoengine.base import (
 )
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
-from mongoengine.context_managers import set_write_concern, switch_collection, switch_db
+from mongoengine.context_managers import (
+    set_write_concern,
+    switch_collection,
+    switch_db,
+)
 from mongoengine.errors import (
     InvalidDocumentError,
     InvalidQueryError,
     SaveConditionError,
 )
 from mongoengine.pymongo_support import list_collection_names
-from mongoengine.queryset import NotUniqueError, OperationError, QuerySet, transform
+from mongoengine.queryset import (
+    NotUniqueError,
+    OperationError,
+    QuerySet,
+    transform,
+)
 
 __all__ = (
     "Document",

mongoengine/errors.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 
-
 __all__ = (
     "NotRegistered",
     "InvalidDocumentError",

mongoengine/fields.py

@@ -9,10 +9,10 @@ import uuid
 from io import BytesIO
 from operator import itemgetter
 
-from bson import Binary, DBRef, ObjectId, SON
-from bson.int64 import Int64
 import gridfs
 import pymongo
+from bson import SON, Binary, DBRef, ObjectId
+from bson.int64 import Int64
 from pymongo import ReturnDocument
 
 try:
@@ -22,7 +22,6 @@ except ImportError:
 else:
     import dateutil.parser
 
-
 from mongoengine.base import (
     BaseDocument,
     BaseField,
@@ -36,7 +35,11 @@ from mongoengine.base.utils import LazyRegexCompiler
 from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
-from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
+from mongoengine.errors import (
+    DoesNotExist,
+    InvalidQueryError,
+    ValidationError,
+)
 from mongoengine.queryset import DO_NOTHING
 from mongoengine.queryset.base import BaseQuerySet
 from mongoengine.queryset.transform import STRING_OPERATORS

mongoengine/mongodb_support.py

@@ -3,7 +3,6 @@ Helper functions, constants, and types to aid with MongoDB version support
 """
 from mongoengine.connection import get_connection
 
-
 # Constant that can be used to compare the version retrieved with
 # get_mongodb_version()
 MONGODB_34 = (3, 4)

mongoengine/queryset/base.py

@@ -2,13 +2,12 @@ import copy
 import itertools
 import re
 import warnings
-
 from collections.abc import Mapping
 
-from bson import SON, json_util
-from bson.code import Code
 import pymongo
 import pymongo.errors
+from bson import SON, json_util
+from bson.code import Code
 from pymongo.collection import ReturnDocument
 from pymongo.common import validate_read_preference
 from pymongo.read_concern import ReadConcern
@@ -34,7 +33,6 @@ from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
 from mongoengine.queryset.visitor import Q, QNode
 
-
 __all__ = ("BaseQuerySet", "DO_NOTHING", "NULLIFY", "CASCADE", "DENY", "PULL")
 
 # Delete rules

mongoengine/queryset/manager.py

@@ -1,4 +1,5 @@
 from functools import partial
+
 from mongoengine.queryset.queryset import QuerySet
 
 __all__ = ("queryset_manager", "QuerySetManager")

mongoengine/queryset/queryset.py

@@ -1,11 +1,11 @@
 from mongoengine.errors import OperationError
 from mongoengine.queryset.base import (
-    BaseQuerySet,
     CASCADE,
     DENY,
     DO_NOTHING,
     NULLIFY,
     PULL,
+    BaseQuerySet,
 )
 
 __all__ = (

mongoengine/queryset/transform.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
 
-from bson import ObjectId, SON
-from bson.dbref import DBRef
 import pymongo
+from bson import SON, ObjectId
+from bson.dbref import DBRef
 
 from mongoengine.base import UPDATE_OPERATORS
 from mongoengine.common import _import_class

requirements-dev.txt

@@ -1,6 +1,5 @@
 black
 flake8
-flake8-import-order
 pre-commit
 pytest
 ipdb

setup.cfg

@@ -2,9 +2,17 @@
 ignore=E501,F403,F405,I201,I202,W504,W605,W503,B007
 exclude=build,dist,docs,venv,venv3,.tox,.eggs,tests
 max-complexity=47
-application-import-names=mongoengine,tests
 
 [tool:pytest]
 # Limits the discovery to tests directory
 # avoids that it runs for instance the benchmark
 testpaths = tests
+
+[isort]
+known_first_party = mongoengine,tests
+default_section = THIRDPARTY
+multi_line_output = 3
+include_trailing_comma = True
+combine_as_imports = True
+line_length = 70
+ensure_newline_before_comments = 1

setup.py

@@ -53,8 +53,8 @@ class PyTest(TestCommand):
 
     def run_tests(self):
         # import here, cause outside the eggs aren't loaded
-        from pkg_resources import _namespace_packages
         import pytest
+        from pkg_resources import _namespace_packages
 
         # Purge modules under test from sys.modules. The test loader will
         # re-import them from the build location. Required when 2to3 is used

tests/document/test_delta.py

@@ -1,6 +1,7 @@
 import unittest
 
 from bson import SON
+
 from mongoengine import *
 from mongoengine.pymongo_support import list_collection_names
 from tests.utils import MongoDBTestCase

tests/document/test_indexes.py

@@ -1,13 +1,16 @@
 import unittest
 from datetime import datetime
 
+import pytest
 from pymongo.collation import Collation
 from pymongo.errors import OperationFailure
-import pytest
 
 from mongoengine import *
 from mongoengine.connection import get_db
-from mongoengine.mongodb_support import MONGODB_42, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_42,
+    get_mongodb_version,
+)
 
 
 class TestIndexes(unittest.TestCase):

tests/document/test_instance.py

@@ -6,9 +6,9 @@ import weakref
 from datetime import datetime
 
 import bson
+import pytest
 from bson import DBRef, ObjectId
 from pymongo.errors import DuplicateKeyError
-import pytest
 
 from mongoengine import *
 from mongoengine import signals
@@ -23,7 +23,11 @@ from mongoengine.errors import (
     NotUniqueError,
     SaveConditionError,
 )
-from mongoengine.mongodb_support import MONGODB_34, MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_34,
+    MONGODB_36,
+    get_mongodb_version,
+)
 from mongoengine.pymongo_support import list_collection_names
 from mongoengine.queryset import NULLIFY, Q
 from tests import fixtures

tests/document/test_json_serialisation.py

@@ -1,7 +1,7 @@
 import unittest
 import uuid
-
 from datetime import datetime
+
 from bson import ObjectId
 
 from mongoengine import *

tests/fields/test_binary_field.py

@@ -1,7 +1,7 @@
 import uuid
 
-from bson import Binary
 import pytest
+from bson import Binary
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase

tests/fields/test_complex_datetime_field.py

@@ -6,7 +6,6 @@ import re
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_datetime_field.py

@@ -9,7 +9,6 @@ except ImportError:
 
 from mongoengine import *
 from mongoengine import connection
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_dict_field.py

@@ -1,10 +1,12 @@
-from bson import InvalidDocument
 import pytest
+from bson import InvalidDocument
 
 from mongoengine import *
 from mongoengine.base import BaseDict
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
-
+    MONGODB_36,
+    get_mongodb_version,
+)
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
 

tests/fields/test_embedded_document_field.py

@@ -12,7 +12,6 @@ from mongoengine import (
     StringField,
     ValidationError,
 )
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_enum_field.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
-from bson import InvalidDocument
 import pytest
+from bson import InvalidDocument
 
 from mongoengine import Document, EnumField, ValidationError
 from tests.utils import MongoDBTestCase, get_as_pymongo

tests/fields/test_fields.py

@@ -1,8 +1,8 @@
 import datetime
 import unittest
 
-from bson import DBRef, ObjectId, SON
 import pytest
+from bson import SON, DBRef, ObjectId
 
 from mongoengine import (
     BooleanField,
@@ -34,9 +34,12 @@ from mongoengine import (
     StringField,
     ValidationError,
 )
-from mongoengine.base import BaseField, EmbeddedDocumentList, _document_registry
+from mongoengine.base import (
+    BaseField,
+    EmbeddedDocumentList,
+    _document_registry,
+)
 from mongoengine.errors import DeprecatedError
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_float_field.py

@@ -1,7 +1,6 @@
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_int_field.py

@@ -1,7 +1,6 @@
 import pytest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_lazy_reference_field.py

@@ -1,10 +1,9 @@
-from bson import DBRef, ObjectId
 import pytest
+from bson import DBRef, ObjectId
 
 from mongoengine import *
 from mongoengine.base import LazyReference
 from mongoengine.context_managers import query_counter
-
 from tests.utils import MongoDBTestCase
 
 

tests/fields/test_long_field.py

@@ -1,9 +1,8 @@
-from bson.int64 import Int64
 import pytest
+from bson.int64 import Int64
 
 from mongoengine import *
 from mongoengine.connection import get_db
-
 from tests.utils import MongoDBTestCase, get_as_pymongo
 
 

tests/fields/test_reference_field.py

@@ -1,5 +1,5 @@
-from bson import DBRef, SON
 import pytest
+from bson import SON, DBRef
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase

tests/fields/test_sequence_field.py

@@ -1,5 +1,4 @@
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/queryset/test_geo.py

@@ -2,7 +2,6 @@ import datetime
 import unittest
 
 from mongoengine import *
-
 from tests.utils import MongoDBTestCase
 
 

tests/queryset/test_modify.py

@@ -1,6 +1,12 @@
 import unittest
 
-from mongoengine import Document, IntField, ListField, StringField, connect
+from mongoengine import (
+    Document,
+    IntField,
+    ListField,
+    StringField,
+    connect,
+)
 
 
 class Doc(Document):

tests/queryset/test_queryset.py

@@ -3,17 +3,20 @@ import unittest
 import uuid
 from decimal import Decimal
 
-from bson import DBRef, ObjectId
 import pymongo
+import pytest
+from bson import DBRef, ObjectId
 from pymongo.read_preferences import ReadPreference
 from pymongo.results import UpdateResult
-import pytest
 
 from mongoengine import *
 from mongoengine.connection import get_db
 from mongoengine.context_managers import query_counter, switch_db
 from mongoengine.errors import InvalidQueryError
-from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version
+from mongoengine.mongodb_support import (
+    MONGODB_36,
+    get_mongodb_version,
+)
 from mongoengine.queryset import (
     DoesNotExist,
     MultipleObjectsReturned,

tests/queryset/test_transform.py

@@ -1,7 +1,7 @@
 import unittest
 
-from bson.son import SON
 import pytest
+from bson.son import SON
 
 from mongoengine import *
 from mongoengine.queryset import Q, transform

tests/queryset/test_visitor.py

@@ -2,8 +2,8 @@ import datetime
 import re
 import unittest
 
-from bson import ObjectId
 import pytest
+from bson import ObjectId
 
 from mongoengine import *
 from mongoengine.errors import InvalidQueryError

tests/test_connection.py

@@ -1,11 +1,11 @@
 import datetime
 import unittest
 
-from bson.tz_util import utc
 import pymongo
+import pytest
+from bson.tz_util import utc
 from pymongo import MongoClient, ReadPreference
 from pymongo.errors import InvalidName, OperationFailure
-import pytest
 
 import mongoengine.connection
 from mongoengine import (
@@ -17,8 +17,8 @@ from mongoengine import (
     register_connection,
 )
 from mongoengine.connection import (
-    ConnectionFailure,
     DEFAULT_DATABASE_NAME,
+    ConnectionFailure,
     disconnect,
     get_connection,
     get_db,

tests/test_connection_mongomock.py

@@ -3,15 +3,9 @@ import unittest
 import pytest
 
 import mongoengine.connection
-from mongoengine import (
+from mongoengine import Document, StringField, connect, disconnect_all
-    Document,
-    StringField,
-    connect,
-    disconnect_all,
-)
 from mongoengine.connection import get_connection
 
-
 try:
     import mongomock
 

tests/test_datastructures.py

@@ -3,7 +3,11 @@ import unittest
 import pytest
 
 from mongoengine import Document
-from mongoengine.base.datastructures import BaseDict, BaseList, StrictDict
+from mongoengine.base.datastructures import (
+    BaseDict,
+    BaseList,
+    StrictDict,
+)
 
 
 class DocumentStub:

tests/test_replicaset_connection.py

@@ -5,7 +5,6 @@ from pymongo import MongoClient, ReadPreference
 import mongoengine
 from mongoengine.connection import ConnectionFailure
 
-
 CONN_CLASS = MongoClient
 READ_PREF = ReadPreference.SECONDARY
 

tests/utils.py

@@ -7,7 +7,6 @@ from mongoengine import connect
 from mongoengine.connection import disconnect_all, get_db
 from mongoengine.mongodb_support import get_mongodb_version
 
-
 MONGO_TEST_DB = "mongoenginetest"  # standard name for the test database