Merge pull request #2310 from bagerard/Pacu2-add-dev-tools

Pacu2 add dev tools

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+fail_fast: false
+repos:
+    - repo: https://github.com/ambv/black
+      rev: 19.10b0
+      hooks:
+        - id: black
+    - repo: https://gitlab.com/pycqa/flake8
+      rev: 3.8.0a2
+      hooks:
+        - id: flake8
+          additional_dependencies:
+            - flake8-import-order

.travis.yml

@@ -33,7 +33,7 @@ env:
     - PYMONGO_3_9=3.9
     - PYMONGO_3_10=3.10
 
-    - MAIN_PYTHON_VERSION = "3.7"
+    - MAIN_PYTHON_VERSION=3.7
   matrix:
     - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
 
@@ -59,18 +59,16 @@ install:
   # Install Python dependencies.
   - pip install --upgrade pip
   - pip install coveralls
-  - pip install flake8 flake8-import-order
+  - pip install pre-commit
   - pip install tox
   # tox dryrun to setup the tox venv (we run a mock test).
   - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
-  # Install black for Python v3.7 only.
-  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pip install black; fi
 
 before_script:
   - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data
   - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
-  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then flake8 .; else echo "flake8 only runs on py37"; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then black --check .; else echo "black only runs on py37"; fi
+  # Run pre-commit hooks (black, flake8, etc) on entire codebase
+  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
   - mongo --eval 'db.version();'    # Make sure mongo is awake
 
 script:

CONTRIBUTING.rst

@@ -35,8 +35,28 @@ Travis runs the tests against the main Python 3.x versions.
 Style Guide
 -----------
 
-MongoEngine uses `black <https://github.com/python/black>`_ for code formatting.
-Black runs as part of the CI so it will fail in case the code is not formatted properly.
+MongoEngine's codebase is formatted with `black <https://github.com/python/black>`_, other tools like
+flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
+
+To install all development tools, simply run the following commands:
+
+.. code-block:: console
+
+    $ python -m pip install -r requirements-dev.txt
+
+
+You can install `pre-commit <https://pre-commit.com/>`_ into your git hooks,
+to automatically check and fix any formatting issue before creating a
+git commit.
+
+To enable ``pre-commit`` simply run:
+
+.. code-block:: console
+
+    $ pre-commit install
+
+See the ``.pre-commit-config.yaml`` configuration file for more information
+on how it works.
 
 Testing
 -------

README.rst

@@ -34,7 +34,7 @@ with MongoDB version > 4.0.
 Installation
 ============
 We recommend the use of `virtualenv <https://virtualenv.pypa.io/>`_ and of
-`pip <https://pip.pypa.io/>`_. You can then use ``pip install -U mongoengine``.
+`pip <https://pip.pypa.io/>`_. You can then use ``python -m pip install -U mongoengine``.
 You may also have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
 and thus you can use ``easy_install -U mongoengine``. Another option is
 `pipenv <https://docs.pipenv.org/>`_. You can then use ``pipenv install mongoengine``
@@ -46,7 +46,7 @@ The support for Python2 was dropped with MongoEngine 0.20.0
 
 Dependencies
 ============
-All of the dependencies can easily be installed via `pip <https://pip.pypa.io/>`_.
+All of the dependencies can easily be installed via `python -m pip <https://pip.pypa.io/>`_.
 At the very least, you'll need these two packages to use MongoEngine:
 
 - pymongo>=3.4
@@ -130,7 +130,7 @@ installed in your environment and then:
 .. code-block:: shell
 
     # Install tox
-    $ pip install tox
+    $ python -m pip install tox
     # Run the test suites
     $ tox
 

docs/changelog.rst

@@ -18,6 +18,8 @@ Development
     - Queryset.slave_okay() was deprecated since pymongo3
     - dropDups was dropped with MongoDB3
     - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes``
+- Added pre-commit #2212
+- Renamed requirements-lint.txt to requirements-dev.txt #2212
 
 Changes in 0.19.1
 =================

docs/django.rst

@@ -13,7 +13,7 @@ Help Wanted!
 
 The MongoEngine team is looking for help contributing and maintaining a new
 Django extension for MongoEngine! If you have Django experience and would like
-to help contribute to the project, please get in touch on the 
-`mailing list <http://groups.google.com/group/mongoengine-users>`_ or by 
+to help contribute to the project, please get in touch on the
+`mailing list <http://groups.google.com/group/mongoengine-users>`_ or by
 simply contributing on
 `GitHub <https://github.com/MongoEngine/django-mongoengine>`_.

docs/faq.rst

@@ -10,4 +10,3 @@ If this is a requirement for your project, check the alternative:  `uMongo`_ and
 
 .. _uMongo: https://umongo.readthedocs.io/
 .. _MotorEngine: https://motorengine.readthedocs.io/
-

docs/guide/connecting.rst

@@ -86,7 +86,7 @@ using 3 different databases to store data::
         connect(alias='user-db-alias', db='user-db')
         connect(alias='book-db-alias', db='book-db')
         connect(alias='users-books-db-alias', db='users-books-db')
-        
+
         class User(Document):
             name = StringField()
 

docs/guide/installing.rst

@@ -12,7 +12,7 @@ MongoEngine is available on PyPI, so you can use :program:`pip`:
 
 .. code-block:: console
 
-    $ pip install mongoengine
+    $ python -m pip install mongoengine
 
 Alternatively, if you don't have setuptools installed, `download it from PyPi
 <http://pypi.python.org/pypi/mongoengine/>`_ and run

docs/guide/signals.rst

@@ -44,8 +44,8 @@ Available signals include:
 
 `post_save`
   Called within :meth:`~mongoengine.Document.save` after most actions
-  (validation, insert/update, and cascades, but not clearing dirty flags) have 
-  completed successfully.  Passed the additional boolean keyword argument 
+  (validation, insert/update, and cascades, but not clearing dirty flags) have
+  completed successfully.  Passed the additional boolean keyword argument
   `created` to indicate if the save was an insert or an update.
 
 `pre_delete`

docs/guide/text-indexes.rst

@@ -8,7 +8,7 @@ After MongoDB 2.4 version, supports search documents by text indexes.
 Defining a Document with text index
 ===================================
 Use the *$* prefix to set a text index, Look the declaration::
-  
+
   class News(Document):
       title = StringField()
       content = StringField()
@@ -35,10 +35,10 @@ Saving a document::
        content="Various improvements").save()
 
 Next, start a text search using :attr:`QuerySet.search_text` method::
-  
+
   document = News.objects.search_text('testing').first()
   document.title # may be: "Using mongodb text search"
-  
+
   document = News.objects.search_text('released').first()
   document.title # may be: "MongoEngine 0.9 released"
 

docs/index.rst

@@ -7,7 +7,7 @@ MongoDB. To install it, simply run
 
 .. code-block:: console
 
-    $ pip install -U mongoengine
+    $ python -m pip install -U mongoengine
 
 :doc:`tutorial`
   A quick tutorial building a tumblelog to get you up and running with
@@ -91,4 +91,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/tutorial.rst

@@ -18,7 +18,7 @@ location --- running it locally will be easier, but if that is not an option
 then it may be run on a remote server. If you haven't installed MongoEngine,
 simply use pip to install it like so::
 
-    $ pip install mongoengine
+    $ python -m pip install mongoengine
 
 Before we can start using MongoEngine, we need to tell it how to connect to our
 instance of :program:`mongod`. For this we use the :func:`~mongoengine.connect`

docs/upgrade.rst

@@ -85,10 +85,10 @@ by default from now on.
 
 The 0.8.7 package on pypi was corrupted.  If upgrading from 0.8.7 to 0.9.0 please follow: ::
 
-    pip uninstall pymongo
-    pip uninstall mongoengine
-    pip install pymongo==2.8
-    pip install mongoengine
+    python -m pip uninstall pymongo
+    python -m pip uninstall mongoengine
+    python -m pip install pymongo==2.8
+    python -m pip install mongoengine
 
 0.8.7
 *****

python-mongoengine.spec

@@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT
 # %{python_sitearch}/*
 
 %changelog
-* See: http://docs.mongoengine.org/en/latest/changelog.html
+* See: http://docs.mongoengine.org/en/latest/changelog.html

requirements-dev.txt

@@ -0,0 +1,8 @@
+black
+flake8
+flake8-import-order
+pre-commit
+pytest
+ipdb
+ipython
+tox

requirements-lint.txt

@@ -1,3 +0,0 @@
-black
-flake8
-flake8-import-order

tests/document/test_instance.py

@@ -1414,7 +1414,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert raw_doc["first_name"] == "John"
 
     def test_inserts_if_you_set_the_pk(self):
-        p1 = self.Person(name="p1", id=bson.ObjectId()).save()
+        _ = self.Person(name="p1", id=bson.ObjectId()).save()
         p2 = self.Person(name="p2")
         p2.id = bson.ObjectId()
         p2.save()
@@ -2195,7 +2195,7 @@ class TestDocumentInstance(MongoDBTestCase):
 
         user = User(name="Mike").save()
         reviewer = User(name="John").save()
-        book = Book(author=user, reviewer=reviewer).save()
+        _ = Book(author=user, reviewer=reviewer).save()
 
         reviewer.delete()
         assert Book.objects.count() == 1
@@ -2221,7 +2221,7 @@ class TestDocumentInstance(MongoDBTestCase):
 
         user_1 = User(id=1).save()
         user_2 = User(id=2).save()
-        book_1 = Book(id=1, author=user_2).save()
+        _ = Book(id=1, author=user_2).save()
         book_2 = Book(id=2, author=user_1).save()
 
         user_2.delete()
@@ -2230,7 +2230,7 @@ class TestDocumentInstance(MongoDBTestCase):
         assert Book.objects.get() == book_2
 
         user_3 = User(id=3).save()
-        book_3 = Book(id=3, author=user_3).save()
+        _ = Book(id=3, author=user_3).save()
 
         user_3.delete()
         # Deleting user_3 should also delete book_3
@@ -3204,7 +3204,7 @@ class TestDocumentInstance(MongoDBTestCase):
     def test_positional_creation(self):
         """Document cannot be instantiated using positional arguments."""
         with pytest.raises(TypeError) as exc_info:
-            person = self.Person("Test User", 42)
+            self.Person("Test User", 42)
 
         expected_msg = (
             "Instantiating a document with positional arguments is not "
@@ -3606,13 +3606,13 @@ class TestDocumentInstance(MongoDBTestCase):
             v = StringField()
 
         class A(Document):
-            l = ListField(EmbeddedDocumentField(B))
+            array = ListField(EmbeddedDocumentField(B))
 
         A.objects.delete()
-        A(l=[B(v="1"), B(v="2"), B(v="3")]).save()
+        A(array=[B(v="1"), B(v="2"), B(v="3")]).save()
         a = A.objects.get()
-        assert a.l._instance == a
-        for idx, b in enumerate(a.l):
+        assert a.array._instance == a
+        for idx, b in enumerate(a.array):
             assert b._instance == a
         assert idx == 2
 

tests/fields/test_dict_field.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-import pytest
 from bson import InvalidDocument
+import pytest
 
 from mongoengine import *
 from mongoengine.base import BaseDict

tests/fields/test_lazy_reference_field.py

@@ -152,7 +152,7 @@ class TestLazyReferenceField(MongoDBTestCase):
             LazyReference(BadDoc, animal.pk),
         ):
             with pytest.raises(ValidationError):
-                p = Ocurrence(person="test", animal=bad).save()
+                Ocurrence(person="test", animal=bad).save()
 
     def test_lazy_reference_query_conversion(self):
         """Ensure that LazyReferenceFields can be queried using objects and values
@@ -386,7 +386,7 @@ class TestGenericLazyReferenceField(MongoDBTestCase):
         mineral = Mineral(name="Granite").save()
 
         occ_animal = Ocurrence(living_thing=animal, thing=animal).save()
-        occ_vegetal = Ocurrence(living_thing=vegetal, thing=vegetal).save()
+        _ = Ocurrence(living_thing=vegetal, thing=vegetal).save()
         with pytest.raises(ValidationError):
             Ocurrence(living_thing=mineral).save()
 
@@ -458,7 +458,7 @@ class TestGenericLazyReferenceField(MongoDBTestCase):
         baddoc = BadDoc().save()
         for bad in (42, "foo", baddoc, LazyReference(BadDoc, animal.pk)):
             with pytest.raises(ValidationError):
-                p = Ocurrence(person="test", animal=bad).save()
+                Ocurrence(person="test", animal=bad).save()
 
     def test_generic_lazy_reference_query_conversion(self):
         class Member(Document):

tests/queryset/test_queryset.py

@@ -116,7 +116,7 @@ class TestQueryset(unittest.TestCase):
     def test_limit(self):
         """Ensure that QuerySet.limit works as expected."""
         user_a = self.Person.objects.create(name="User A", age=20)
-        user_b = self.Person.objects.create(name="User B", age=30)
+        _ = self.Person.objects.create(name="User B", age=30)
 
         # Test limit on a new queryset
         people = list(self.Person.objects.limit(1))
@@ -148,6 +148,11 @@ class TestQueryset(unittest.TestCase):
         user_b = self.Person.objects.create(name="User B", age=30)
 
         # Test skip on a new queryset
+        people = list(self.Person.objects.skip(0))
+        assert len(people) == 2
+        assert people[0] == user_a
+        assert people[1] == user_b
+
         people = list(self.Person.objects.skip(1))
         assert len(people) == 1
         assert people[0] == user_b
@@ -2586,13 +2591,8 @@ class TestQueryset(unittest.TestCase):
             age = IntField()
 
         with db_ops_tracker() as q:
-            adult1 = (
-                User.objects.filter(age__gte=18).comment("looking for an adult").first()
-            )
-
-            adult2 = (
-                User.objects.comment("looking for an adult").filter(age__gte=18).first()
-            )
+            User.objects.filter(age__gte=18).comment("looking for an adult").first()
+            User.objects.comment("looking for an adult").filter(age__gte=18).first()
 
             ops = q.get_ops()
             assert len(ops) == 2
@@ -4518,7 +4518,7 @@ class TestQueryset(unittest.TestCase):
 
         foos_without_y = list(Foo.objects.order_by("y").fields(y=0))
 
-        assert all(o.y is None for o in foos_with_x)
+        assert all(o.y is None for o in foos_without_y)
 
         foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1))
 
@@ -5595,7 +5595,7 @@ class TestQueryset(unittest.TestCase):
         self.Person.objects.create(name="Baz")
         assert self.Person.objects.count(with_limit_and_skip=True) == 3
 
-        newPerson = self.Person.objects.create(name="Foo_1")
+        self.Person.objects.create(name="Foo_1")
         assert self.Person.objects.count(with_limit_and_skip=True) == 4
 
     def test_no_cursor_timeout(self):

tests/test_connection.py

@@ -348,7 +348,7 @@ class ConnectionTest(unittest.TestCase):
 
     def test_disconnect_cleans_cached_collection_attribute_in_document(self):
         """Ensure that the disconnect() method works properly"""
-        conn1 = connect("mongoenginetest")
+        connect("mongoenginetest")
 
         class History(Document):
             pass
@@ -518,7 +518,7 @@ class ConnectionTest(unittest.TestCase):
         """Ensure connect() uses the username & password params if the URI
         doesn't explicitly specify them.
         """
-        c = connect(
+        connect(
             host="mongodb://localhost/mongoenginetest", username="user", password="pass"
         )
 
@@ -632,7 +632,7 @@ class ConnectionTest(unittest.TestCase):
         """Ensure connect() works when specifying a replicaSet via the
         MongoDB URI.
         """
-        c = connect(host="mongodb://localhost/test?replicaSet=local-rs")
+        connect(host="mongodb://localhost/test?replicaSet=local-rs")
         db = get_db()
         assert isinstance(db, pymongo.database.Database)
         assert db.name == "test"

tests/test_context_managers.py

@@ -216,7 +216,7 @@ class TestContextManagers:
 
     def test_query_counter_does_not_swallow_exception(self):
         with pytest.raises(TypeError):
-            with query_counter() as q:
+            with query_counter():
                 raise TypeError()
 
     def test_query_counter_temporarily_modifies_profiling_level(self):
@@ -226,12 +226,12 @@ class TestContextManagers:
         initial_profiling_level = db.profiling_level()
 
         try:
-            NEW_LEVEL = 1
-            db.set_profiling_level(NEW_LEVEL)
-            assert db.profiling_level() == NEW_LEVEL
-            with query_counter() as q:
+            new_level = 1
+            db.set_profiling_level(new_level)
+            assert db.profiling_level() == new_level
+            with query_counter():
                 assert db.profiling_level() == 2
-            assert db.profiling_level() == NEW_LEVEL
+            assert db.profiling_level() == new_level
         except Exception:
             db.set_profiling_level(
                 initial_profiling_level

tests/test_signals.py

@@ -267,7 +267,7 @@ class TestSignal(unittest.TestCase):
             a = self.Author(name="Bill Shakespeare")
             a.save()
             self.get_signal_output(lambda: None)  # eliminate signal output
-            a1 = self.Author.objects(name="Bill Shakespeare")[0]
+            _ = self.Author.objects(name="Bill Shakespeare")[0]
 
         assert self.get_signal_output(create_author) == [
             "pre_init signal, Author",