Fix connect/disconnect functions

- expose disconnect
- disconnect cleans _connection_settings
- disconnect cleans cached collection in Document._collection
- re-connecting with the same alias raise an error (must call disconnect in between)

docs/changelog.rst

@@ -4,6 +4,11 @@ Changelog
 
 Development
 ===========
+- expose `mongoengine.connection.disconnect` and `mongoengine.connection.disconnect_all`
+- POTENTIAL BREAKING CHANGE: Fixes in connect/disconnect methods
+    - calling `connect` 2 times with the same alias and different parameter will raise an error (should call disconnect first)
+    - disconnect now clears `mongoengine.connection._connection_settings`
+    - disconnect now clears the cached attribute `Document._collection`
 - POTENTIAL BREAKING CHANGE: Aggregate gives wrong results when used with a queryset having limit and skip #2029
 - mongoengine now requires pymongo>=3.5 #2017
 - Generate Unique Indices for SortedListField and EmbeddedDocumentListFields #2020

mongoengine/base/common.py

@@ -13,7 +13,7 @@ _document_registry = {}
 
 
 def get_document(name):
-    """Get a document class by name."""
+    """Get a registered Document class by name."""
     doc = _document_registry.get(name, None)
     if not doc:
         # Possible old style name
@@ -30,3 +30,12 @@ def get_document(name):
             been imported?
         """.strip() % name)
     return doc
+
+
+def _get_documents_by_db(connection_alias, default_connection_alias):
+    """Get all registered Documents class attached to a given database"""
+    def get_doc_alias(doc_cls):
+        return doc_cls._meta.get('db_alias', default_connection_alias)
+
+    return [doc_cls for doc_cls in _document_registry.values()
+            if get_doc_alias(doc_cls) == connection_alias]

mongoengine/connection.py

@@ -3,11 +3,13 @@ import six
 
 from mongoengine.pymongo_support import IS_PYMONGO_3
 
-__all__ = ['MongoEngineConnectionError', 'connect', 'register_connection',
+__all__ = ['MongoEngineConnectionError', 'connect', 'disconnect', 'disconnect_all',
-           'DEFAULT_CONNECTION_NAME', 'get_db']
+           'register_connection', 'DEFAULT_CONNECTION_NAME', 'DEFAULT_DATABASE_NAME',
+           'get_db', 'get_connection']
 
 
 DEFAULT_CONNECTION_NAME = 'default'
+DEFAULT_DATABASE_NAME = 'test'
 
 if IS_PYMONGO_3:
     READ_PREFERENCE = ReadPreference.PRIMARY
@@ -28,18 +30,17 @@ _connections = {}
 _dbs = {}
 
 
-def register_connection(alias, db=None, name=None, host=None, port=None,
+def _get_connection_settings(
-                        read_preference=READ_PREFERENCE,
+        db=None, name=None, host=None, port=None,
-                        username=None, password=None,
+        read_preference=READ_PREFERENCE,
-                        authentication_source=None,
+        username=None, password=None,
-                        authentication_mechanism=None,
+        authentication_source=None,
-                        **kwargs):
+        authentication_mechanism=None,
-    """Add a connection.
+        **kwargs):
+    """Get the connection settings as a dict
 
-    :param alias: the name that will be used to refer to this connection
-        throughout MongoEngine
-    :param name: the name of the specific database to use
     :param db: the name of the database to use, for compatibility with connect
+    :param name: the name of the specific database to use
     :param host: the host name of the :program:`mongod` instance to connect to
     :param port: the port that the :program:`mongod` instance is running on
     :param read_preference: The read preference for the collection
@@ -59,7 +60,7 @@ def register_connection(alias, db=None, name=None, host=None, port=None,
     .. versionchanged:: 0.10.6 - added mongomock support
     """
     conn_settings = {
-        'name': name or db or 'test',
+        'name': name or db or DEFAULT_DATABASE_NAME,
         'host': host or 'localhost',
         'port': port or 27017,
         'read_preference': read_preference,
@@ -125,17 +126,74 @@ def register_connection(alias, db=None, name=None, host=None, port=None,
     kwargs.pop('is_slave', None)
 
     conn_settings.update(kwargs)
+    return conn_settings
+
+
+def register_connection(alias, db=None, name=None, host=None, port=None,
+                        read_preference=READ_PREFERENCE,
+                        username=None, password=None,
+                        authentication_source=None,
+                        authentication_mechanism=None,
+                        **kwargs):
+    """Register the connection settings.
+
+    :param alias: the name that will be used to refer to this connection
+        throughout MongoEngine
+    :param name: the name of the specific database to use
+    :param db: the name of the database to use, for compatibility with connect
+    :param host: the host name of the :program:`mongod` instance to connect to
+    :param port: the port that the :program:`mongod` instance is running on
+    :param read_preference: The read preference for the collection
+       ** Added pymongo 2.1
+    :param username: username to authenticate with
+    :param password: password to authenticate with
+    :param authentication_source: database to authenticate against
+    :param authentication_mechanism: database authentication mechanisms.
+        By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
+        MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
+    :param is_mock: explicitly use mongomock for this connection
+        (can also be done by using `mongomock://` as db host prefix)
+    :param kwargs: ad-hoc parameters to be passed into the pymongo driver,
+        for example maxpoolsize, tz_aware, etc. See the documentation
+        for pymongo's `MongoClient` for a full list.
+
+    .. versionchanged:: 0.10.6 - added mongomock support
+    """
+    conn_settings = _get_connection_settings(
+        db=db, name=name, host=host, port=port,
+        read_preference=read_preference,
+        username=username, password=password,
+        authentication_source=authentication_source,
+        authentication_mechanism=authentication_mechanism,
+        **kwargs)
     _connection_settings[alias] = conn_settings
 
 
 def disconnect(alias=DEFAULT_CONNECTION_NAME):
     """Close the connection with a given alias."""
+    from mongoengine.base.common import _get_documents_by_db
+
     if alias in _connections:
         get_connection(alias=alias).close()
         del _connections[alias]
+
     if alias in _dbs:
+        # Detach all cached collections in Documents
+        for doc_cls in _get_documents_by_db(alias, DEFAULT_CONNECTION_NAME):
+            if hasattr(doc_cls, '_disconnect'):
+                doc_cls._disconnect()
+
         del _dbs[alias]
 
+    if alias in _connection_settings:
+        del _connection_settings[alias]
+
+
+def disconnect_all():
+    """Close all registered database."""
+    for alias in list(_connections.keys()):
+        disconnect(alias)
+
 
 def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
     """Return a connection with a given alias."""
@@ -265,7 +323,14 @@ def connect(db=None, alias=DEFAULT_CONNECTION_NAME, **kwargs):
 
     .. versionchanged:: 0.6 - added multiple database support.
     """
-    if alias not in _connections:
+    if alias in _connections:
+        prev_conn_setting = _connection_settings[alias]
+        new_conn_settings = _get_connection_settings(db, **kwargs)
+
+        if new_conn_settings != prev_conn_setting:
+            raise MongoEngineConnectionError(
+                'A different connection with alias `%s` was already registered. Use disconnect() first' % alias)
+    else:
         register_connection(alias, db, **kwargs)
 
     return get_connection(alias)

mongoengine/document.py

@@ -188,10 +188,16 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)):
         return get_db(cls._meta.get('db_alias', DEFAULT_CONNECTION_NAME))
 
     @classmethod
-    def _get_collection(cls):
+    def _disconnect(cls):
-        """Return a PyMongo collection for the document."""
+        """Detach the Document class from the (cached) database collection"""
-        if not hasattr(cls, '_collection') or cls._collection is None:
+        cls._collection = None
 
+    @classmethod
+    def _get_collection(cls):
+        """Return the corresponding PyMongo collection of this document.
+        Upon the first call, it will ensure that indexes gets created. The returned collection then gets cached
+        """
+        if not hasattr(cls, '_collection') or cls._collection is None:
             # Get the collection, either capped or regular.
             if cls._meta.get('max_size') or cls._meta.get('max_documents'):
                 cls._collection = cls._get_capped_collection()

tests/test_connection.py

@@ -12,12 +12,12 @@ from bson.tz_util import utc
 
 from mongoengine import (
     connect, register_connection,
-    Document, DateTimeField
+    Document, DateTimeField,
-)
+    disconnect_all, StringField)
 from mongoengine.pymongo_support import IS_PYMONGO_3
 import mongoengine.connection
 from mongoengine.connection import (MongoEngineConnectionError, get_db,
-                                    get_connection)
+                                    get_connection, disconnect, DEFAULT_DATABASE_NAME)
 
 
 def get_tz_awareness(connection):
@@ -29,6 +29,14 @@ def get_tz_awareness(connection):
 
 class ConnectionTest(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        disconnect_all()
+
+    @classmethod
+    def tearDownClass(cls):
+        disconnect_all()
+
     def tearDown(self):
         mongoengine.connection._connection_settings = {}
         mongoengine.connection._connections = {}
@@ -49,6 +57,117 @@ class ConnectionTest(unittest.TestCase):
         conn = get_connection('testdb')
         self.assertIsInstance(conn, pymongo.mongo_client.MongoClient)
 
+    def test_connect_disconnect_works_properly(self):
+        class History1(Document):
+            name = StringField()
+            meta = {'db_alias': 'db1'}
+
+        class History2(Document):
+            name = StringField()
+            meta = {'db_alias': 'db2'}
+
+        connect('db1', alias='db1')
+        connect('db2', alias='db2')
+
+        History1.drop_collection()
+        History2.drop_collection()
+
+        h = History1(name='default').save()
+        h1 = History2(name='db1').save()
+
+        self.assertEqual(list(History1.objects().as_pymongo()),
+                         [{'_id': h.id, 'name': 'default'}])
+        self.assertEqual(list(History2.objects().as_pymongo()),
+                         [{'_id': h1.id, 'name': 'db1'}])
+
+        disconnect('db1')
+        disconnect('db2')
+
+        with self.assertRaises(MongoEngineConnectionError):
+            list(History1.objects().as_pymongo())
+
+        with self.assertRaises(MongoEngineConnectionError):
+            list(History2.objects().as_pymongo())
+
+        connect('db1', alias='db1')
+        connect('db2', alias='db2')
+
+        self.assertEqual(list(History1.objects().as_pymongo()),
+                         [{'_id': h.id, 'name': 'default'}])
+        self.assertEqual(list(History2.objects().as_pymongo()),
+                         [{'_id': h1.id, 'name': 'db1'}])
+
+    def test_connect_different_documents_to_different_database(self):
+        class History(Document):
+            name = StringField()
+
+        class History1(Document):
+            name = StringField()
+            meta = {'db_alias': 'db1'}
+
+        class History2(Document):
+            name = StringField()
+            meta = {'db_alias': 'db2'}
+
+        connect()
+        connect('db1', alias='db1')
+        connect('db2', alias='db2')
+
+        History.drop_collection()
+        History1.drop_collection()
+        History2.drop_collection()
+
+        h = History(name='default').save()
+        h1 = History1(name='db1').save()
+        h2 = History2(name='db2').save()
+
+        self.assertEqual(History._collection.database.name, DEFAULT_DATABASE_NAME)
+        self.assertEqual(History1._collection.database.name, 'db1')
+        self.assertEqual(History2._collection.database.name, 'db2')
+
+        self.assertEqual(list(History.objects().as_pymongo()),
+                         [{'_id': h.id, 'name': 'default'}])
+        self.assertEqual(list(History1.objects().as_pymongo()),
+                         [{'_id': h1.id, 'name': 'db1'}])
+        self.assertEqual(list(History2.objects().as_pymongo()),
+                         [{'_id': h2.id, 'name': 'db2'}])
+
+    def test_connect_fails_if_connect_2_times_with_default_alias(self):
+        connect('mongoenginetest')
+
+        with self.assertRaises(MongoEngineConnectionError) as ctx_err:
+            connect('mongoenginetest2')
+        self.assertEqual("A different connection with alias `default` was already registered. Use disconnect() first", str(ctx_err.exception))
+
+    def test_connect_fails_if_connect_2_times_with_custom_alias(self):
+        connect('mongoenginetest', alias='alias1')
+
+        with self.assertRaises(MongoEngineConnectionError) as ctx_err:
+            connect('mongoenginetest2', alias='alias1')
+
+        self.assertEqual("A different connection with alias `alias1` was already registered. Use disconnect() first", str(ctx_err.exception))
+
+    def test_connect_fails_if_similar_connection_settings_arent_defined_the_same_way(self):
+        """Intended to keep the detecton function simple but robust"""
+        db_name = 'mongoenginetest'
+        db_alias = 'alias1'
+        connect(db=db_name, alias=db_alias, host='localhost', port=27017)
+
+        with self.assertRaises(MongoEngineConnectionError):
+            connect(host='mongodb://localhost:27017/%s' % db_name, alias=db_alias)
+
+    def test_connect_passes_silently_connect_multiple_times_with_same_config(self):
+        # test default connection to `test`
+        connect()
+        connect()
+        self.assertEqual(len(mongoengine.connection._connections), 1)
+        connect('test01', alias='test01')
+        connect('test01', alias='test01')
+        self.assertEqual(len(mongoengine.connection._connections), 2)
+        connect(host='mongodb://localhost:27017/mongoenginetest02', alias='test02')
+        connect(host='mongodb://localhost:27017/mongoenginetest02', alias='test02')
+        self.assertEqual(len(mongoengine.connection._connections), 3)
+
     def test_connect_in_mocking(self):
         """Ensure that the connect() method works properly in mocking.
         """
@@ -120,13 +239,93 @@ class ConnectionTest(unittest.TestCase):
         self.assertIsInstance(conn, mongomock.MongoClient)
 
     def test_disconnect(self):
-        """Ensure that the disconnect() method works properly
+        """Ensure that the disconnect() method works properly"""
-        """
+        connections = mongoengine.connection._connections
+        dbs = mongoengine.connection._dbs
+        connection_settings = mongoengine.connection._connection_settings
+
         conn1 = connect('mongoenginetest')
-        mongoengine.connection.disconnect()
+
+        class History(Document):
+            pass
+
+        self.assertIsNone(History._collection)
+
+        History.drop_collection()
+        History.objects.first()     # will trigger the caching of _collection attribute
+
+        self.assertIsNotNone(History._collection)
+
+        self.assertEqual(len(connections), 1)
+        self.assertEqual(len(dbs), 1)
+        self.assertEqual(len(connection_settings), 1)
+
+        disconnect()
+
+        self.assertIsNone(History._collection)
+
+        self.assertEqual(len(connections), 0)
+        self.assertEqual(len(dbs), 0)
+        self.assertEqual(len(connection_settings), 0)
+
+        with self.assertRaises(MongoEngineConnectionError) as ctx_err:
+            History.objects.first()
+        self.assertEqual("You have not defined a default connection", str(ctx_err.exception))
+
         conn2 = connect('mongoenginetest')
+        History.objects.first()     # Make sure its back on track
         self.assertTrue(conn1 is not conn2)
 
+    def test_disconnect_silently_pass_if_alias_does_not_exist(self):
+        connections = mongoengine.connection._connections
+        self.assertEqual(len(connections), 0)
+        disconnect(alias='not_exist')
+
+    def test_disconnect_all(self):
+        connections = mongoengine.connection._connections
+        dbs = mongoengine.connection._dbs
+        connection_settings = mongoengine.connection._connection_settings
+
+        connect('mongoenginetest')
+        connect('mongoenginetest2', alias='db1')
+
+        class History(Document):
+            pass
+
+        class History1(Document):
+            name = StringField()
+            meta = {'db_alias': 'db1'}
+
+        History.drop_collection()   # will trigger the caching of _collection attribute
+        History.objects.first()
+        History1.drop_collection()
+        History1.objects.first()
+
+        self.assertIsNotNone(History._collection)
+        self.assertIsNotNone(History1._collection)
+
+        self.assertEqual(len(connections), 2)
+        self.assertEqual(len(dbs), 2)
+        self.assertEqual(len(connection_settings), 2)
+
+        disconnect_all()
+
+        self.assertIsNone(History._collection)
+        self.assertIsNone(History1._collection)
+
+        self.assertEqual(len(connections), 0)
+        self.assertEqual(len(dbs), 0)
+        self.assertEqual(len(connection_settings), 0)
+
+        with self.assertRaises(MongoEngineConnectionError):
+            History.objects.first()
+
+        with self.assertRaises(MongoEngineConnectionError):
+            History1.objects.first()
+
+    def test_disconnect_all_silently_pass_if_no_connection_exist(self):
+        disconnect_all()
+
     def test_sharing_connections(self):
         """Ensure that connections are shared when the connection settings are exactly the same
         """
@@ -342,7 +541,7 @@ class ConnectionTest(unittest.TestCase):
             with self.assertRaises(MongoEngineConnectionError):
                 c = connect(replicaset='local-rs')
 
-    def test_datetime(self):
+    def test_connect_tz_aware(self):
         connect('mongoenginetest', tz_aware=True)
         d = datetime.datetime(2010, 5, 5, tzinfo=utc)
 

tests/utils.py

@@ -4,7 +4,7 @@ import unittest
 from nose.plugins.skip import SkipTest
 
 from mongoengine import connect
-from mongoengine.connection import get_db
+from mongoengine.connection import get_db, disconnect_all
 from mongoengine.mongodb_support import get_mongodb_version, MONGODB_26, MONGODB_3, MONGODB_32, MONGODB_34
 from mongoengine.pymongo_support import IS_PYMONGO_3
 
@@ -19,6 +19,7 @@ class MongoDBTestCase(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        disconnect_all()
         cls._connection = connect(db=MONGO_TEST_DB)
         cls._connection.drop_database(MONGO_TEST_DB)
         cls.db = get_db()
@@ -26,6 +27,7 @@ class MongoDBTestCase(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         cls._connection.drop_database(MONGO_TEST_DB)
+        disconnect_all()
 
 
 def get_as_pymongo(doc):