add support to m2m

change cli options
add init-db cmd

.gitignore

@@ -141,4 +141,5 @@ dmypy.json
 cython_debug/
 
 .idea
-migrations
+migrations
+aerich.ini

CHANGELOG.rst

@@ -2,9 +2,14 @@
 ChangeLog
 =========
 
-
 0.1
 ===
+0.1.2
+-----
+- Now aerich support m2m.
+- Add cli cmd init-db.
+- Change cli options.
+
 0.1.1
 -----
 - Now aerich is basic worked.

README.rst

@@ -35,41 +35,66 @@ Quick Start
     Usage: aerich [OPTIONS] COMMAND [ARGS]...
 
     Options:
-      --config TEXT        Tortoise-ORM config module, will auto read config dict variable
+      -c, --config TEXT  Config file.  [default: aerich.ini]
-                           from it.  [default: settings]
+      --app TEXT         Tortoise-ORM app name.  [default: models]
-      --tortoise-orm TEXT  Tortoise-ORM config dict variable.  [default:
+      -n, --name TEXT    Name of section in .ini file to use for aerich config.
-                           TORTOISE_ORM]
+                         [default: aerich]
-      --location TEXT      Migrate store location.  [default: ./migrations]
+      -h, --help         Show this message and exit.
-      --app TEXT           Tortoise-ORM app name.  [default: models]
-      -h, --help           Show this message and exit.
 
     Commands:
       downgrade  Downgrade to previous version.
       heads      Show current available heads in migrate location.
       history    List all migrate items.
-      init       Init migrate location and generate schema, you must exec first.
+      init       Init config file and generate migrate location.
+      init-db    Generate schema.
       migrate    Generate migrate changes file.
       upgrade    Upgrade to latest version.
 
 Usage
 =====
 
-Init schema and migrate location
+Initialization
---------------------------------
+--------------
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql init
+    $ aerich init -h
 
-    Success create migrate location ./migrations/models
+    Usage: aerich init [OPTIONS]
-    Success init for app "models"
+
+      Init config file and generate migrate location, you must exec first.
+
+    Options:
+      -t, --tortoise-orm TEXT  Tortoise-ORM config module dict variable.
+                               [required]
+      --location TEXT          Migrate store location.  [default: ./migrations]
+      -h, --help               Show this message and exit.
+
+Init config file and location:
+
+.. code-block:: shell
+
+    $ aerich init -t tests.backends.mysql.TORTOISE_ORM
+
+    Success create migrate location ./migrations
+    Success generate config file aerich.ini
+
+Init db
+-------
+
+.. code-block:: shell
+
+    $ aerich init-db
+
+    Success create app migrate location ./migrations/models
+    Success generate schema for app "models"
 
 Update models and make migrate
 ------------------------------
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql migrate --name drop_column
+    $ aerich migrate --name drop_column
 
     Success migrate 1_202029051520102929_drop_column.json
 
@@ -80,7 +105,7 @@ Upgrade to latest version
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql upgrade
+    $ aerich upgrade
 
     Success upgrade 1_202029051520102929_drop_column.json
 
@@ -91,7 +116,7 @@ Downgrade to previous version
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql downgrade
+    $ aerich downgrade
 
     Success downgrade 1_202029051520102929_drop_column.json
 
@@ -102,7 +127,7 @@ Show history
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql history
+    $ aerich history
 
     1_202029051520102929_drop_column.json
 
@@ -111,7 +136,7 @@ Show heads to be migrated
 
 .. code-block:: shell
 
-    $ aerich --config tests.backends.mysql heads
+    $ aerich heads
 
     1_202029051520102929_drop_column.json
 

aerich/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.1.2"

aerich/cli.py

@@ -1,15 +1,15 @@
-import importlib
 import json
 import os
 import sys
+from configparser import ConfigParser
 from enum import Enum
 
 import asyncclick as click
-from asyncclick import BadOptionUsage, Context, UsageError
+from asyncclick import Context, UsageError
 from tortoise import Tortoise, generate_schema_for_client
 
 from aerich.migrate import Migrate
-from aerich.utils import get_app_connection
+from aerich.utils import get_app_connection, get_tortoise_config
 
 
 class Color(str, Enum):
@@ -18,51 +18,43 @@ class Color(str, Enum):
     yellow = "yellow"
 
 
+parser = ConfigParser()
+
+
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
 @click.option(
-    "--config",
+    "-c", "--config", default="aerich.ini", show_default=True, help="Config file.",
-    default="settings",
-    show_default=True,
-    help="Tortoise-ORM config module, will auto read dict config variable from it.",
-)
-@click.option(
-    "--tortoise-orm",
-    default="TORTOISE_ORM",
-    show_default=True,
-    help="Tortoise-ORM config dict variable.",
-)
-@click.option(
-    "--location", default="./migrations", show_default=True, help="Migrate store location."
 )
 @click.option("--app", default="models", show_default=True, help="Tortoise-ORM app name.")
+@click.option(
+    "-n",
+    "--name",
+    default="aerich",
+    show_default=True,
+    help="Name of section in .ini file to use for aerich config.",
+)
 @click.pass_context
-async def cli(ctx: Context, config, tortoise_orm, location, app):
+async def cli(ctx: Context, config, app, name):
     ctx.ensure_object(dict)
-    try:
-        config_module = importlib.import_module(config, ".")
-    except ModuleNotFoundError:
-        raise BadOptionUsage(ctx=ctx, message=f'No module named "{config}"', option_name="--config")
-    config = getattr(config_module, tortoise_orm, None)
-    if not config:
-        raise BadOptionUsage(
-            option_name="--config",
-            message=f'Can\'t get "{tortoise_orm}" from module "{config_module}"',
-            ctx=ctx,
-        )
-
-    if app not in config.get("apps").keys():
-        raise BadOptionUsage(option_name="--config", message=f'No app found in "{config}"', ctx=ctx)
-
     ctx.obj["config"] = config
-    ctx.obj["location"] = location
+    ctx.obj["name"] = name
-    ctx.obj["app"] = app
+    invoked_subcommand = ctx.invoked_subcommand
-
+    if invoked_subcommand != "init":
-    if ctx.invoked_subcommand == "init":
+        if not os.path.exists(config):
-        await Tortoise.init(config=config)
-    else:
-        if not os.path.isdir(location):
             raise UsageError("You must exec init first", ctx=ctx)
-        await Migrate.init_with_old_models(config, app, location)
+        parser.read(config)
+
+        location = parser[name]["location"]
+        tortoise_orm = parser[name]["tortoise_orm"]
+
+        tortoise_config = get_tortoise_config(ctx, tortoise_orm)
+
+        ctx.obj["config"] = tortoise_config
+        ctx.obj["location"] = location
+        ctx.obj["app"] = app
+
+        if invoked_subcommand != "init-db":
+            await Migrate.init_with_old_models(tortoise_config, app, location)
 
 
 @cli.command(help="Generate migrate changes file.")
@@ -100,7 +92,7 @@ async def upgrade(ctx: Context):
 
             with open(file_path, "w") as f:
                 content["migrate"] = True
-                json.dump(content, f, indent=4, ensure_ascii=False)
+                json.dump(content, f, indent=2, ensure_ascii=False)
                 click.secho(f"Success upgrade {file}", fg=Color.green)
 
 
@@ -127,7 +119,7 @@ async def downgrade(ctx: Context):
                     continue
             with open(file_path, "w") as f:
                 content["migrate"] = False
-                json.dump(content, f, indent=4, ensure_ascii=False)
+                json.dump(content, f, indent=2, ensure_ascii=False)
                 return click.secho(f"Success downgrade {file}", fg=Color.green)
 
 
@@ -135,17 +127,45 @@ async def downgrade(ctx: Context):
 @click.pass_context
 def heads(ctx: Context):
     for version in Migrate.get_all_version_files(is_all=False):
-        click.secho(version, fg=Color.yellow)
+        click.secho(version, fg=Color.green)
 
 
 @cli.command(help="List all migrate items.")
 @click.pass_context
 def history(ctx):
     for version in Migrate.get_all_version_files():
-        click.secho(version, fg=Color.yellow)
+        click.secho(version, fg=Color.green)
 
 
-@cli.command(help="Init migrate location and generate schema, you must exec first.")
+@cli.command(help="Init config file and generate migrate location.")
+@click.option(
+    "-t", "--tortoise-orm", required=True, help="Tortoise-ORM config module dict variable.",
+)
+@click.option(
+    "--location", default="./migrations", show_default=True, help="Migrate store location."
+)
+@click.pass_context
+async def init(
+    ctx: Context, tortoise_orm, location,
+):
+    config = ctx.obj["config"]
+    name = ctx.obj["name"]
+
+    parser.add_section(name)
+    parser.set(name, "tortoise_orm", tortoise_orm)
+    parser.set(name, "location", location)
+
+    with open(config, "w") as f:
+        parser.write(f)
+
+    if not os.path.isdir(location):
+        os.mkdir(location)
+
+    click.secho(f"Success create migrate location {location}", fg=Color.green)
+    click.secho(f"Success generate config file {config}", fg=Color.green)
+
+
+@cli.command(help="Generate schema.")
 @click.option(
     "--safe",
     is_flag=True,
@@ -154,27 +174,25 @@ def history(ctx):
     show_default=True,
 )
 @click.pass_context
-async def init(ctx: Context, safe):
+async def init_db(ctx: Context, safe):
+    config = ctx.obj["config"]
     location = ctx.obj["location"]
     app = ctx.obj["app"]
-    config = ctx.obj["config"]
-
-    if not os.path.isdir(location):
-        os.mkdir(location)
 
     dirname = os.path.join(location, app)
     if not os.path.isdir(dirname):
         os.mkdir(dirname)
-        click.secho(f"Success create migrate location {dirname}", fg=Color.green)
+        click.secho(f"Success create app migrate location {dirname}", fg=Color.green)
     else:
         return click.secho(f'Already inited app "{app}"', fg=Color.yellow)
 
     Migrate.write_old_models(config, app, location)
 
+    await Tortoise.init(config=config)
     connection = get_app_connection(config, app)
     await generate_schema_for_client(connection, safe)
 
-    return click.secho(f'Success init for app "{app}"', fg=Color.green)
+    return click.secho(f'Success generate schema for app "{app}"', fg=Color.green)
 
 
 def main():

aerich/ddl/__init__.py

@@ -1,6 +1,6 @@
 from typing import List, Type
 
-from tortoise import BaseDBAsyncClient, ForeignKeyFieldInstance, Model
+from tortoise import BaseDBAsyncClient, ForeignKeyFieldInstance, ManyToManyFieldInstance, Model
 from tortoise.backends.base.schema_generator import BaseSchemaGenerator
 from tortoise.fields import Field
 
@@ -8,7 +8,7 @@ from tortoise.fields import Field
 class BaseDDL:
     schema_generator_cls: Type[BaseSchemaGenerator] = BaseSchemaGenerator
     DIALECT = "sql"
-    _DROP_TABLE_TEMPLATE = "DROP TABLE {table_name} IF EXISTS"
+    _DROP_TABLE_TEMPLATE = "DROP TABLE IF EXISTS {table_name}"
     _ADD_COLUMN_TEMPLATE = "ALTER TABLE {table_name} ADD {column}"
     _DROP_COLUMN_TEMPLATE = "ALTER TABLE {table_name} DROP COLUMN {column_name}"
     _ADD_INDEX_TEMPLATE = (
@@ -17,6 +17,7 @@ class BaseDDL:
     _DROP_INDEX_TEMPLATE = "ALTER TABLE {table_name} DROP INDEX {index_name}"
     _ADD_FK_TEMPLATE = "ALTER TABLE {table_name} ADD CONSTRAINT `{fk_name}` FOREIGN KEY (`{db_column}`) REFERENCES `{table}` (`{field}`) ON DELETE {on_delete}"
     _DROP_FK_TEMPLATE = "ALTER TABLE {table_name} DROP FOREIGN KEY {fk_name}"
+    _M2M_TABLE_TEMPLATE = "CREATE TABLE {table_name} ({backward_key} {backward_type} NOT NULL REFERENCES {backward_table} ({backward_field}) ON DELETE CASCADE,{forward_key} {forward_type} NOT NULL REFERENCES {forward_table} ({forward_field}) ON DELETE CASCADE){extra}{comment};"
 
     def __init__(self, client: "BaseDBAsyncClient"):
         self.client = client
@@ -25,6 +26,12 @@ class BaseDDL:
     def create_table(self, model: "Type[Model]"):
         raise NotImplementedError
 
+    def create_m2m_table(self, model: "Type[Model]", field: ManyToManyFieldInstance):
+        raise NotImplementedError
+
+    def drop_m2m(self, field: ManyToManyFieldInstance):
+        raise NotImplementedError
+
     def drop_table(self, model: "Type[Model]"):
         raise NotImplementedError
 

aerich/ddl/mysql/__init__.py

@@ -1,6 +1,6 @@
 from typing import List, Type
 
-from tortoise import ForeignKeyFieldInstance, Model
+from tortoise import ForeignKeyFieldInstance, ManyToManyFieldInstance, Model
 from tortoise.backends.mysql.schema_generator import MySQLSchemaGenerator
 from tortoise.fields import Field, JSONField, TextField, UUIDField
 
@@ -17,6 +17,28 @@ class MysqlDDL(BaseDDL):
     def drop_table(self, model: "Type[Model]"):
         return self._DROP_TABLE_TEMPLATE.format(table_name=model._meta.db_table)
 
+    def create_m2m_table(self, model: "Type[Model]", field: ManyToManyFieldInstance):
+        return self._M2M_TABLE_TEMPLATE.format(
+            table_name=field.through,
+            backward_table=model._meta.db_table,
+            forward_table=field.related_model._meta.db_table,
+            backward_field=model._meta.db_pk_column,
+            forward_field=field.related_model._meta.db_pk_column,
+            backward_key=field.backward_key,
+            backward_type=model._meta.pk.get_for_dialect(self.DIALECT, "SQL_TYPE"),
+            forward_key=field.forward_key,
+            forward_type=field.related_model._meta.pk.get_for_dialect(self.DIALECT, "SQL_TYPE"),
+            extra=self.schema_generator._table_generate_extra(table=field.through),
+            comment=self.schema_generator._table_comment_generator(
+                table=field.through, comment=field.description
+            )
+            if field.description
+            else "",
+        )
+
+    def drop_m2m(self, field: ManyToManyFieldInstance):
+        return self._DROP_TABLE_TEMPLATE.format(table_name=field.through)
+
     def add_column(self, model: "Type[Model]", field_object: Field):
         db_table = model._meta.db_table
         default = field_object.default

aerich/migrate.py

@@ -5,7 +5,14 @@ from copy import deepcopy
 from datetime import datetime
 from typing import Dict, List, Type
 
-from tortoise import BackwardFKRelation, ForeignKeyFieldInstance, Model, Tortoise
+from tortoise import (
+    BackwardFKRelation,
+    BackwardOneToOneRelation,
+    ForeignKeyFieldInstance,
+    ManyToManyFieldInstance,
+    Model,
+    Tortoise,
+)
 from tortoise.backends.mysql.schema_generator import MySQLSchemaGenerator
 from tortoise.fields import Field
 
@@ -18,8 +25,10 @@ from aerich.utils import get_app_connection
 class Migrate:
     upgrade_operators: List[str] = []
     downgrade_operators: List[str] = []
-    _upgrade_fk_operators: List[str] = []
+    _upgrade_fk_m2m_operators: List[str] = []
-    _downgrade_fk_operators: List[str] = []
+    _downgrade_fk_m2m_operators: List[str] = []
+    _upgrade_m2m: List[str] = []
+    _downgrade_m2m: List[str] = []
 
     ddl: BaseDDL
     migrate_config: dict
@@ -80,7 +89,7 @@ class Migrate:
             "migrate": False,
         }
         with open(os.path.join(cls.migrate_location, filename), "w") as f:
-            json.dump(content, f, indent=4, ensure_ascii=False)
+            json.dump(content, f, indent=2, ensure_ascii=False)
         return filename
 
     @classmethod
@@ -99,11 +108,11 @@ class Migrate:
         cls._diff_models(diff_models, app_models)
         cls._diff_models(app_models, diff_models, False)
 
+        cls._merge_operators()
+
         if not cls.upgrade_operators:
             return False
 
-        cls._merge_operators()
-
         return cls._generate_diff_sql(name)
 
     @classmethod
@@ -112,17 +121,17 @@ class Migrate:
         add operator,differentiate fk because fk is order limit
         :param operator:
         :param upgrade:
-        :param fk:
+        :param fk_m2m:
         :return:
         """
         if upgrade:
             if fk:
-                cls._upgrade_fk_operators.append(operator)
+                cls._upgrade_fk_m2m_operators.append(operator)
             else:
                 cls.upgrade_operators.append(operator)
         else:
             if fk:
-                cls._downgrade_fk_operators.append(operator)
+                cls._downgrade_fk_m2m_operators.append(operator)
             else:
                 cls.downgrade_operators.append(operator)
 
@@ -132,6 +141,7 @@ class Migrate:
     ):
         """
         cp currents models to old_model_files
+        :param app:
         :param model_files:
         :param old_model_file:
         :return:
@@ -219,13 +229,13 @@ class Migrate:
         new_keys = new_fields_map.keys()
         for new_key in new_keys:
             new_field = new_fields_map.get(new_key)
-            if cls._exclude_field(new_field):
+            if cls._exclude_field(new_field, upgrade):
                 continue
             if new_key not in old_keys:
                 cls._add_operator(
                     cls._add_field(new_model, new_field),
                     upgrade,
-                    isinstance(new_field, ForeignKeyFieldInstance),
+                    isinstance(new_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)),
                 )
             else:
                 old_field = old_fields_map.get(new_key)
@@ -233,22 +243,22 @@ class Migrate:
                     cls._add_operator(
                         cls._remove_index(old_model, old_field),
                         upgrade,
-                        isinstance(old_field, ForeignKeyFieldInstance),
+                        isinstance(old_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)),
                     )
                 elif new_field.index and not old_field.index:
                     cls._add_operator(
                         cls._add_index(new_model, new_field),
                         upgrade,
-                        isinstance(new_field, ForeignKeyFieldInstance),
+                        isinstance(new_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)),
                     )
 
         for old_key in old_keys:
             field = old_fields_map.get(old_key)
-            if old_key not in new_keys and not cls._exclude_field(field):
+            if old_key not in new_keys and not cls._exclude_field(field, upgrade):
                 cls._add_operator(
                     cls._remove_field(old_model, field),
                     upgrade,
-                    isinstance(field, ForeignKeyFieldInstance),
+                    isinstance(field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)),
                 )
 
     @classmethod
@@ -260,25 +270,42 @@ class Migrate:
         return cls.ddl.add_index(model, [field.model_field_name], field.unique)
 
     @classmethod
-    def _exclude_field(cls, field: Field):
+    def _exclude_field(cls, field: Field, upgrade=False):
         """
-        exclude BackwardFKRelation
+        exclude BackwardFKRelation and repeat m2m field
         :param field:
         :return:
         """
-        return isinstance(field, BackwardFKRelation)
+        if isinstance(field, ManyToManyFieldInstance):
+            through = field.through
+            if upgrade:
+                if through in cls._upgrade_m2m:
+                    return True
+                else:
+                    cls._upgrade_m2m.append(through)
+                    return False
+            else:
+                if through in cls._downgrade_m2m:
+                    return True
+                else:
+                    cls._downgrade_m2m.append(through)
+                    return False
+        return isinstance(field, (BackwardFKRelation, BackwardOneToOneRelation))
 
     @classmethod
     def _add_field(cls, model: Type[Model], field: Field):
         if isinstance(field, ForeignKeyFieldInstance):
             return cls.ddl.add_fk(model, field)
-        else:
+        if isinstance(field, ManyToManyFieldInstance):
-            return cls.ddl.add_column(model, field)
+            return cls.ddl.create_m2m_table(model, field)
+        return cls.ddl.add_column(model, field)
 
     @classmethod
     def _remove_field(cls, model: Type[Model], field: Field):
         if isinstance(field, ForeignKeyFieldInstance):
             return cls.ddl.drop_fk(model, field)
+        if isinstance(field, ManyToManyFieldInstance):
+            return cls.ddl.drop_m2m(field)
         return cls.ddl.drop_column(model, field.model_field_name)
 
     @classmethod
@@ -304,16 +331,17 @@ class Migrate:
     @classmethod
     def _merge_operators(cls):
         """
-        fk must be last when add,first when drop
+        fk/m2m must be last when add,first when drop
         :return:
         """
-        for _upgrade_fk_operator in cls._upgrade_fk_operators:
+        for _upgrade_fk_m2m_operator in cls._upgrade_fk_m2m_operators:
-            if "ADD" in _upgrade_fk_operator:
+            if "ADD" in _upgrade_fk_m2m_operator or "CREATE" in _upgrade_fk_m2m_operator:
-                cls.upgrade_operators.append(_upgrade_fk_operator)
+                cls.upgrade_operators.append(_upgrade_fk_m2m_operator)
             else:
-                cls.upgrade_operators.insert(0, _upgrade_fk_operator)
+                cls.upgrade_operators.insert(0, _upgrade_fk_m2m_operator)
-        for _downgrade_fk_operator in cls._downgrade_fk_operators:
+
-            if "ADD" in _downgrade_fk_operator:
+        for _downgrade_fk_m2m_operator in cls._downgrade_fk_m2m_operators:
-                cls.downgrade_operators.append(_downgrade_fk_operator)
+            if "ADD" in _downgrade_fk_m2m_operator or "CREATE" in _downgrade_fk_m2m_operator:
+                cls.downgrade_operators.append(_downgrade_fk_m2m_operator)
             else:
-                cls.downgrade_operators.insert(0, _downgrade_fk_operator)
+                cls.downgrade_operators.insert(0, _downgrade_fk_m2m_operator)

aerich/utils.py

@@ -1,3 +1,6 @@
+import importlib
+
+from asyncclick import BadOptionUsage, Context
 from tortoise import Tortoise
 
 
@@ -9,3 +12,30 @@ def get_app_connection(config, app):
     :return:
     """
     return Tortoise.get_connection(config.get("apps").get(app).get("default_connection"))
+
+
+def get_tortoise_config(ctx: Context, tortoise_orm: str) -> dict:
+    """
+    get tortoise config from module
+    :param ctx:
+    :param tortoise_orm:
+    :return:
+    """
+    splits = tortoise_orm.split(".")
+    config_path = ".".join(splits[:-1])
+    tortoise_config = splits[-1]
+    try:
+        config_module = importlib.import_module(config_path)
+    except (ModuleNotFoundError, AttributeError):
+        raise BadOptionUsage(
+            ctx=ctx, message=f'No config named "{config_path}"', option_name="--config"
+        )
+
+    config = getattr(config_module, tortoise_config, None)
+    if not config:
+        raise BadOptionUsage(
+            option_name="--config",
+            message=f'Can\'t get "{tortoise_config}" from module "{config_module}"',
+            ctx=ctx,
+        )
+    return config