diff --git a/.gitignore b/.gitignore index e941050..9ba7de7 100644 --- a/.gitignore +++ b/.gitignore @@ -141,4 +141,5 @@ dmypy.json cython_debug/ .idea -migrations \ No newline at end of file +migrations +aerich.ini \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fcdd7ce..cdc3089 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,9 +2,14 @@ ChangeLog ========= - 0.1 === +0.1.2 +----- +- Now aerich support m2m. +- Add cli cmd init-db. +- Change cli options. + 0.1.1 ----- - Now aerich is basic worked. \ No newline at end of file diff --git a/README.rst b/README.rst index 13117c6..b04ad2c 100644 --- a/README.rst +++ b/README.rst @@ -35,41 +35,66 @@ Quick Start Usage: aerich [OPTIONS] COMMAND [ARGS]... Options: - --config TEXT Tortoise-ORM config module, will auto read config dict variable - from it. [default: settings] - --tortoise-orm TEXT Tortoise-ORM config dict variable. [default: - TORTOISE_ORM] - --location TEXT Migrate store location. [default: ./migrations] - --app TEXT Tortoise-ORM app name. [default: models] - -h, --help Show this message and exit. + -c, --config TEXT Config file. [default: aerich.ini] + --app TEXT Tortoise-ORM app name. [default: models] + -n, --name TEXT Name of section in .ini file to use for aerich config. + [default: aerich] + -h, --help Show this message and exit. Commands: downgrade Downgrade to previous version. heads Show current available heads in migrate location. history List all migrate items. - init Init migrate location and generate schema, you must exec first. + init Init config file and generate migrate location. + init-db Generate schema. migrate Generate migrate changes file. upgrade Upgrade to latest version. Usage ===== -Init schema and migrate location --------------------------------- +Initialization +-------------- .. code-block:: shell - $ aerich --config tests.backends.mysql init + $ aerich init -h - Success create migrate location ./migrations/models - Success init for app "models" + Usage: aerich init [OPTIONS] + + Init config file and generate migrate location, you must exec first. + + Options: + -t, --tortoise-orm TEXT Tortoise-ORM config module dict variable. + [required] + --location TEXT Migrate store location. [default: ./migrations] + -h, --help Show this message and exit. + +Init config file and location: + +.. code-block:: shell + + $ aerich init -t tests.backends.mysql.TORTOISE_ORM + + Success create migrate location ./migrations + Success generate config file aerich.ini + +Init db +------- + +.. code-block:: shell + + $ aerich init-db + + Success create app migrate location ./migrations/models + Success generate schema for app "models" Update models and make migrate ------------------------------ .. code-block:: shell - $ aerich --config tests.backends.mysql migrate --name drop_column + $ aerich migrate --name drop_column Success migrate 1_202029051520102929_drop_column.json @@ -80,7 +105,7 @@ Upgrade to latest version .. code-block:: shell - $ aerich --config tests.backends.mysql upgrade + $ aerich upgrade Success upgrade 1_202029051520102929_drop_column.json @@ -91,7 +116,7 @@ Downgrade to previous version .. code-block:: shell - $ aerich --config tests.backends.mysql downgrade + $ aerich downgrade Success downgrade 1_202029051520102929_drop_column.json @@ -102,7 +127,7 @@ Show history .. code-block:: shell - $ aerich --config tests.backends.mysql history + $ aerich history 1_202029051520102929_drop_column.json @@ -111,7 +136,7 @@ Show heads to be migrated .. code-block:: shell - $ aerich --config tests.backends.mysql heads + $ aerich heads 1_202029051520102929_drop_column.json diff --git a/aerich/__init__.py b/aerich/__init__.py index 485f44a..b3f4756 100644 --- a/aerich/__init__.py +++ b/aerich/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/aerich/cli.py b/aerich/cli.py index 8160e74..a304fbc 100644 --- a/aerich/cli.py +++ b/aerich/cli.py @@ -1,15 +1,15 @@ -import importlib import json import os import sys +from configparser import ConfigParser from enum import Enum import asyncclick as click -from asyncclick import BadOptionUsage, Context, UsageError +from asyncclick import Context, UsageError from tortoise import Tortoise, generate_schema_for_client from aerich.migrate import Migrate -from aerich.utils import get_app_connection +from aerich.utils import get_app_connection, get_tortoise_config class Color(str, Enum): @@ -18,51 +18,43 @@ class Color(str, Enum): yellow = "yellow" +parser = ConfigParser() + + @click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( - "--config", - default="settings", - show_default=True, - help="Tortoise-ORM config module, will auto read dict config variable from it.", -) -@click.option( - "--tortoise-orm", - default="TORTOISE_ORM", - show_default=True, - help="Tortoise-ORM config dict variable.", -) -@click.option( - "--location", default="./migrations", show_default=True, help="Migrate store location." + "-c", "--config", default="aerich.ini", show_default=True, help="Config file.", ) @click.option("--app", default="models", show_default=True, help="Tortoise-ORM app name.") +@click.option( + "-n", + "--name", + default="aerich", + show_default=True, + help="Name of section in .ini file to use for aerich config.", +) @click.pass_context -async def cli(ctx: Context, config, tortoise_orm, location, app): +async def cli(ctx: Context, config, app, name): ctx.ensure_object(dict) - try: - config_module = importlib.import_module(config, ".") - except ModuleNotFoundError: - raise BadOptionUsage(ctx=ctx, message=f'No module named "{config}"', option_name="--config") - config = getattr(config_module, tortoise_orm, None) - if not config: - raise BadOptionUsage( - option_name="--config", - message=f'Can\'t get "{tortoise_orm}" from module "{config_module}"', - ctx=ctx, - ) - - if app not in config.get("apps").keys(): - raise BadOptionUsage(option_name="--config", message=f'No app found in "{config}"', ctx=ctx) - ctx.obj["config"] = config - ctx.obj["location"] = location - ctx.obj["app"] = app - - if ctx.invoked_subcommand == "init": - await Tortoise.init(config=config) - else: - if not os.path.isdir(location): + ctx.obj["name"] = name + invoked_subcommand = ctx.invoked_subcommand + if invoked_subcommand != "init": + if not os.path.exists(config): raise UsageError("You must exec init first", ctx=ctx) - await Migrate.init_with_old_models(config, app, location) + parser.read(config) + + location = parser[name]["location"] + tortoise_orm = parser[name]["tortoise_orm"] + + tortoise_config = get_tortoise_config(ctx, tortoise_orm) + + ctx.obj["config"] = tortoise_config + ctx.obj["location"] = location + ctx.obj["app"] = app + + if invoked_subcommand != "init-db": + await Migrate.init_with_old_models(tortoise_config, app, location) @cli.command(help="Generate migrate changes file.") @@ -100,7 +92,7 @@ async def upgrade(ctx: Context): with open(file_path, "w") as f: content["migrate"] = True - json.dump(content, f, indent=4, ensure_ascii=False) + json.dump(content, f, indent=2, ensure_ascii=False) click.secho(f"Success upgrade {file}", fg=Color.green) @@ -127,7 +119,7 @@ async def downgrade(ctx: Context): continue with open(file_path, "w") as f: content["migrate"] = False - json.dump(content, f, indent=4, ensure_ascii=False) + json.dump(content, f, indent=2, ensure_ascii=False) return click.secho(f"Success downgrade {file}", fg=Color.green) @@ -135,17 +127,45 @@ async def downgrade(ctx: Context): @click.pass_context def heads(ctx: Context): for version in Migrate.get_all_version_files(is_all=False): - click.secho(version, fg=Color.yellow) + click.secho(version, fg=Color.green) @cli.command(help="List all migrate items.") @click.pass_context def history(ctx): for version in Migrate.get_all_version_files(): - click.secho(version, fg=Color.yellow) + click.secho(version, fg=Color.green) -@cli.command(help="Init migrate location and generate schema, you must exec first.") +@cli.command(help="Init config file and generate migrate location.") +@click.option( + "-t", "--tortoise-orm", required=True, help="Tortoise-ORM config module dict variable.", +) +@click.option( + "--location", default="./migrations", show_default=True, help="Migrate store location." +) +@click.pass_context +async def init( + ctx: Context, tortoise_orm, location, +): + config = ctx.obj["config"] + name = ctx.obj["name"] + + parser.add_section(name) + parser.set(name, "tortoise_orm", tortoise_orm) + parser.set(name, "location", location) + + with open(config, "w") as f: + parser.write(f) + + if not os.path.isdir(location): + os.mkdir(location) + + click.secho(f"Success create migrate location {location}", fg=Color.green) + click.secho(f"Success generate config file {config}", fg=Color.green) + + +@cli.command(help="Generate schema.") @click.option( "--safe", is_flag=True, @@ -154,27 +174,25 @@ def history(ctx): show_default=True, ) @click.pass_context -async def init(ctx: Context, safe): +async def init_db(ctx: Context, safe): + config = ctx.obj["config"] location = ctx.obj["location"] app = ctx.obj["app"] - config = ctx.obj["config"] - - if not os.path.isdir(location): - os.mkdir(location) dirname = os.path.join(location, app) if not os.path.isdir(dirname): os.mkdir(dirname) - click.secho(f"Success create migrate location {dirname}", fg=Color.green) + click.secho(f"Success create app migrate location {dirname}", fg=Color.green) else: return click.secho(f'Already inited app "{app}"', fg=Color.yellow) Migrate.write_old_models(config, app, location) + await Tortoise.init(config=config) connection = get_app_connection(config, app) await generate_schema_for_client(connection, safe) - return click.secho(f'Success init for app "{app}"', fg=Color.green) + return click.secho(f'Success generate schema for app "{app}"', fg=Color.green) def main(): diff --git a/aerich/ddl/__init__.py b/aerich/ddl/__init__.py index affff60..91bb088 100644 --- a/aerich/ddl/__init__.py +++ b/aerich/ddl/__init__.py @@ -1,6 +1,6 @@ from typing import List, Type -from tortoise import BaseDBAsyncClient, ForeignKeyFieldInstance, Model +from tortoise import BaseDBAsyncClient, ForeignKeyFieldInstance, ManyToManyFieldInstance, Model from tortoise.backends.base.schema_generator import BaseSchemaGenerator from tortoise.fields import Field @@ -8,7 +8,7 @@ from tortoise.fields import Field class BaseDDL: schema_generator_cls: Type[BaseSchemaGenerator] = BaseSchemaGenerator DIALECT = "sql" - _DROP_TABLE_TEMPLATE = "DROP TABLE {table_name} IF EXISTS" + _DROP_TABLE_TEMPLATE = "DROP TABLE IF EXISTS {table_name}" _ADD_COLUMN_TEMPLATE = "ALTER TABLE {table_name} ADD {column}" _DROP_COLUMN_TEMPLATE = "ALTER TABLE {table_name} DROP COLUMN {column_name}" _ADD_INDEX_TEMPLATE = ( @@ -17,6 +17,7 @@ class BaseDDL: _DROP_INDEX_TEMPLATE = "ALTER TABLE {table_name} DROP INDEX {index_name}" _ADD_FK_TEMPLATE = "ALTER TABLE {table_name} ADD CONSTRAINT `{fk_name}` FOREIGN KEY (`{db_column}`) REFERENCES `{table}` (`{field}`) ON DELETE {on_delete}" _DROP_FK_TEMPLATE = "ALTER TABLE {table_name} DROP FOREIGN KEY {fk_name}" + _M2M_TABLE_TEMPLATE = "CREATE TABLE {table_name} ({backward_key} {backward_type} NOT NULL REFERENCES {backward_table} ({backward_field}) ON DELETE CASCADE,{forward_key} {forward_type} NOT NULL REFERENCES {forward_table} ({forward_field}) ON DELETE CASCADE){extra}{comment};" def __init__(self, client: "BaseDBAsyncClient"): self.client = client @@ -25,6 +26,12 @@ class BaseDDL: def create_table(self, model: "Type[Model]"): raise NotImplementedError + def create_m2m_table(self, model: "Type[Model]", field: ManyToManyFieldInstance): + raise NotImplementedError + + def drop_m2m(self, field: ManyToManyFieldInstance): + raise NotImplementedError + def drop_table(self, model: "Type[Model]"): raise NotImplementedError diff --git a/aerich/ddl/mysql/__init__.py b/aerich/ddl/mysql/__init__.py index a2ea10c..4617cbf 100644 --- a/aerich/ddl/mysql/__init__.py +++ b/aerich/ddl/mysql/__init__.py @@ -1,6 +1,6 @@ from typing import List, Type -from tortoise import ForeignKeyFieldInstance, Model +from tortoise import ForeignKeyFieldInstance, ManyToManyFieldInstance, Model from tortoise.backends.mysql.schema_generator import MySQLSchemaGenerator from tortoise.fields import Field, JSONField, TextField, UUIDField @@ -17,6 +17,28 @@ class MysqlDDL(BaseDDL): def drop_table(self, model: "Type[Model]"): return self._DROP_TABLE_TEMPLATE.format(table_name=model._meta.db_table) + def create_m2m_table(self, model: "Type[Model]", field: ManyToManyFieldInstance): + return self._M2M_TABLE_TEMPLATE.format( + table_name=field.through, + backward_table=model._meta.db_table, + forward_table=field.related_model._meta.db_table, + backward_field=model._meta.db_pk_column, + forward_field=field.related_model._meta.db_pk_column, + backward_key=field.backward_key, + backward_type=model._meta.pk.get_for_dialect(self.DIALECT, "SQL_TYPE"), + forward_key=field.forward_key, + forward_type=field.related_model._meta.pk.get_for_dialect(self.DIALECT, "SQL_TYPE"), + extra=self.schema_generator._table_generate_extra(table=field.through), + comment=self.schema_generator._table_comment_generator( + table=field.through, comment=field.description + ) + if field.description + else "", + ) + + def drop_m2m(self, field: ManyToManyFieldInstance): + return self._DROP_TABLE_TEMPLATE.format(table_name=field.through) + def add_column(self, model: "Type[Model]", field_object: Field): db_table = model._meta.db_table default = field_object.default diff --git a/aerich/migrate.py b/aerich/migrate.py index d189ea3..74feae6 100644 --- a/aerich/migrate.py +++ b/aerich/migrate.py @@ -5,7 +5,14 @@ from copy import deepcopy from datetime import datetime from typing import Dict, List, Type -from tortoise import BackwardFKRelation, ForeignKeyFieldInstance, Model, Tortoise +from tortoise import ( + BackwardFKRelation, + BackwardOneToOneRelation, + ForeignKeyFieldInstance, + ManyToManyFieldInstance, + Model, + Tortoise, +) from tortoise.backends.mysql.schema_generator import MySQLSchemaGenerator from tortoise.fields import Field @@ -18,8 +25,10 @@ from aerich.utils import get_app_connection class Migrate: upgrade_operators: List[str] = [] downgrade_operators: List[str] = [] - _upgrade_fk_operators: List[str] = [] - _downgrade_fk_operators: List[str] = [] + _upgrade_fk_m2m_operators: List[str] = [] + _downgrade_fk_m2m_operators: List[str] = [] + _upgrade_m2m: List[str] = [] + _downgrade_m2m: List[str] = [] ddl: BaseDDL migrate_config: dict @@ -80,7 +89,7 @@ class Migrate: "migrate": False, } with open(os.path.join(cls.migrate_location, filename), "w") as f: - json.dump(content, f, indent=4, ensure_ascii=False) + json.dump(content, f, indent=2, ensure_ascii=False) return filename @classmethod @@ -99,11 +108,11 @@ class Migrate: cls._diff_models(diff_models, app_models) cls._diff_models(app_models, diff_models, False) + cls._merge_operators() + if not cls.upgrade_operators: return False - cls._merge_operators() - return cls._generate_diff_sql(name) @classmethod @@ -112,17 +121,17 @@ class Migrate: add operator,differentiate fk because fk is order limit :param operator: :param upgrade: - :param fk: + :param fk_m2m: :return: """ if upgrade: if fk: - cls._upgrade_fk_operators.append(operator) + cls._upgrade_fk_m2m_operators.append(operator) else: cls.upgrade_operators.append(operator) else: if fk: - cls._downgrade_fk_operators.append(operator) + cls._downgrade_fk_m2m_operators.append(operator) else: cls.downgrade_operators.append(operator) @@ -132,6 +141,7 @@ class Migrate: ): """ cp currents models to old_model_files + :param app: :param model_files: :param old_model_file: :return: @@ -219,13 +229,13 @@ class Migrate: new_keys = new_fields_map.keys() for new_key in new_keys: new_field = new_fields_map.get(new_key) - if cls._exclude_field(new_field): + if cls._exclude_field(new_field, upgrade): continue if new_key not in old_keys: cls._add_operator( cls._add_field(new_model, new_field), upgrade, - isinstance(new_field, ForeignKeyFieldInstance), + isinstance(new_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)), ) else: old_field = old_fields_map.get(new_key) @@ -233,22 +243,22 @@ class Migrate: cls._add_operator( cls._remove_index(old_model, old_field), upgrade, - isinstance(old_field, ForeignKeyFieldInstance), + isinstance(old_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)), ) elif new_field.index and not old_field.index: cls._add_operator( cls._add_index(new_model, new_field), upgrade, - isinstance(new_field, ForeignKeyFieldInstance), + isinstance(new_field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)), ) for old_key in old_keys: field = old_fields_map.get(old_key) - if old_key not in new_keys and not cls._exclude_field(field): + if old_key not in new_keys and not cls._exclude_field(field, upgrade): cls._add_operator( cls._remove_field(old_model, field), upgrade, - isinstance(field, ForeignKeyFieldInstance), + isinstance(field, (ForeignKeyFieldInstance, ManyToManyFieldInstance)), ) @classmethod @@ -260,25 +270,42 @@ class Migrate: return cls.ddl.add_index(model, [field.model_field_name], field.unique) @classmethod - def _exclude_field(cls, field: Field): + def _exclude_field(cls, field: Field, upgrade=False): """ - exclude BackwardFKRelation + exclude BackwardFKRelation and repeat m2m field :param field: :return: """ - return isinstance(field, BackwardFKRelation) + if isinstance(field, ManyToManyFieldInstance): + through = field.through + if upgrade: + if through in cls._upgrade_m2m: + return True + else: + cls._upgrade_m2m.append(through) + return False + else: + if through in cls._downgrade_m2m: + return True + else: + cls._downgrade_m2m.append(through) + return False + return isinstance(field, (BackwardFKRelation, BackwardOneToOneRelation)) @classmethod def _add_field(cls, model: Type[Model], field: Field): if isinstance(field, ForeignKeyFieldInstance): return cls.ddl.add_fk(model, field) - else: - return cls.ddl.add_column(model, field) + if isinstance(field, ManyToManyFieldInstance): + return cls.ddl.create_m2m_table(model, field) + return cls.ddl.add_column(model, field) @classmethod def _remove_field(cls, model: Type[Model], field: Field): if isinstance(field, ForeignKeyFieldInstance): return cls.ddl.drop_fk(model, field) + if isinstance(field, ManyToManyFieldInstance): + return cls.ddl.drop_m2m(field) return cls.ddl.drop_column(model, field.model_field_name) @classmethod @@ -304,16 +331,17 @@ class Migrate: @classmethod def _merge_operators(cls): """ - fk must be last when add,first when drop + fk/m2m must be last when add,first when drop :return: """ - for _upgrade_fk_operator in cls._upgrade_fk_operators: - if "ADD" in _upgrade_fk_operator: - cls.upgrade_operators.append(_upgrade_fk_operator) + for _upgrade_fk_m2m_operator in cls._upgrade_fk_m2m_operators: + if "ADD" in _upgrade_fk_m2m_operator or "CREATE" in _upgrade_fk_m2m_operator: + cls.upgrade_operators.append(_upgrade_fk_m2m_operator) else: - cls.upgrade_operators.insert(0, _upgrade_fk_operator) - for _downgrade_fk_operator in cls._downgrade_fk_operators: - if "ADD" in _downgrade_fk_operator: - cls.downgrade_operators.append(_downgrade_fk_operator) + cls.upgrade_operators.insert(0, _upgrade_fk_m2m_operator) + + for _downgrade_fk_m2m_operator in cls._downgrade_fk_m2m_operators: + if "ADD" in _downgrade_fk_m2m_operator or "CREATE" in _downgrade_fk_m2m_operator: + cls.downgrade_operators.append(_downgrade_fk_m2m_operator) else: - cls.downgrade_operators.insert(0, _downgrade_fk_operator) + cls.downgrade_operators.insert(0, _downgrade_fk_m2m_operator) diff --git a/aerich/utils.py b/aerich/utils.py index 539eb55..4e58c36 100644 --- a/aerich/utils.py +++ b/aerich/utils.py @@ -1,3 +1,6 @@ +import importlib + +from asyncclick import BadOptionUsage, Context from tortoise import Tortoise @@ -9,3 +12,30 @@ def get_app_connection(config, app): :return: """ return Tortoise.get_connection(config.get("apps").get(app).get("default_connection")) + + +def get_tortoise_config(ctx: Context, tortoise_orm: str) -> dict: + """ + get tortoise config from module + :param ctx: + :param tortoise_orm: + :return: + """ + splits = tortoise_orm.split(".") + config_path = ".".join(splits[:-1]) + tortoise_config = splits[-1] + try: + config_module = importlib.import_module(config_path) + except (ModuleNotFoundError, AttributeError): + raise BadOptionUsage( + ctx=ctx, message=f'No config named "{config_path}"', option_name="--config" + ) + + config = getattr(config_module, tortoise_config, None) + if not config: + raise BadOptionUsage( + option_name="--config", + message=f'Can\'t get "{tortoise_config}" from module "{config_module}"', + ctx=ctx, + ) + return config