merge dev

CHANGELOG.md

@@ -1,8 +1,17 @@
 # ChangeLog
 
+## 0.8
+
+### 0.8.0
+
+- Fix mysql drop unique index raises OperationalError. (#346)
+
+  **Upgrade note:**
+    1. Use column name as unique key name for mysql
+
 ## 0.7
 
-### 0.7.2
+### [0.7.2] - 2023-07-20
 
 - Support virtual fields.
 - Fix modify multiple times. (#279)

Makefile

@@ -20,7 +20,7 @@ style: deps
 
 check: deps
 	@black --check $(black_opts) $(checkfiles) || (echo "Please run 'make style' to auto-fix style issues" && false)
-	@ruff $(checkfiles)
+	@ruff check $(checkfiles)
 
 test: deps
 	$(py_warn) TEST_DB=sqlite://:memory: py.test

README.md

@@ -46,7 +46,7 @@ Commands:
 
 ## Usage
 
-You need add `aerich.models` to your `Tortoise-ORM` config first. Example:
+You need to add `aerich.models` to your `Tortoise-ORM` config first. Example:
 
 ```python
 TORTOISE_ORM = {

aerich/ddl/__init__.py

@@ -66,11 +66,11 @@ class BaseDDL:
             forward_type=db_field_types.get(self.DIALECT) or db_field_types.get(""),
             on_delete=field_describe.get("on_delete"),
             extra=self.schema_generator._table_generate_extra(table=through),
-            comment=self.schema_generator._table_comment_generator(
+            comment=(
-                table=through, comment=description
+                self.schema_generator._table_comment_generator(table=through, comment=description)
-            )
+                if description
-            if description
+                else ""
-            else "",
+            ),
         )
 
     def drop_m2m(self, table_name: str):
@@ -121,13 +121,15 @@ class BaseDDL:
                 field_type=db_field_types.get(self.DIALECT, db_field_types.get("")),
                 nullable="NOT NULL" if not field_describe.get("nullable") else "",
                 unique="UNIQUE" if field_describe.get("unique") else "",
-                comment=self.schema_generator._column_comment_generator(
+                comment=(
-                    table=db_table,
+                    self.schema_generator._column_comment_generator(
-                    column=db_column,
+                        table=db_table,
-                    comment=field_describe.get("description"),
+                        column=db_column,
-                )
+                        comment=field_describe.get("description"),
-                if description
+                    )
-                else "",
+                    if description
+                    else ""
+                ),
                 is_primary_key=is_pk,
                 default=default,
             ),
@@ -151,13 +153,15 @@ class BaseDDL:
                 field_type=db_field_types.get(self.DIALECT) or db_field_types.get(""),
                 nullable="NOT NULL" if not field_describe.get("nullable") else "",
                 unique="",
-                comment=self.schema_generator._column_comment_generator(
+                comment=(
-                    table=db_table,
+                    self.schema_generator._column_comment_generator(
-                    column=field_describe.get("db_column"),
+                        table=db_table,
-                    comment=field_describe.get("description"),
+                        column=field_describe.get("db_column"),
-                )
+                        comment=field_describe.get("description"),
-                if field_describe.get("description")
+                    )
-                else "",
+                    if field_describe.get("description")
+                    else ""
+                ),
                 is_primary_key=is_pk,
                 default=default,
             ),

aerich/ddl/mysql/__init__.py

@@ -1,7 +1,12 @@
+from typing import TYPE_CHECKING, List, Type
+
 from tortoise.backends.mysql.schema_generator import MySQLSchemaGenerator
 
 from aerich.ddl import BaseDDL
 
+if TYPE_CHECKING:
+    from tortoise import Model  # noqa:F401
+
 
 class MysqlDDL(BaseDDL):
     schema_generator_cls = MySQLSchemaGenerator
@@ -30,3 +35,29 @@ class MysqlDDL(BaseDDL):
     )
     _MODIFY_COLUMN_TEMPLATE = "ALTER TABLE `{table_name}` MODIFY COLUMN {column}"
     _RENAME_TABLE_TEMPLATE = "ALTER TABLE `{old_table_name}` RENAME TO `{new_table_name}`"
+
+    def _index_name(self, unique: bool, model: "Type[Model]", field_names: List[str]) -> str:
+        if unique:
+            if len(field_names) == 1:
+                # Example: `email = CharField(max_length=50, unique=True)`
+                # Generate schema: `"email" VARCHAR(10) NOT NULL UNIQUE`
+                # Unique index key is the same as field name: `email`
+                return field_names[0]
+            index_prefix = "uid"
+        else:
+            index_prefix = "idx"
+        return self.schema_generator._generate_index_name(index_prefix, model, field_names)
+
+    def add_index(self, model: "Type[Model]", field_names: List[str], unique=False) -> str:
+        return self._ADD_INDEX_TEMPLATE.format(
+            unique="UNIQUE " if unique else "",
+            index_name=self._index_name(unique, model, field_names),
+            table_name=model._meta.db_table,
+            column_names=", ".join(self.schema_generator.quote(f) for f in field_names),
+        )
+
+    def drop_index(self, model: "Type[Model]", field_names: List[str], unique=False) -> str:
+        return self._DROP_INDEX_TEMPLATE.format(
+            index_name=self._index_name(unique, model, field_names),
+            table_name=model._meta.db_table,
+        )

aerich/ddl/postgres/__init__.py

@@ -43,7 +43,9 @@ class PostgresDDL(BaseDDL):
         return self._SET_COMMENT_TEMPLATE.format(
             table_name=db_table,
             column=field_describe.get("db_column") or field_describe.get("raw_field"),
-            comment="'{}'".format(field_describe.get("description"))
+            comment=(
-            if field_describe.get("description")
+                "'{}'".format(field_describe.get("description"))
-            else "NULL",
+                if field_describe.get("description")
+                else "NULL"
+            ),
         )

aerich/migrate.py

@@ -473,16 +473,16 @@ class Migrate:
                         _, option, old_new = change
                         if option == "indexed":
                             # change index
-                            unique = new_data_field.get("unique")
                             if old_new[0] is False and old_new[1] is True:
-                                add_or_drop_index = cls._add_index
+                                unique = new_data_field.get("unique")
+                                cls._add_operator(
+                                    cls._add_index(model, (field_name,), unique), upgrade, True
+                                )
                             else:
-                                # For drop case, unique value should get from old data
+                                unique = old_data_field.get("unique")
-                                # TODO: unique = old_data_field.get("unique")
+                                cls._add_operator(
-                                add_or_drop_index = cls._drop_index
+                                    cls._drop_index(model, (field_name,), unique), upgrade, True
-                            cls._add_operator(
+                                )
-                                add_or_drop_index(model, (field_name,), unique), upgrade, True
-                            )
                         elif option == "db_field_types.":
                             if new_data_field.get("field_type") == "DecimalField":
                                 # modify column

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "aerich"
-version = "0.7.2"
+version = "0.8.0"
 description = "A database migrations tool for Tortoise ORM."
 authors = ["long2ice <long2ice@gmail.com>"]
 license = "Apache-2.0"
@@ -15,30 +15,34 @@ packages = [
 include = ["CHANGELOG.md", "LICENSE", "README.md"]
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.8"
 tortoise-orm = "*"
 click = "*"
 asyncpg = { version = "*", optional = true }
-asyncmy = { version = "^0.2.8rc1", optional = true, allow-prereleases = true }
+asyncmy = { version = "^0.2.9", optional = true, allow-prereleases = true }
 pydantic = "^2.0"
 dictdiffer = "*"
 tomlkit = "*"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 ruff = "*"
 isort = "*"
 black = "*"
 pytest = "*"
 pytest-xdist = "*"
-pytest-asyncio = "*"
+# Breaking change in 0.23.*
+# https://github.com/pytest-dev/pytest-asyncio/issues/706
+pytest-asyncio = "^0.21.2"
 bandit = "*"
 pytest-mock = "*"
 cryptography = "*"
+mypy = "^1.10.0"
 
 [tool.poetry.extras]
 asyncmy = ["asyncmy"]
 asyncpg = ["asyncpg"]
 
+
 [tool.aerich]
 tortoise_orm = "conftest.tortoise_orm"
 location = "./migrations"
@@ -53,7 +57,7 @@ aerich = "aerich.cli:main"
 
 [tool.black]
 line-length = 100
-target-version = ['py36', 'py37', 'py38', 'py39']
+target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
 
 [tool.pytest.ini_options]
 asyncio_mode = 'auto'
@@ -62,5 +66,5 @@ asyncio_mode = 'auto'
 pretty = true
 ignore_missing_imports = true
 
-[tool.ruff]
+[tool.ruff.lint]
 ignore = ['E501']

tests/models.py

@@ -48,6 +48,7 @@ class Category(Model):
     slug = fields.CharField(max_length=100)
     name = fields.CharField(max_length=200, null=True, default=default_name)
     user = fields.ForeignKeyField("models.User", description="User")
+    title = fields.CharField(max_length=20, unique=False)
     created_at = fields.DatetimeField(auto_now_add=True)
 
 

tests/test_ddl.py

@@ -14,6 +14,7 @@ def test_create_table():
     `id` INT NOT NULL PRIMARY KEY AUTO_INCREMENT,
     `slug` VARCHAR(100) NOT NULL,
     `name` VARCHAR(200),
+    `title` VARCHAR(20) NOT NULL,
     `created_at` DATETIME(6) NOT NULL  DEFAULT CURRENT_TIMESTAMP(6),
     `user_id` INT NOT NULL COMMENT 'User',
     CONSTRAINT `fk_category_user_e2e3874c` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE CASCADE
@@ -27,6 +28,7 @@ def test_create_table():
     "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
     "slug" VARCHAR(100) NOT NULL,
     "name" VARCHAR(200),
+    "title" VARCHAR(20) NOT NULL,
     "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
     "user_id" INT NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE /* User */
 )"""
@@ -39,6 +41,7 @@ def test_create_table():
     "id" SERIAL NOT NULL PRIMARY KEY,
     "slug" VARCHAR(100) NOT NULL,
     "name" VARCHAR(200),
+    "title" VARCHAR(20) NOT NULL,
     "created_at" TIMESTAMPTZ NOT NULL  DEFAULT CURRENT_TIMESTAMP,
     "user_id" INT NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE
 );
@@ -151,9 +154,7 @@ def test_add_index():
     index_u = Migrate.ddl.add_index(Category, ["name"], True)
     if isinstance(Migrate.ddl, MysqlDDL):
         assert index == "ALTER TABLE `category` ADD INDEX `idx_category_name_8b0cb9` (`name`)"
-        assert (
+        assert index_u == "ALTER TABLE `category` ADD UNIQUE INDEX `name` (`name`)"
-            index_u == "ALTER TABLE `category` ADD UNIQUE INDEX `uid_category_name_8b0cb9` (`name`)"
-        )
     elif isinstance(Migrate.ddl, PostgresDDL):
         assert index == 'CREATE INDEX "idx_category_name_8b0cb9" ON "category" ("name")'
         assert index_u == 'CREATE UNIQUE INDEX "uid_category_name_8b0cb9" ON "category" ("name")'
@@ -169,7 +170,7 @@ def test_drop_index():
     ret_u = Migrate.ddl.drop_index(Category, ["name"], True)
     if isinstance(Migrate.ddl, MysqlDDL):
         assert ret == "ALTER TABLE `category` DROP INDEX `idx_category_name_8b0cb9`"
-        assert ret_u == "ALTER TABLE `category` DROP INDEX `uid_category_name_8b0cb9`"
+        assert ret_u == "ALTER TABLE `category` DROP INDEX `name`"
     elif isinstance(Migrate.ddl, PostgresDDL):
         assert ret == 'DROP INDEX "idx_category_name_8b0cb9"'
         assert ret_u == 'DROP INDEX "uid_category_name_8b0cb9"'

tests/test_migrate.py

@@ -2,6 +2,7 @@ import tempfile
 from pathlib import Path
 
 import pytest
+import tortoise
 from pytest_mock import MockerFixture
 
 from aerich.ddl.mysql import MysqlDDL
@@ -103,6 +104,21 @@ old_models_describe = {
                 "constraints": {"ge": 1, "le": 2147483647},
                 "db_field_types": {"": "INT"},
             },
+            {
+                "name": "title",
+                "field_type": "CharField",
+                "db_column": "title",
+                "python_type": "str",
+                "generated": False,
+                "nullable": False,
+                "unique": True,
+                "indexed": True,
+                "default": None,
+                "description": None,
+                "docstring": None,
+                "constraints": {"max_length": 20},
+                "db_field_types": {"": "VARCHAR(20)"},
+            },
         ],
         "fk_fields": [
             {
@@ -776,6 +792,16 @@ old_models_describe = {
 }
 
 
+def should_add_user_id_column_type_alter_sql() -> bool:
+    # tortoise-orm>=0.21 changes IntField constraints
+    # from {"ge": 1, "le": 2147483647} to {"ge": -2147483648,"le": 2147483647}
+    user_id_constraints = old_models_describe["models.Category"]["data_fields"][-1]["constraints"]
+    return (
+        tortoise.__version__ >= "0.21"
+        and tortoise.fields.data.IntField.constraints != user_id_constraints
+    )
+
+
 def test_migrate(mocker: MockerFixture):
     """
     models.py diff with old_models.py
@@ -786,7 +812,8 @@ def test_migrate(mocker: MockerFixture):
     - drop field: User.avatar
     - add index: Email.email
     - add many to many: Email.users
-    - remove unique: User.username
+    - remove unique: Category.title
+    - add unique: User.username
     - change column: length User.password
     - add unique_together: (name,type) of Product
     - alter default: Config.status
@@ -808,6 +835,7 @@ def test_migrate(mocker: MockerFixture):
         expected_upgrade_operators = {
             "ALTER TABLE `category` MODIFY COLUMN `name` VARCHAR(200)",
             "ALTER TABLE `category` MODIFY COLUMN `slug` VARCHAR(100) NOT NULL",
+            "ALTER TABLE `category` DROP INDEX `title`",
             "ALTER TABLE `config` ADD `user_id` INT NOT NULL  COMMENT 'User'",
             "ALTER TABLE `config` ADD CONSTRAINT `fk_config_user_17daa970` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE CASCADE",
             "ALTER TABLE `config` ALTER COLUMN `status` DROP DEFAULT",
@@ -830,7 +858,7 @@ def test_migrate(mocker: MockerFixture):
             "ALTER TABLE `user` MODIFY COLUMN `is_active` BOOL NOT NULL  COMMENT 'Is Active' DEFAULT 1",
             "ALTER TABLE `user` MODIFY COLUMN `is_superuser` BOOL NOT NULL  COMMENT 'Is SuperUser' DEFAULT 0",
             "ALTER TABLE `user` MODIFY COLUMN `longitude` DECIMAL(10,8) NOT NULL",
-            "ALTER TABLE `user` ADD UNIQUE INDEX `uid_user_usernam_9987ab` (`username`)",
+            "ALTER TABLE `user` ADD UNIQUE INDEX `username` (`username`)",
             "CREATE TABLE `email_user` (\n    `email_id` INT NOT NULL REFERENCES `email` (`email_id`) ON DELETE CASCADE,\n    `user_id` INT NOT NULL REFERENCES `user` (`id`) ON DELETE CASCADE\n) CHARACTER SET utf8mb4",
             "CREATE TABLE IF NOT EXISTS `newmodel` (\n    `id` INT NOT NULL PRIMARY KEY AUTO_INCREMENT,\n    `name` VARCHAR(50) NOT NULL\n) CHARACTER SET utf8mb4",
             "ALTER TABLE `category` MODIFY COLUMN `created_at` DATETIME(6) NOT NULL  DEFAULT CURRENT_TIMESTAMP(6)",
@@ -840,6 +868,7 @@ def test_migrate(mocker: MockerFixture):
         expected_downgrade_operators = {
             "ALTER TABLE `category` MODIFY COLUMN `name` VARCHAR(200) NOT NULL",
             "ALTER TABLE `category` MODIFY COLUMN `slug` VARCHAR(200) NOT NULL",
+            "ALTER TABLE `category` ADD UNIQUE INDEX `title` (`title`)",
             "ALTER TABLE `config` DROP COLUMN `user_id`",
             "ALTER TABLE `config` DROP FOREIGN KEY `fk_config_user_17daa970`",
             "ALTER TABLE `config` ALTER COLUMN `status` SET DEFAULT 1",
@@ -853,7 +882,7 @@ def test_migrate(mocker: MockerFixture):
             "ALTER TABLE `product` DROP INDEX `uid_product_name_869427`",
             "ALTER TABLE `product` ALTER COLUMN `view_num` DROP DEFAULT",
             "ALTER TABLE `user` ADD `avatar` VARCHAR(200) NOT NULL  DEFAULT ''",
-            "ALTER TABLE `user` DROP INDEX `idx_user_usernam_9987ab`",
+            "ALTER TABLE `user` DROP INDEX `username`",
             "ALTER TABLE `user` MODIFY COLUMN `password` VARCHAR(200) NOT NULL",
             "DROP TABLE IF EXISTS `email_user`",
             "DROP TABLE IF EXISTS `newmodel`",
@@ -869,6 +898,10 @@ def test_migrate(mocker: MockerFixture):
             "ALTER TABLE `product` MODIFY COLUMN `body` LONGTEXT NOT NULL",
             "ALTER TABLE `email` MODIFY COLUMN `is_primary` BOOL NOT NULL  DEFAULT 0",
         }
+        if should_add_user_id_column_type_alter_sql():
+            sql = "ALTER TABLE `category` MODIFY COLUMN `user_id` INT NOT NULL  COMMENT 'User'"
+            expected_upgrade_operators.add(sql)
+            expected_downgrade_operators.add(sql)
         assert not set(Migrate.upgrade_operators).symmetric_difference(expected_upgrade_operators)
 
         assert not set(Migrate.downgrade_operators).symmetric_difference(
@@ -877,6 +910,7 @@ def test_migrate(mocker: MockerFixture):
 
     elif isinstance(Migrate.ddl, PostgresDDL):
         expected_upgrade_operators = {
+            'DROP INDEX "uid_category_title_f7fc03"',
             'ALTER TABLE "category" ALTER COLUMN "name" DROP NOT NULL',
             'ALTER TABLE "category" ALTER COLUMN "slug" TYPE VARCHAR(100) USING "slug"::VARCHAR(100)',
             'ALTER TABLE "category" ALTER COLUMN "created_at" TYPE TIMESTAMPTZ USING "created_at"::TIMESTAMPTZ',
@@ -909,6 +943,7 @@ def test_migrate(mocker: MockerFixture):
             'CREATE UNIQUE INDEX "uid_user_usernam_9987ab" ON "user" ("username")',
         }
         expected_downgrade_operators = {
+            'CREATE UNIQUE INDEX "uid_category_title_f7fc03" ON "category" ("title")',
             'ALTER TABLE "category" ALTER COLUMN "name" SET NOT NULL',
             'ALTER TABLE "category" ALTER COLUMN "slug" TYPE VARCHAR(200) USING "slug"::VARCHAR(200)',
             'ALTER TABLE "category" ALTER COLUMN "created_at" TYPE TIMESTAMPTZ USING "created_at"::TIMESTAMPTZ',
@@ -935,11 +970,15 @@ def test_migrate(mocker: MockerFixture):
             'ALTER TABLE "product" ALTER COLUMN "body" TYPE TEXT USING "body"::TEXT',
             'DROP INDEX "idx_product_name_869427"',
             'DROP INDEX "idx_email_email_4a1a33"',
-            'DROP INDEX "idx_user_usernam_9987ab"',
+            'DROP INDEX "uid_user_usernam_9987ab"',
             'DROP INDEX "uid_product_name_869427"',
             'DROP TABLE IF EXISTS "email_user"',
             'DROP TABLE IF EXISTS "newmodel"',
         }
+        if should_add_user_id_column_type_alter_sql():
+            sql = 'ALTER TABLE "category" ALTER COLUMN "user_id" TYPE INT USING "user_id"::INT'
+            expected_upgrade_operators.add(sql)
+            expected_downgrade_operators.add(sql)
         assert not set(Migrate.upgrade_operators).symmetric_difference(expected_upgrade_operators)
         assert not set(Migrate.downgrade_operators).symmetric_difference(
             expected_downgrade_operators