Bump version to 6.8b5

In Django 4.2, `process_lhs()` is called for IsNull lookups,
which is how this bug came to appear. It was already there
and could be reproduced on Django 3.2.

The issue is when `LocalizedRef` is combined with a localized
lookup. The lookup is unaware of the existing localized ref
and adds another transform on top. This results in the
expression becoming:

```
field->'en'->'en'
```

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
     test-python36:
         docker:
             - image: python:3.6-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -11,26 +11,25 @@ jobs:
         steps:
             - checkout
             - run:
-                name: Install packages
-                command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
 
             - run:
-                name: Install Python packages
-                command: pip install --progress-bar off .[test]
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
 
             - run:
-                name: Run tests
-                command: tox -e 'py36-dj{20,21,22,30,31}'
-                environment:
-                    DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+                  name: Run tests
+                  command: tox -e 'py36-dj{20,21,22,30,31,32}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
-                path: reports
-
+                  path: reports
     test-python37:
         docker:
             - image: python:3.7-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -38,26 +37,25 @@ jobs:
         steps:
             - checkout
             - run:
-                name: Install packages
-                command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
 
             - run:
-                name: Install Python packages
-                command: pip install --progress-bar off .[test]
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
 
             - run:
-                name: Run tests
-                command: tox -e 'py37-dj{20,21,22,30,31}'
-                environment:
-                    DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+                  name: Run tests
+                  command: tox -e 'py37-dj{20,21,22,30,31,32}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
-                path: reports
-
+                  path: reports
     test-python38:
         docker:
             - image: python:3.8-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -74,7 +72,7 @@ jobs:
 
             - run:
                 name: Run tests
-                command: tox -e 'py38-dj{20,21,22,30,31}'
+                command: tox -e 'py38-dj{20,21,22,30,31,32,40,41,42}'
                 environment:
                     DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
@@ -84,7 +82,7 @@ jobs:
     test-python39:
         docker:
             - image: python:3.9-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -101,16 +99,69 @@ jobs:
 
             - run:
                 name: Run tests
-                command: tox -e 'py39-dj{21,22,30,31}'
+                command: tox -e 'py39-dj{30,31,32,40,41,42}'
                 environment:
                     DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
                 path: reports
 
+    test-python310:
+        docker:
+            - image: python:3.10-alpine
+            - image: postgres:12.0
+              environment:
+                  POSTGRES_DB: 'localizedfields'
+                  POSTGRES_USER: 'localizedfields'
+                  POSTGRES_PASSWORD: 'localizedfields'
+        steps:
+            - checkout
+            - run:
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+
+            - run:
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
+
+            - run:
+                  name: Run tests
+                  command: tox -e 'py310-dj{32,40,41,42,50}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+
+            - store_test_results:
+                  path: reports
+    test-python311:
+        docker:
+            - image: python:3.11-alpine
+            - image: postgres:12.0
+              environment:
+                  POSTGRES_DB: 'localizedfields'
+                  POSTGRES_USER: 'localizedfields'
+                  POSTGRES_PASSWORD: 'localizedfields'
+        steps:
+            - checkout
+            - run:
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+
+            - run:
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
+
+            - run:
+                  name: Run tests
+                  command: tox -e 'py311-dj{42,50}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+
+            - store_test_results:
+                  path: reports
+
     analysis:
         docker:
-            - image: python:3.7-alpine
+            - image: python:3.8-alpine
         steps:
             - checkout
             - run:
@@ -134,4 +185,6 @@ workflows:
             - test-python37
             - test-python38
             - test-python39
+            - test-python310
+            - test-python311
             - analysis

README.md

@@ -1,13 +1,13 @@
-|  |  |  |
-|--------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|  |  |                                                                                                                                                                             |
+|--------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | :white_check_mark: | **Tests** | [![CircleCI](https://circleci.com/gh/SectorLabs/django-localized-fields/tree/master.svg?style=svg)](https://circleci.com/gh/SectorLabs/django-localized-fields/tree/master) |
-| :memo: | **License** | [![License](https://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org) |
-| :package: | **PyPi** | [![PyPi](https://badge.fury.io/py/django-localized-fields.svg)](https://pypi.python.org/pypi/django-localized-fields) |
-| <img src="https://cdn.iconscout.com/icon/free/png-256/django-1-282754.png" width="22px" height="22px" align="center" /> | **Django Versions** | 2.0, 2.1, 2.2, 3.0, 3.1, 3.2 |
-| <img src="http://www.iconarchive.com/download/i73027/cornmanthe3rd/plex/Other-python.ico" width="22px" height="22px" align="center" /> | **Python Versions** | 3.6, 3.7, 3.8, 3.9 |
-| :book: | **Documentation** | [Read The Docs](https://django-localized-fields.readthedocs.io) |
-| :warning: | **Upgrade** | [Upgrade fom v5.x](https://django-localized-fields.readthedocs.io/en/latest/releases.html#v6-0)
-| :checkered_flag: | **Installation** | [Installation Guide](https://django-localized-fields.readthedocs.io/en/latest/installation.html) |
+| :memo: | **License** | [![License](https://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org)                                                                                     |
+| :package: | **PyPi** | [![PyPi](https://badge.fury.io/py/django-localized-fields.svg)](https://pypi.python.org/pypi/django-localized-fields)                                                       |
+| <img src="https://cdn.iconscout.com/icon/free/png-256/django-1-282754.png" width="22px" height="22px" align="center" /> | **Django Versions** | 2.0, 2.1, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1, 4.2, 5.0                                                                                                                            |
+| <img src="http://www.iconarchive.com/download/i73027/cornmanthe3rd/plex/Other-python.ico" width="22px" height="22px" align="center" /> | **Python Versions** | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11                                                                                                                                              |
+| :book: | **Documentation** | [Read The Docs](https://django-localized-fields.readthedocs.io)                                                                                                             |
+| :warning: | **Upgrade** | [Upgrade fom v5.x](https://django-localized-fields.readthedocs.io/en/latest/releases.html#v6-0)                                                                             
+| :checkered_flag: | **Installation** | [Installation Guide](https://django-localized-fields.readthedocs.io/en/latest/installation.html)                                                                            |
 
 `django-localized-fields` is an implementation of a field class for Django models that allows the field's value to be set in multiple languages. It does this by utilizing the ``hstore`` type (PostgreSQL specific), which is available as `models.HStoreField` since Django 1.10.
 
@@ -20,7 +20,7 @@
 ## Working with the code
 ### Prerequisites
 
-* PostgreSQL 10 or newer.
+* PostgreSQL 12 or newer.
 * Django 2.0 or newer.
 * Python 3.6 or newer.
 

localized_fields/fields/bleach_field.py

@@ -9,6 +9,8 @@ class LocalizedBleachField(LocalizedField):
     """Custom version of :see:BleachField that is actually a
     :see:LocalizedField."""
 
+    DEFAULT_SHOULD_ESCAPE = True
+
     def __init__(self, *args, escape=True, **kwargs):
         """Initializes a new instance of :see:LocalizedBleachField."""
 
@@ -16,6 +18,14 @@ class LocalizedBleachField(LocalizedField):
 
         super().__init__(*args, **kwargs)
 
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+
+        if self.escape != self.DEFAULT_SHOULD_ESCAPE:
+            kwargs["escape"] = self.escape
+
+        return name, path, args, kwargs
+
     def pre_save(self, instance, add: bool):
         """Ran just before the model is saved, allows us to built the slug.
 

localized_fields/fields/field.py

@@ -28,18 +28,26 @@ class LocalizedField(HStoreField):
     descriptor_class = LocalizedValueDescriptor
 
     def __init__(
-        self, *args, required: Union[bool, List[str]] = None, **kwargs
+        self,
+        *args,
+        required: Optional[Union[bool, List[str]]] = None,
+        blank: bool = False,
+        **kwargs
     ):
         """Initializes a new instance of :see:LocalizedField."""
 
-        super(LocalizedField, self).__init__(*args, required=required, **kwargs)
-
-        if (self.required is None and self.blank) or self.required is False:
+        if (required is None and blank) or required is False:
             self.required = []
-        elif self.required is None and not self.blank:
+        elif required is None and not blank:
             self.required = [settings.LANGUAGE_CODE]
-        elif self.required is True:
+        elif required is True:
             self.required = [lang_code for lang_code, _ in settings.LANGUAGES]
+        else:
+            self.required = required
+
+        super(LocalizedField, self).__init__(
+            *args, required=self.required, blank=blank, **kwargs
+        )
 
     def contribute_to_class(self, model, name, **kwargs):
         """Adds this field to the specifed model.

localized_fields/lookups.py

@@ -23,6 +23,7 @@ from django.db.models.lookups import (
     StartsWith,
 )
 from django.utils import translation
+from psqlextra.expressions import HStoreColumn
 
 from .fields import LocalizedField
 
@@ -37,12 +38,35 @@ except ImportError:
 
 class LocalizedLookupMixin:
     def process_lhs(self, qn, connection):
-        if isinstance(self.lhs, Col):
-            language = translation.get_language() or settings.LANGUAGE_CODE
-            self.lhs = KeyTransform(language, self.lhs)
+        # If the LHS is already a reference to a specific hstore key, there
+        # is nothing to be done since it already references as specific language.
+        if isinstance(self.lhs, HStoreColumn) or isinstance(
+            self.lhs, KeyTransform
+        ):
+            return super().process_lhs(qn, connection)
+
+        # If this is something custom expression, we don't really know how to
+        # handle that, so we better do nothing.
+        if not isinstance(self.lhs, Col):
+            return super().process_lhs(qn, connection)
+
+        # Select the key for the current language. We do this so that
+        #
+        # myfield__<lookup>=
+        #
+        # Is converted into:
+        #
+        # myfield__<lookup>__<current language>=
+        language = translation.get_language() or settings.LANGUAGE_CODE
+        self.lhs = KeyTransform(language, self.lhs)
+
         return super().process_lhs(qn, connection)
 
     def get_prep_lookup(self):
+        # Django 4.0 removed the ability for isnull fields to be something
+        # other than a bool We should NOT convert them to strings
+        if isinstance(self.rhs, bool):
+            return self.rhs
         return str(self.rhs)
 
 

localized_fields/static/localized_fields/localized-fields-admin.css

@@ -34,10 +34,11 @@
 
 .localized-fields-widget.tabs .localized-fields-widget.tab label {
     padding: 5px 10px;
-    display: inline-block;
+    display: inline;
     text-decoration: none;
     color: #fff;
     width: initial;
+    cursor: pointer;
 }
 
 .localized-fields-widget.tabs .localized-fields-widget.tab.active,

pytest.ini

@@ -3,3 +3,5 @@ DJANGO_SETTINGS_MODULE=settings
 testpaths=tests
 addopts=-m "not benchmark"
 junit_family=legacy
+filterwarnings=
+    ignore::DeprecationWarning:localized_fields.fields.autoslug_field

settings.py

@@ -1,3 +1,4 @@
+import django
 import dj_database_url
 
 DEBUG = True
@@ -51,6 +52,12 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
 ]
 
+# See: https://github.com/psycopg/psycopg2/issues/1293
+if django.VERSION >= (3, 1):
+    USE_TZ = True
+    USE_I18N = True
+    TIME_ZONE = 'UTC'
+
 # set to a lower number than the default, since
 # we want the tests to be fast, default is 100
 LOCALIZED_FIELDS_MAX_RETRIES = 3

setup.py

@@ -36,7 +36,7 @@ with open(
 
 setup(
     name="django-localized-fields",
-    version="6.8b1",
+    version="6.8b5",
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,
     license="MIT License",
@@ -67,6 +67,8 @@ setup(
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Internet :: WWW/HTTP",
         "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
     ],
@@ -74,20 +76,20 @@ setup(
     install_requires=[
         "Django>=2.0",
         "django-postgres-extra>=2.0,<3.0",
-        "deprecation==2.0.7",
+        "deprecation>=2.0.7",
     ],
     extras_require={
         ':python_version <= "3.6"': ["dataclasses"],
         "docs": ["Sphinx==2.2.0", "sphinx-rtd-theme==0.4.3"],
         "test": [
-            "tox==3.14.3",
-            "pytest==5.3.2",
-            "pytest-django==3.7.0",
-            "pytest-cov==2.8.1",
+            "tox==3.28.0",
+            "pytest==7.0.1",
+            "pytest-django==4.5.2",
+            "pytest-cov==2.12.1",
             "dj-database-url==0.5.0",
-            "django-autoslug==1.9.6",
-            "django-bleach==0.6.1",
-            "psycopg2==2.8.4",
+            "django-autoslug==1.9.9",
+            "django-bleach==0.9.0",
+            "psycopg2==2.9.8",
         ],
         "analysis": [
             "black==22.3.0",
@@ -96,6 +98,7 @@ setup(
             "autopep8==1.4.4",
             "isort==4.3.20",
             "sl-docformatter==1.4",
+            "click==8.0.2",
         ],
     },
     cmdclass={

tests/test_isnull.py

@@ -0,0 +1,51 @@
+import django
+import pytest
+
+from django.test import TestCase
+
+from localized_fields.fields import LocalizedField
+from localized_fields.value import LocalizedValue
+
+from .fake_model import get_fake_model
+
+
+class LocalizedIsNullLookupsTestCase(TestCase):
+    """Tests whether ref lookups properly work with."""
+
+    TestModel1 = None
+
+    @classmethod
+    def setUpClass(cls):
+        """Creates the test model in the database."""
+        super(LocalizedIsNullLookupsTestCase, cls).setUpClass()
+        cls.TestModel = get_fake_model(
+            {"text": LocalizedField(null=True, required=[])}
+        )
+        cls.TestModel.objects.create(
+            text=LocalizedValue(dict(en="text_en", ro="text_ro", nl="text_nl"))
+        )
+        cls.TestModel.objects.create(
+            text=None,
+        )
+
+    def test_isnull_lookup_valid_values(self):
+        """Test whether isnull properly works with valid values."""
+        assert self.TestModel.objects.filter(text__isnull=True).exists()
+        assert self.TestModel.objects.filter(text__isnull=False).exists()
+
+    def test_isnull_lookup_null(self):
+        """Test whether isnull crashes with None as value."""
+
+        with pytest.raises(ValueError):
+            assert self.TestModel.objects.filter(text__isnull=None).exists()
+
+    def test_isnull_lookup_string(self):
+        """Test whether isnull properly works with string values on the
+        corresponding Django version."""
+        if django.VERSION < (4, 0):
+            assert self.TestModel.objects.filter(text__isnull="True").exists()
+        else:
+            with pytest.raises(ValueError):
+                assert self.TestModel.objects.filter(
+                    text__isnull="True"
+                ).exists()

tests/test_lookups.py

@@ -3,6 +3,7 @@ from django.conf import settings
 from django.test import TestCase, override_settings
 from django.utils import translation
 
+from localized_fields.expressions import LocalizedRef
 from localized_fields.fields import LocalizedField
 from localized_fields.value import LocalizedValue
 
@@ -49,6 +50,18 @@ class LocalizedLookupsTestCase(TestCase):
         # ensure that hstore lookups still work
         assert self.TestModel.objects.filter(text__ro="text_ro").exists()
 
+    def test_localized_lookup_specific_isnull(self):
+        self.TestModel.objects.create(
+            text=LocalizedValue(dict(en="text_en", ro="text_ro", nl=None))
+        )
+
+        translation.activate("nl")
+        assert (
+            self.TestModel.objects.annotate(text_localized=LocalizedRef("text"))
+            .filter(text_localized__isnull=True)
+            .exists()
+        )
+
 
 class LocalizedRefLookupsTestCase(TestCase):
     """Tests whether ref lookups properly work with."""

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36-dj{20,21,22,30,31}, py37-dj{20,21,22,30,31}, py38-dj{20,21,22,30,31}, py39-dj{21,22,30,31}
+envlist = py36-dj{20,21,22,30,31,32}, py37-dj{20,21,22,30,31,32}, py38-dj{20,21,22,30,31,32,40,41,42}, py39-dj{30,31,32,40,41,42}, py310-dj{32,40,41,42,50}, py311-dj{42,50}
 
 [testenv]
 deps =
@@ -8,6 +8,11 @@ deps =
     dj22: Django>=2.2,<2.3
     dj30: Django>=3.0,<3.0.2
     dj31: Django>=3.1,<3.2
+    dj32: Django>=3.2,<4.0
+    dj40: Django>=4.0,<4.1
+    dj41: Django>=4.1,<4.2
+    dj42: Django>=4.2,<5.0
+    dj50: Django>=5.0,<5.1
     .[test]
 setenv =
     DJANGO_SETTINGS_MODULE=settings