Bump version to 6.8b5

In Django 4.2, `process_lhs()` is called for IsNull lookups,
which is how this bug came to appear. It was already there
and could be reproduced on Django 3.2.

The issue is when `LocalizedRef` is combined with a localized
lookup. The lookup is unaware of the existing localized ref
and adds another transform on top. This results in the
expression becoming:

```
field->'en'->'en'
```

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
     test-python36:
         docker:
             - image: python:3.6-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -11,26 +11,25 @@ jobs:
         steps:
             - checkout
             - run:
-                name: Install packages
-                command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
 
             - run:
-                name: Install Python packages
-                command: pip install --progress-bar off .[test]
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
 
             - run:
-                name: Run tests
-                command: tox -e 'py36-dj{20,21,22,30,31}'
-                environment:
-                    DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+                  name: Run tests
+                  command: tox -e 'py36-dj{20,21,22,30,31,32}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
-                path: reports
-
+                  path: reports
     test-python37:
         docker:
             - image: python:3.7-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -38,26 +37,25 @@ jobs:
         steps:
             - checkout
             - run:
-                name: Install packages
-                command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
 
             - run:
-                name: Install Python packages
-                command: pip install --progress-bar off .[test]
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
 
             - run:
-                name: Run tests
-                command: tox -e 'py37-dj{20,21,22,30,31}'
-                environment:
-                    DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+                  name: Run tests
+                  command: tox -e 'py37-dj{20,21,22,30,31,32}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
-                path: reports
-
+                  path: reports
     test-python38:
         docker:
             - image: python:3.8-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -74,7 +72,7 @@ jobs:
 
             - run:
                 name: Run tests
-                command: tox -e 'py38-dj{20,21,22,30,31}'
+                command: tox -e 'py38-dj{20,21,22,30,31,32,40,41,42}'
                 environment:
                     DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
@@ -84,7 +82,7 @@ jobs:
     test-python39:
         docker:
             - image: python:3.9-alpine
-            - image: postgres:11.0
+            - image: postgres:12.0
               environment:
                   POSTGRES_DB: 'localizedfields'
                   POSTGRES_USER: 'localizedfields'
@@ -101,16 +99,69 @@ jobs:
 
             - run:
                 name: Run tests
-                command: tox -e 'py39-dj{21,22,30,31}'
+                command: tox -e 'py39-dj{30,31,32,40,41,42}'
                 environment:
                     DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
 
             - store_test_results:
                 path: reports
 
+    test-python310:
+        docker:
+            - image: python:3.10-alpine
+            - image: postgres:12.0
+              environment:
+                  POSTGRES_DB: 'localizedfields'
+                  POSTGRES_USER: 'localizedfields'
+                  POSTGRES_PASSWORD: 'localizedfields'
+        steps:
+            - checkout
+            - run:
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+
+            - run:
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
+
+            - run:
+                  name: Run tests
+                  command: tox -e 'py310-dj{32,40,41,42,50}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+
+            - store_test_results:
+                  path: reports
+    test-python311:
+        docker:
+            - image: python:3.11-alpine
+            - image: postgres:12.0
+              environment:
+                  POSTGRES_DB: 'localizedfields'
+                  POSTGRES_USER: 'localizedfields'
+                  POSTGRES_PASSWORD: 'localizedfields'
+        steps:
+            - checkout
+            - run:
+                  name: Install packages
+                  command: apk add postgresql-libs gcc musl-dev postgresql-dev git
+
+            - run:
+                  name: Install Python packages
+                  command: pip install --progress-bar off .[test]
+
+            - run:
+                  name: Run tests
+                  command: tox -e 'py311-dj{42,50}'
+                  environment:
+                      DATABASE_URL: 'postgres://localizedfields:localizedfields@localhost:5432/localizedfields'
+
+            - store_test_results:
+                  path: reports
+
     analysis:
         docker:
-            - image: python:3.7-alpine
+            - image: python:3.8-alpine
         steps:
             - checkout
             - run:
@@ -134,4 +185,6 @@ workflows:
             - test-python37
             - test-python38
             - test-python39
+            - test-python310
+            - test-python311
             - analysis

.gitignore

@@ -15,6 +15,7 @@ reports/
 # Ignore build results
 *.egg-info/
 dist/
+build/
 pip-wheel-metadata
 
 # Ignore stupid .DS_Store

README.md

@@ -1,13 +1,13 @@
-|  |  |  |
-|--------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|  |  |                                                                                                                                                                             |
+|--------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | :white_check_mark: | **Tests** | [![CircleCI](https://circleci.com/gh/SectorLabs/django-localized-fields/tree/master.svg?style=svg)](https://circleci.com/gh/SectorLabs/django-localized-fields/tree/master) |
-| :memo: | **License** | [![License](https://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org) |
-| :package: | **PyPi** | [![PyPi](https://badge.fury.io/py/django-localized-fields.svg)](https://pypi.python.org/pypi/django-localized-fields) |
-| <img src="https://icon-library.net/images/django-icon/django-icon-0.jpg" width="22px" height="22px" align="center" /> | **Django Versions** | 2.0, 2.1, 2.2, 3.0, 3.1 |
-| <img src="http://www.iconarchive.com/download/i73027/cornmanthe3rd/plex/Other-python.ico" width="22px" height="22px" align="center" /> | **Python Versions** | 3.6, 3.7, 3.8, 3.9 |
-| :book: | **Documentation** | [Read The Docs](https://django-localized-fields.readthedocs.io) |
-| :warning: | **Upgrade** | [Upgrade fom v5.x](https://django-localized-fields.readthedocs.io/en/latest/releases.html#v6-0)
-| :checkered_flag: | **Installation** | [Installation Guide](https://django-localized-fields.readthedocs.io/en/latest/installation.html) |
+| :memo: | **License** | [![License](https://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org)                                                                                     |
+| :package: | **PyPi** | [![PyPi](https://badge.fury.io/py/django-localized-fields.svg)](https://pypi.python.org/pypi/django-localized-fields)                                                       |
+| <img src="https://cdn.iconscout.com/icon/free/png-256/django-1-282754.png" width="22px" height="22px" align="center" /> | **Django Versions** | 2.0, 2.1, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1, 4.2, 5.0                                                                                                                            |
+| <img src="http://www.iconarchive.com/download/i73027/cornmanthe3rd/plex/Other-python.ico" width="22px" height="22px" align="center" /> | **Python Versions** | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11                                                                                                                                              |
+| :book: | **Documentation** | [Read The Docs](https://django-localized-fields.readthedocs.io)                                                                                                             |
+| :warning: | **Upgrade** | [Upgrade fom v5.x](https://django-localized-fields.readthedocs.io/en/latest/releases.html#v6-0)                                                                             
+| :checkered_flag: | **Installation** | [Installation Guide](https://django-localized-fields.readthedocs.io/en/latest/installation.html)                                                                            |
 
 `django-localized-fields` is an implementation of a field class for Django models that allows the field's value to be set in multiple languages. It does this by utilizing the ``hstore`` type (PostgreSQL specific), which is available as `models.HStoreField` since Django 1.10.
 
@@ -20,7 +20,7 @@
 ## Working with the code
 ### Prerequisites
 
-* PostgreSQL 10 or newer.
+* PostgreSQL 12 or newer.
 * Django 2.0 or newer.
 * Python 3.6 or newer.
 
@@ -38,7 +38,7 @@
 
 3. Create a postgres user for use in tests (skip if your default user is a postgres superuser):
 
-       λ createuser --superuser psqlextra --pwprompt
+       λ createuser --superuser localized_fields --pwprompt
        λ export DATABASE_URL=postgres://localized_fields:<password>@localhost/localized_fields
 
    Hint: if you're using virtualenvwrapper, you might find it beneficial to put

localized_fields/__init__.py

@@ -1 +1,4 @@
-default_app_config = "localized_fields.apps.LocalizedFieldsConfig"
+import django
+
+if django.VERSION < (3, 2):
+    default_app_config = "localized_fields.apps.LocalizedFieldsConfig"

localized_fields/admin.py

@@ -1,5 +1,6 @@
 from . import widgets
 from .fields import (
+    LocalizedBooleanField,
     LocalizedCharField,
     LocalizedField,
     LocalizedFileField,
@@ -11,6 +12,7 @@ FORMFIELD_FOR_LOCALIZED_FIELDS_DEFAULTS = {
     LocalizedCharField: {"widget": widgets.AdminLocalizedCharFieldWidget},
     LocalizedTextField: {"widget": widgets.AdminLocalizedFieldWidget},
     LocalizedFileField: {"widget": widgets.AdminLocalizedFileFieldWidget},
+    LocalizedBooleanField: {"widget": widgets.AdminLocalizedBooleanFieldWidget},
 }
 
 

localized_fields/fields/__init__.py

@@ -1,4 +1,5 @@
 from .autoslug_field import LocalizedAutoSlugField
+from .boolean_field import LocalizedBooleanField
 from .char_field import LocalizedCharField
 from .field import LocalizedField
 from .file_field import LocalizedFileField
@@ -16,6 +17,7 @@ __all__ = [
     "LocalizedFileField",
     "LocalizedIntegerField",
     "LocalizedFloatField",
+    "LocalizedBooleanField",
 ]
 
 try:

localized_fields/fields/autoslug_field.py

@@ -16,14 +16,14 @@ from .field import LocalizedField
 class LocalizedAutoSlugField(LocalizedField):
     """Automatically provides slugs for a localized field upon saving."""
 
-    warnings.warn(
-        "LocalizedAutoSlug is deprecated and will be removed in the next major version.",
-        DeprecationWarning,
-    )
-
     def __init__(self, *args, **kwargs):
         """Initializes a new instance of :see:LocalizedAutoSlugField."""
 
+        warnings.warn(
+            "LocalizedAutoSlug is deprecated and will be removed in the next major version.",
+            DeprecationWarning,
+        )
+
         self.populate_from = kwargs.pop("populate_from", None)
         self.include_time = kwargs.pop("include_time", False)
 

localized_fields/fields/bleach_field.py

@@ -1,3 +1,5 @@
+import html
+
 from django.conf import settings
 
 from .field import LocalizedField
@@ -7,6 +9,23 @@ class LocalizedBleachField(LocalizedField):
     """Custom version of :see:BleachField that is actually a
     :see:LocalizedField."""
 
+    DEFAULT_SHOULD_ESCAPE = True
+
+    def __init__(self, *args, escape=True, **kwargs):
+        """Initializes a new instance of :see:LocalizedBleachField."""
+
+        self.escape = escape
+
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+
+        if self.escape != self.DEFAULT_SHOULD_ESCAPE:
+            kwargs["escape"] = self.escape
+
+        return name, path, args, kwargs
+
     def pre_save(self, instance, add: bool):
         """Ran just before the model is saved, allows us to built the slug.
 
@@ -42,8 +61,14 @@ class LocalizedBleachField(LocalizedField):
             if not value:
                 continue
 
+            cleaned_value = bleach.clean(
+                value if self.escape else html.unescape(value),
+                **get_bleach_default_options()
+            )
+
             localized_value.set(
-                lang_code, bleach.clean(value, **get_bleach_default_options())
+                lang_code,
+                cleaned_value if self.escape else html.unescape(cleaned_value),
             )
 
         return localized_value

localized_fields/fields/boolean_field.py

@@ -0,0 +1,110 @@
+from typing import Dict, Optional, Union
+
+from django.conf import settings
+from django.db.utils import IntegrityError
+
+from ..forms import LocalizedBooleanFieldForm
+from ..value import LocalizedBooleanValue, LocalizedValue
+from .field import LocalizedField
+
+
+class LocalizedBooleanField(LocalizedField):
+    """Stores booleans as a localized value."""
+
+    attr_class = LocalizedBooleanValue
+
+    @classmethod
+    def from_db_value(cls, value, *_) -> Optional[LocalizedBooleanValue]:
+        db_value = super().from_db_value(value)
+
+        if db_value is None:
+            return db_value
+
+        if isinstance(db_value, str):
+            if db_value.lower() == "true":
+                return True
+            return False
+
+        if not isinstance(db_value, LocalizedValue):
+            return db_value
+
+        return cls._convert_localized_value(db_value)
+
+    def to_python(
+        self, value: Union[Dict[str, str], str, None]
+    ) -> LocalizedBooleanValue:
+        """Converts the value from a database value into a Python value."""
+
+        db_value = super().to_python(value)
+        return self._convert_localized_value(db_value)
+
+    def get_prep_value(self, value: LocalizedBooleanValue) -> dict:
+        """Gets the value in a format to store into the database."""
+
+        # apply default values
+        default_values = LocalizedBooleanValue(self.default)
+        if isinstance(value, LocalizedBooleanValue):
+            for lang_code, _ in settings.LANGUAGES:
+                local_value = value.get(lang_code)
+                if local_value is None:
+                    value.set(lang_code, default_values.get(lang_code, None))
+
+        prepped_value = super().get_prep_value(value)
+        if prepped_value is None:
+            return None
+
+        # make sure all values are proper values to be converted to bool
+        for lang_code, _ in settings.LANGUAGES:
+            local_value = prepped_value[lang_code]
+
+            if local_value is not None and local_value.lower() not in (
+                "false",
+                "true",
+            ):
+                raise IntegrityError(
+                    'non-boolean value in column "%s.%s" violates '
+                    "boolean constraint" % (self.name, lang_code)
+                )
+
+            # convert to a string before saving because the underlying
+            # type is hstore, which only accept strings
+            prepped_value[lang_code] = (
+                str(local_value) if local_value is not None else None
+            )
+
+        return prepped_value
+
+    def formfield(self, **kwargs):
+        """Gets the form field associated with this field."""
+        defaults = {"form_class": LocalizedBooleanFieldForm}
+
+        defaults.update(kwargs)
+        return super().formfield(**defaults)
+
+    @staticmethod
+    def _convert_localized_value(
+        value: LocalizedValue,
+    ) -> LocalizedBooleanValue:
+        """Converts from :see:LocalizedValue to :see:LocalizedBooleanValue."""
+
+        integer_values = {}
+        for lang_code, _ in settings.LANGUAGES:
+            local_value = value.get(lang_code, None)
+
+            if isinstance(local_value, str):
+                if local_value.lower() == "false":
+                    local_value = False
+                elif local_value.lower() == "true":
+                    local_value = True
+                else:
+                    raise ValueError(
+                        f"Could not convert value {local_value} to boolean."
+                    )
+
+                integer_values[lang_code] = local_value
+            elif local_value is not None:
+                raise TypeError(
+                    f"Expected value of type str instead of {type(local_value)}."
+                )
+
+        return LocalizedBooleanValue(integer_values)

localized_fields/fields/field.py

@@ -28,18 +28,26 @@ class LocalizedField(HStoreField):
     descriptor_class = LocalizedValueDescriptor
 
     def __init__(
-        self, *args, required: Union[bool, List[str]] = None, **kwargs
+        self,
+        *args,
+        required: Optional[Union[bool, List[str]]] = None,
+        blank: bool = False,
+        **kwargs
     ):
         """Initializes a new instance of :see:LocalizedField."""
 
-        super(LocalizedField, self).__init__(*args, required=required, **kwargs)
-
-        if (self.required is None and self.blank) or self.required is False:
+        if (required is None and blank) or required is False:
             self.required = []
-        elif self.required is None and not self.blank:
+        elif required is None and not blank:
             self.required = [settings.LANGUAGE_CODE]
-        elif self.required is True:
+        elif required is True:
             self.required = [lang_code for lang_code, _ in settings.LANGUAGES]
+        else:
+            self.required = required
+
+        super(LocalizedField, self).__init__(
+            *args, required=self.required, blank=blank, **kwargs
+        )
 
     def contribute_to_class(self, model, name, **kwargs):
         """Adds this field to the specifed model.

localized_fields/forms.py

@@ -6,12 +6,14 @@ from django.core.exceptions import ValidationError
 from django.forms.widgets import FILE_INPUT_CONTRADICTION
 
 from .value import (
+    LocalizedBooleanValue,
     LocalizedFileValue,
     LocalizedIntegerValue,
     LocalizedStringValue,
     LocalizedValue,
 )
 from .widgets import (
+    AdminLocalizedBooleanFieldWidget,
     AdminLocalizedIntegerFieldWidget,
     LocalizedCharFieldWidget,
     LocalizedFieldWidget,
@@ -102,6 +104,15 @@ class LocalizedIntegerFieldForm(LocalizedFieldForm):
     value_class = LocalizedIntegerValue
 
 
+class LocalizedBooleanFieldForm(LocalizedFieldForm, forms.BooleanField):
+    """Form for a localized boolean field, allows editing the field in multiple
+    languages."""
+
+    widget = AdminLocalizedBooleanFieldWidget
+    field_class = forms.fields.BooleanField
+    value_class = LocalizedBooleanValue
+
+
 class LocalizedFileFieldForm(LocalizedFieldForm, forms.FileField):
     """Form for a localized file field, allows editing the field in multiple
     languages."""

localized_fields/lookups.py

@@ -23,6 +23,7 @@ from django.db.models.lookups import (
     StartsWith,
 )
 from django.utils import translation
+from psqlextra.expressions import HStoreColumn
 
 from .fields import LocalizedField
 
@@ -37,12 +38,35 @@ except ImportError:
 
 class LocalizedLookupMixin:
     def process_lhs(self, qn, connection):
-        if isinstance(self.lhs, Col):
-            language = translation.get_language() or settings.LANGUAGE_CODE
-            self.lhs = KeyTransform(language, self.lhs)
+        # If the LHS is already a reference to a specific hstore key, there
+        # is nothing to be done since it already references as specific language.
+        if isinstance(self.lhs, HStoreColumn) or isinstance(
+            self.lhs, KeyTransform
+        ):
+            return super().process_lhs(qn, connection)
+
+        # If this is something custom expression, we don't really know how to
+        # handle that, so we better do nothing.
+        if not isinstance(self.lhs, Col):
+            return super().process_lhs(qn, connection)
+
+        # Select the key for the current language. We do this so that
+        #
+        # myfield__<lookup>=
+        #
+        # Is converted into:
+        #
+        # myfield__<lookup>__<current language>=
+        language = translation.get_language() or settings.LANGUAGE_CODE
+        self.lhs = KeyTransform(language, self.lhs)
+
         return super().process_lhs(qn, connection)
 
     def get_prep_lookup(self):
+        # Django 4.0 removed the ability for isnull fields to be something
+        # other than a bool We should NOT convert them to strings
+        if isinstance(self.rhs, bool):
+            return self.rhs
         return str(self.rhs)
 
 

localized_fields/static/localized_fields/localized-fields-admin.css

@@ -34,10 +34,11 @@
 
 .localized-fields-widget.tabs .localized-fields-widget.tab label {
     padding: 5px 10px;
-    display: inline-block;
+    display: inline;
     text-decoration: none;
     color: #fff;
     width: initial;
+    cursor: pointer;
 }
 
 .localized-fields-widget.tabs .localized-fields-widget.tab.active,

localized_fields/value.py

@@ -136,6 +136,15 @@ class LocalizedValue(dict):
 
         return None
 
+    def is_empty(self) -> bool:
+        """Gets whether all the languages contain the default value."""
+
+        for lang_code, _ in settings.LANGUAGES:
+            if self.get(lang_code) != self.default_value:
+                return False
+
+        return True
+
     def __str__(self) -> str:
         """Gets the value in the current language or falls back to the next
         language if there's no value in the current language."""
@@ -228,6 +237,35 @@ class LocalizedFileValue(LocalizedValue):
         return self.get(translation.get_language())
 
 
+class LocalizedBooleanValue(LocalizedValue):
+    def translate(self):
+        """Gets the value in the current language, or in the configured fallbck
+        language."""
+
+        value = super().translate()
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            return None
+
+        if isinstance(value, bool):
+            return value
+
+        if value.lower() == "true":
+            return True
+        return False
+
+    def __bool__(self):
+        """Gets the value in the current language as a boolean."""
+        value = self.translate()
+
+        return value
+
+    def __str__(self):
+        """Returns string representation of value."""
+
+        value = self.translate()
+        return str(value) if value is not None else ""
+
+
 class LocalizedNumericValue(LocalizedValue):
     def __int__(self):
         """Gets the value in the current language as an integer."""

localized_fields/widgets.py

@@ -120,6 +120,18 @@ class AdminLocalizedFieldWidget(LocalizedFieldWidget):
     widget = widgets.AdminTextareaWidget
 
 
+class AdminLocalizedBooleanFieldWidget(LocalizedFieldWidget):
+    widget = forms.Select
+
+    def __init__(self, *args, **kwargs):
+        """Initializes a new instance of :see:LocalizedBooleanFieldWidget."""
+
+        super().__init__(*args, **kwargs)
+
+        for widget in self.widgets:
+            widget.choices = [("False", False), ("True", True)]
+
+
 class AdminLocalizedCharFieldWidget(AdminLocalizedFieldWidget):
     widget = widgets.AdminTextInputWidget
 

pytest.ini

@@ -3,3 +3,5 @@ DJANGO_SETTINGS_MODULE=settings
 testpaths=tests
 addopts=-m "not benchmark"
 junit_family=legacy
+filterwarnings=
+    ignore::DeprecationWarning:localized_fields.fields.autoslug_field

settings.py

@@ -1,3 +1,4 @@
+import django
 import dj_database_url
 
 DEBUG = True
@@ -8,7 +9,7 @@ SECRET_KEY = 'this is my secret key'  # NOQA
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 DATABASES = {
-    'default': dj_database_url.config(default='postgres:///psqlextra'),
+    'default': dj_database_url.config(default='postgres:///localized_fields'),
 }
 
 DATABASES['default']['ENGINE'] = 'psqlextra.backend'
@@ -51,6 +52,12 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
 ]
 
+# See: https://github.com/psycopg/psycopg2/issues/1293
+if django.VERSION >= (3, 1):
+    USE_TZ = True
+    USE_I18N = True
+    TIME_ZONE = 'UTC'
+
 # set to a lower number than the default, since
 # we want the tests to be fast, default is 100
 LOCALIZED_FIELDS_MAX_RETRIES = 3

setup.py

@@ -36,7 +36,7 @@ with open(
 
 setup(
     name="django-localized-fields",
-    version="6.3",
+    version="6.8b5",
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,
     license="MIT License",
@@ -64,6 +64,11 @@ setup(
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Internet :: WWW/HTTP",
         "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
     ],
@@ -71,28 +76,29 @@ setup(
     install_requires=[
         "Django>=2.0",
         "django-postgres-extra>=2.0,<3.0",
-        "deprecation==2.0.7",
+        "deprecation>=2.0.7",
     ],
     extras_require={
         ':python_version <= "3.6"': ["dataclasses"],
         "docs": ["Sphinx==2.2.0", "sphinx-rtd-theme==0.4.3"],
         "test": [
-            "tox==3.14.3",
-            "pytest==5.3.2",
-            "pytest-django==3.7.0",
-            "pytest-cov==2.8.1",
+            "tox==3.28.0",
+            "pytest==7.0.1",
+            "pytest-django==4.5.2",
+            "pytest-cov==2.12.1",
             "dj-database-url==0.5.0",
-            "django-autoslug==1.9.6",
-            "django-bleach==0.6.1",
-            "psycopg2==2.8.4",
+            "django-autoslug==1.9.9",
+            "django-bleach==0.9.0",
+            "psycopg2==2.9.8",
         ],
         "analysis": [
-            "black==19.3b0",
+            "black==22.3.0",
             "flake8==3.7.7",
             "autoflake==1.3",
             "autopep8==1.4.4",
             "isort==4.3.20",
             "sl-docformatter==1.4",
+            "click==8.0.2",
         ],
     },
     cmdclass={

tests/test_bleach_field.py

@@ -1,6 +1,7 @@
 """isort:skip_file."""
 
 import sys
+import html
 
 import pytest
 
@@ -66,14 +67,24 @@ class LocalizedBleachFieldTestCase(TestCase):
         bleached_value = field.pre_save(model, False)
         self._validate(value, bleached_value)
 
+    def test_pre_save_do_not_escape(self):
+        """Tests whether the :see:pre_save function works properly when field
+        escape argument is set to False."""
+
+        value = self._get_test_value()
+        model, field = self._get_test_model(value, escape=False)
+
+        bleached_value = field.pre_save(model, False)
+        self._validate(value, bleached_value, False)
+
     @staticmethod
-    def _get_test_model(value):
-        """Gets a test model and a artifically constructed
+    def _get_test_model(value, escape=True):
+        """Gets a test model and an artificially constructed
         :see:LocalizedBleachField instance to test with."""
 
         model = ModelTest(value)
 
-        field = LocalizedBleachField()
+        field = LocalizedBleachField(escape=escape)
         field.attname = "value"
         return model, field
 
@@ -89,7 +100,7 @@ class LocalizedBleachFieldTestCase(TestCase):
         return value
 
     @staticmethod
-    def _validate(non_bleached_value, bleached_value):
+    def _validate(non_bleached_value, bleached_value, escaped_value=True):
         """Validates whether the specified non-bleached value ended up being
         correctly bleached.
 
@@ -100,14 +111,20 @@ class LocalizedBleachFieldTestCase(TestCase):
             bleached_value:
                 The value after bleaching.
         """
-
         for lang_code, _ in settings.LANGUAGES:
             if not non_bleached_value.get(lang_code):
                 assert not bleached_value.get(lang_code)
                 continue
 
-            expected_value = bleach.clean(
-                non_bleached_value.get(lang_code), get_bleach_default_options()
+            cleaned_value = bleach.clean(
+                non_bleached_value.get(lang_code)
+                if escaped_value
+                else html.unescape(non_bleached_value.get(lang_code)),
+                get_bleach_default_options(),
+            )
+
+            expected_value = (
+                cleaned_value if escaped_value else html.unescape(cleaned_value)
             )
 
             assert bleached_value.get(lang_code) == expected_value

tests/test_boolean_field.py

@@ -0,0 +1,211 @@
+from django.conf import settings
+from django.db import connection
+from django.db.utils import IntegrityError
+from django.test import TestCase
+from django.utils import translation
+
+from localized_fields.fields import LocalizedBooleanField
+from localized_fields.value import LocalizedBooleanValue
+
+from .fake_model import get_fake_model
+
+
+class LocalizedBooleanFieldTestCase(TestCase):
+    """Tests whether the :see:LocalizedBooleanField and
+    :see:LocalizedIntegerValue works properly."""
+
+    TestModel = None
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        cls.TestModel = get_fake_model({"translated": LocalizedBooleanField()})
+
+    def test_basic(self):
+        """Tests the basics of storing boolean values."""
+
+        obj = self.TestModel()
+        for lang_code, _ in settings.LANGUAGES:
+            obj.translated.set(lang_code, False)
+        obj.save()
+
+        obj = self.TestModel.objects.all().first()
+        for lang_code, _ in settings.LANGUAGES:
+            assert obj.translated.get(lang_code) is False
+
+    def test_primary_language_required(self):
+        """Tests whether the primary language is required by default and all
+        other languages are optional."""
+
+        # not filling in anything should raise IntegrityError,
+        # the primary language is required
+        with self.assertRaises(IntegrityError):
+            obj = self.TestModel()
+            obj.save()
+
+        # when filling all other languages besides the primary language
+        # should still raise an error because the primary is always required
+        with self.assertRaises(IntegrityError):
+            obj = self.TestModel()
+            for lang_code, _ in settings.LANGUAGES:
+                if lang_code == settings.LANGUAGE_CODE:
+                    continue
+                obj.translated.set(lang_code, True)
+            obj.save()
+
+    def test_default_value_none(self):
+        """Tests whether the default value for optional languages is
+        NoneType."""
+
+        obj = self.TestModel()
+        obj.translated.set(settings.LANGUAGE_CODE, True)
+        obj.save()
+
+        for lang_code, _ in settings.LANGUAGES:
+            if lang_code == settings.LANGUAGE_CODE:
+                continue
+
+            assert obj.translated.get(lang_code) is None
+
+    def test_translate(self):
+        """Tests whether casting the value to a boolean results in the value
+        being returned in the currently active language as a boolean."""
+
+        obj = self.TestModel()
+        for lang_code, _ in settings.LANGUAGES:
+            obj.translated.set(lang_code, True)
+        obj.save()
+
+        obj.refresh_from_db()
+        for lang_code, _ in settings.LANGUAGES:
+            with translation.override(lang_code):
+                assert bool(obj.translated) is True
+                assert obj.translated.translate() is True
+
+    def test_translate_primary_fallback(self):
+        """Tests whether casting the value to a boolean results in the value
+        being returned in the active language and falls back to the primary
+        language if there is no value in that language."""
+
+        obj = self.TestModel()
+        obj.translated.set(settings.LANGUAGE_CODE, True)
+
+        secondary_language = settings.LANGUAGES[-1][0]
+        assert obj.translated.get(secondary_language) is None
+
+        with translation.override(secondary_language):
+            assert obj.translated.translate() is True
+            assert bool(obj.translated) is True
+
+    def test_get_default_value(self):
+        """Tests whether getting the value in a specific language properly
+        returns the specified default in case it is not available."""
+
+        obj = self.TestModel()
+        obj.translated.set(settings.LANGUAGE_CODE, True)
+
+        secondary_language = settings.LANGUAGES[-1][0]
+        assert obj.translated.get(secondary_language) is None
+        assert obj.translated.get(secondary_language, False) is False
+
+    def test_completely_optional(self):
+        """Tests whether having all languages optional works properly."""
+
+        model = get_fake_model(
+            {
+                "translated": LocalizedBooleanField(
+                    null=True, required=[], blank=True
+                )
+            }
+        )
+
+        obj = model()
+        obj.save()
+
+        for lang_code, _ in settings.LANGUAGES:
+            assert getattr(obj.translated, lang_code) is None
+
+    def test_store_string(self):
+        """Tests whether the field properly raises an error when trying to
+        store a non-boolean."""
+
+        for lang_code, _ in settings.LANGUAGES:
+            obj = self.TestModel()
+            with self.assertRaises(IntegrityError):
+                obj.translated.set(lang_code, "haha")
+                obj.save()
+
+    def test_none_if_illegal_value_stored(self):
+        """Tests whether None is returned for a language if the value stored in
+        the database is not a boolean."""
+
+        obj = self.TestModel()
+        obj.translated.set(settings.LANGUAGE_CODE, False)
+        obj.save()
+
+        with connection.cursor() as cursor:
+            table_name = self.TestModel._meta.db_table
+            cursor.execute("update %s set translated = 'en=>haha'" % table_name)
+
+        with self.assertRaises(ValueError):
+            obj.refresh_from_db()
+
+    def test_default_value(self):
+        """Tests whether a default is properly set when specified."""
+
+        model = get_fake_model(
+            {
+                "translated": LocalizedBooleanField(
+                    default={settings.LANGUAGE_CODE: True}
+                )
+            }
+        )
+
+        obj = model.objects.create()
+        assert obj.translated.get(settings.LANGUAGE_CODE) is True
+
+        obj = model()
+        for lang_code, _ in settings.LANGUAGES:
+            obj.translated.set(lang_code, None)
+        obj.save()
+
+        for lang_code, _ in settings.LANGUAGES:
+            if lang_code == settings.LANGUAGE_CODE:
+                assert obj.translated.get(lang_code) is True
+            else:
+                assert obj.translated.get(lang_code) is None
+
+    def test_default_value_update(self):
+        """Tests whether a default is properly set when specified during
+        updates."""
+
+        model = get_fake_model(
+            {
+                "translated": LocalizedBooleanField(
+                    default={settings.LANGUAGE_CODE: True}, null=True
+                )
+            }
+        )
+
+        obj = model.objects.create(
+            translated=LocalizedBooleanValue({settings.LANGUAGE_CODE: False})
+        )
+        assert obj.translated.get(settings.LANGUAGE_CODE) is False
+
+        model.objects.update(
+            translated=LocalizedBooleanValue({settings.LANGUAGE_CODE: None})
+        )
+        obj.refresh_from_db()
+        assert obj.translated.get(settings.LANGUAGE_CODE) is True
+
+    def test_callable_default_value(self):
+        output = {"en": True}
+
+        def func():
+            return output
+
+        model = get_fake_model({"test": LocalizedBooleanField(default=func)})
+        obj = model.objects.create()
+
+        assert obj.test["en"] == output["en"]

tests/test_field.py

@@ -121,8 +121,8 @@ class LocalizedFieldTestCase(TestCase):
 
     @staticmethod
     def test_get_prep_value():
-        """"Tests whether the :see:get_prep_value function produces the
-        expected dictionary."""
+        """Tests whether the :see:get_prep_value function produces the expected
+        dictionary."""
 
         input_data = get_init_values()
         localized_value = LocalizedValue(input_data)

tests/test_isnull.py

@@ -0,0 +1,51 @@
+import django
+import pytest
+
+from django.test import TestCase
+
+from localized_fields.fields import LocalizedField
+from localized_fields.value import LocalizedValue
+
+from .fake_model import get_fake_model
+
+
+class LocalizedIsNullLookupsTestCase(TestCase):
+    """Tests whether ref lookups properly work with."""
+
+    TestModel1 = None
+
+    @classmethod
+    def setUpClass(cls):
+        """Creates the test model in the database."""
+        super(LocalizedIsNullLookupsTestCase, cls).setUpClass()
+        cls.TestModel = get_fake_model(
+            {"text": LocalizedField(null=True, required=[])}
+        )
+        cls.TestModel.objects.create(
+            text=LocalizedValue(dict(en="text_en", ro="text_ro", nl="text_nl"))
+        )
+        cls.TestModel.objects.create(
+            text=None,
+        )
+
+    def test_isnull_lookup_valid_values(self):
+        """Test whether isnull properly works with valid values."""
+        assert self.TestModel.objects.filter(text__isnull=True).exists()
+        assert self.TestModel.objects.filter(text__isnull=False).exists()
+
+    def test_isnull_lookup_null(self):
+        """Test whether isnull crashes with None as value."""
+
+        with pytest.raises(ValueError):
+            assert self.TestModel.objects.filter(text__isnull=None).exists()
+
+    def test_isnull_lookup_string(self):
+        """Test whether isnull properly works with string values on the
+        corresponding Django version."""
+        if django.VERSION < (4, 0):
+            assert self.TestModel.objects.filter(text__isnull="True").exists()
+        else:
+            with pytest.raises(ValueError):
+                assert self.TestModel.objects.filter(
+                    text__isnull="True"
+                ).exists()

tests/test_lookups.py

@@ -3,6 +3,7 @@ from django.conf import settings
 from django.test import TestCase, override_settings
 from django.utils import translation
 
+from localized_fields.expressions import LocalizedRef
 from localized_fields.fields import LocalizedField
 from localized_fields.value import LocalizedValue
 
@@ -49,6 +50,18 @@ class LocalizedLookupsTestCase(TestCase):
         # ensure that hstore lookups still work
         assert self.TestModel.objects.filter(text__ro="text_ro").exists()
 
+    def test_localized_lookup_specific_isnull(self):
+        self.TestModel.objects.create(
+            text=LocalizedValue(dict(en="text_en", ro="text_ro", nl=None))
+        )
+
+        translation.activate("nl")
+        assert (
+            self.TestModel.objects.annotate(text_localized=LocalizedRef("text"))
+            .filter(text_localized__isnull=True)
+            .exists()
+        )
+
 
 class LocalizedRefLookupsTestCase(TestCase):
     """Tests whether ref lookups properly work with."""

tests/test_value.py

@@ -38,6 +38,17 @@ class LocalizedValueTestCase(TestCase):
         for lang_code, _ in settings.LANGUAGES:
             assert getattr(value, lang_code) is None
 
+    @staticmethod
+    def test_is_empty():
+        """Tests whether a newly constructed :see:LocalizedValue without any
+        content is considered "empty"."""
+
+        value = LocalizedValue()
+        assert value.is_empty()
+
+        value.set(settings.LANGUAGE_CODE, "my value")
+        assert not value.is_empty()
+
     @staticmethod
     def test_init_array():
         """Tests whether the __init__ function of :see:LocalizedValue properly

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36-dj{20,21,22,30,31}, py37-dj{20,21,22,30,31}, py38-dj{20,21,22,30,31}, py39-dj{21,22,30,31}
+envlist = py36-dj{20,21,22,30,31,32}, py37-dj{20,21,22,30,31,32}, py38-dj{20,21,22,30,31,32,40,41,42}, py39-dj{30,31,32,40,41,42}, py310-dj{32,40,41,42,50}, py311-dj{42,50}
 
 [testenv]
 deps =
@@ -8,6 +8,11 @@ deps =
     dj22: Django>=2.2,<2.3
     dj30: Django>=3.0,<3.0.2
     dj31: Django>=3.1,<3.2
+    dj32: Django>=3.2,<4.0
+    dj40: Django>=4.0,<4.1
+    dj41: Django>=4.1,<4.2
+    dj42: Django>=4.2,<5.0
+    dj50: Django>=5.0,<5.1
     .[test]
 setenv =
     DJANGO_SETTINGS_MODULE=settings