Merge branch 'master_gh'

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [Ubuntu, MacOS, Windows]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
     steps:
       - uses: actions/checkout@v3
 

.pre-commit-config.yaml

@@ -8,9 +8,10 @@ repos:
       - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
+        args: ["--target-version", "py310"]
 
   - repo: https://github.com/PyCQA/doc8
     rev: 0.10.1

CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Versions suffixed with `b*` are in `beta` and can be installed with `pip install --pre betterproto`.
 
+## [2.0.0b6] - 2023-06-25
+
+- **Breaking**: the minimum Python version has been bumped to `3.7` [#444](https://github.com/danielgtaylor/python-betterproto/pull/444)
+
+- Support generating [Pydantic dataclasses](https://docs.pydantic.dev/latest/usage/dataclasses). 
+  Pydantic dataclasses are are drop-in replacement for dataclasses in the standard library that additionally supports validation.
+  Pass `--python_betterproto_opt=pydantic_dataclasses` to enable this feature.
+  Refer to [#406](https://github.com/danielgtaylor/python-betterproto/pull/406)
+  and [README.md](https://github.com/danielgtaylor/python-betterproto#generating-pydantic-models) for more information.
+  
+- Added support for `@generated` marker [#382](https://github.com/danielgtaylor/python-betterproto/pull/382)
+- Pull down the `include_default_values` argument to `to_json()` [#405](https://github.com/danielgtaylor/python-betterproto/pull/405)
+- Pythonize input_type name in py_input_message [#436](https://github.com/danielgtaylor/python-betterproto/pull/436)
+- Widen `from_dict()` to accept any `Mapping` [#451](https://github.com/danielgtaylor/python-betterproto/pull/451)
+- Replace `pkg_resources` with `importlib` [#462](https://github.com/danielgtaylor/python-betterproto/pull/462)
+  
+- Fix typechecker compatiblity checks in server streaming methods [#413](https://github.com/danielgtaylor/python-betterproto/pull/413)
+- Fix "empty-valued" repeated fields not being serialised [#417](https://github.com/danielgtaylor/python-betterproto/pull/417)
+- Fix `dict` encoding for timezone-aware `datetimes` [#468](https://github.com/danielgtaylor/python-betterproto/pull/468)
+- Fix `to_pydict()` serialization for optional fields [#495](https://github.com/danielgtaylor/python-betterproto/pull/495)
+- Handle empty value objects properly [#481](https://github.com/danielgtaylor/python-betterproto/pull/481)
+  
 ## [2.0.0b5] - 2022-08-01
 
 - **Breaking**: Client and Service Stubs no longer pack and unpack the input message fields as parameters [#331](https://github.com/danielgtaylor/python-betterproto/pull/311)

LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Daniel G. Taylor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "betterproto"
-version = "2.0.0b5"
+version = "2.0.0b6"
 description = "A better Protobuf / gRPC generator & library"
 authors = ["Daniel G. Taylor <danielgtaylor@gmail.com>"]
 readme = "README.md"
@@ -13,7 +13,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.7"
-black = { version = ">=19.3b0", optional = true }
+black = { version = ">=23.1.0", optional = true }
 grpclib = "^0.4.1"
 importlib-metadata = { version = ">=1.6.0", python = "<3.8" }
 jinja2 = { version = ">=3.0.3", optional = true }
@@ -23,11 +23,11 @@ isort = {version = "^5.11.5", optional = true}
 [tool.poetry.dev-dependencies]
 asv = "^0.4.2"
 bpython = "^0.19"
-grpcio-tools = "^1.40.0"
+grpcio-tools = "^1.54.2"
 jinja2 = ">=3.0.3"
 mypy = "^0.930"
 poethepoet = ">=0.9.0"
-protobuf = "^3.12.2"
+protobuf = "^4.21.6"
 pytest = "^6.2.5"
 pytest-asyncio = "^0.12.0"
 pytest-cov = "^2.9.0"
@@ -62,7 +62,7 @@ cmd  = "mypy src --ignore-missing-imports"
 help = "Check types with mypy"
 
 [tool.poe.tasks.format]
-cmd  = "black . --exclude tests/output_"
+cmd  = "black . --exclude tests/output_ --target-version py310"
 help = "Apply black formatting to source code"
 
 [tool.poe.tasks.docs]

src/betterproto/__init__.py

@@ -693,8 +693,28 @@ class Message(ABC):
         def __getattribute__(self, name: str) -> Any:
             """
             Lazily initialize default values to avoid infinite recursion for recursive
-            message types
+            message types.
+            Raise :class:`AttributeError` on attempts to access unset ``oneof`` fields.
             """
+            try:
+                group_current = super().__getattribute__("_group_current")
+            except AttributeError:
+                pass
+            else:
+                if name not in {"__class__", "_betterproto"}:
+                    group = self._betterproto.oneof_group_by_field.get(name)
+                    if group is not None and group_current[group] != name:
+                        if sys.version_info < (3, 10):
+                            raise AttributeError(
+                                f"{group!r} is set to {group_current[group]!r}, not {name!r}"
+                            )
+                        else:
+                            raise AttributeError(
+                                f"{group!r} is set to {group_current[group]!r}, not {name!r}",
+                                name=name,
+                                obj=self,
+                            )
+
             value = super().__getattribute__(name)
             if value is not PLACEHOLDER:
                 return value
@@ -704,6 +724,13 @@ class Message(ABC):
             return value
 
     def __setattr__(self, attr: str, value: Any) -> None:
+        if (
+            isinstance(value, Message)
+            and hasattr(value, "_betterproto")
+            and not value._betterproto.meta_by_field_name
+        ):
+            value._serialized_on_wire = True
+
         if attr != "_serialized_on_wire":
             # Track when a field has been set.
             self.__dict__["_serialized_on_wire"] = True
@@ -754,7 +781,10 @@ class Message(ABC):
         """
         output = bytearray()
         for field_name, meta in self._betterproto.meta_by_field_name.items():
+            try:
                 value = getattr(self, field_name)
+            except AttributeError:
+                continue
 
             if value is None:
                 # Optional items should be skipped. This is used for the Google
@@ -768,9 +798,7 @@ class Message(ABC):
             # Note that proto3 field presence/optional fields are put in a
             # synthetic single-item oneof by protoc, which helps us ensure we
             # send the value even if the value is the default zero value.
-            selected_in_group = (
+            selected_in_group = bool(meta.group)
-                meta.group and self._group_current[meta.group] == field_name
-            )
 
             # Empty messages can still be sent on the wire if they were
             # set (or received empty).
@@ -1009,7 +1037,12 @@ class Message(ABC):
                     parsed.wire_type, meta, field_name, parsed.value
                 )
 
+            try:
                 current = getattr(self, field_name)
+            except AttributeError:
+                current = self._get_field_default(field_name)
+                setattr(self, field_name, current)
+
             if meta.proto_type == TYPE_MAP:
                 # Value represents a single key/value pair entry in the map.
                 current[value.key] = value.value
@@ -1070,7 +1103,10 @@ class Message(ABC):
         defaults = self._betterproto.default_gen
         for field_name, meta in self._betterproto.meta_by_field_name.items():
             field_is_repeated = defaults[field_name] is list
+            try:
                 value = getattr(self, field_name)
+            except AttributeError:
+                value = self._get_field_default(field_name)
             cased_name = casing(field_name).rstrip("_")  # type: ignore
             if meta.proto_type == TYPE_MESSAGE:
                 if isinstance(value, datetime):
@@ -1202,7 +1238,7 @@ class Message(ABC):
 
             if value[key] is not None:
                 if meta.proto_type == TYPE_MESSAGE:
-                    v = getattr(self, field_name)
+                    v = self._get_field_default(field_name)
                     cls = self._betterproto.cls_by_field[field_name]
                     if isinstance(v, list):
                         if cls == datetime:
@@ -1479,7 +1515,6 @@ class Message(ABC):
         field_name_to_meta = cls._betterproto_meta.meta_by_field_name  # type: ignore
 
         for group, field_set in group_to_one_ofs.items():
-
             if len(field_set) == 1:
                 (field,) = field_set
                 field_name = field.name

src/betterproto/grpc/grpclib_server.py

@@ -21,7 +21,6 @@ class ServiceBase(ABC):
         stream: grpclib.server.Stream,
         request: Any,
     ) -> None:
-
         response_iter = handler(request)
         # check if response is actually an AsyncIterator
         # this might be false if the method just returns without

src/betterproto/plugin/compiler.py

@@ -21,7 +21,6 @@ from .models import OutputTemplate
 
 
 def outputfile_compiler(output_file: OutputTemplate) -> str:
-
     templates_folder = os.path.abspath(
         os.path.join(os.path.dirname(__file__), "..", "templates")
     )

src/betterproto/plugin/parser.py

@@ -159,7 +159,6 @@ def _make_one_of_field_compiler(
     proto_obj: "FieldDescriptorProto",
     path: List[int],
 ) -> FieldCompiler:
-
     pydantic = output_package.pydantic_dataclasses
     Cls = PydanticOneOfFieldCompiler if pydantic else OneOfFieldCompiler
     return Cls(

tests/inputs/google_impl_behavior_equivalence/google_impl_behavior_equivalence.proto

@@ -2,9 +2,7 @@ syntax = "proto3";
 
 package google_impl_behavior_equivalence;
 
-message Foo{
+message Foo { int64 bar = 1; }
-  int64 bar = 1;
-}
 
 message Test {
   oneof group {
@@ -13,3 +11,7 @@ message Test{
     Foo foo = 3;
   }
 }
+
+message Request { Empty foo = 1; }
+
+message Empty {}

tests/inputs/google_impl_behavior_equivalence/test_google_impl_behavior_equivalence.py

@@ -3,17 +3,20 @@ from google.protobuf import json_format
 
 import betterproto
 from tests.output_betterproto.google_impl_behavior_equivalence import (
+    Empty,
     Foo,
+    Request,
     Test,
 )
 from tests.output_reference.google_impl_behavior_equivalence.google_impl_behavior_equivalence_pb2 import (
+    Empty as ReferenceEmpty,
     Foo as ReferenceFoo,
+    Request as ReferenceRequest,
     Test as ReferenceTest,
 )
 
 
 def test_oneof_serializes_similar_to_google_oneof():
-
     tests = [
         (Test(string="abc"), ReferenceTest(string="abc")),
         (Test(integer=2), ReferenceTest(integer=2)),
@@ -30,7 +33,6 @@ def test_oneof_serializes_similar_to_google_oneof():
 
 
 def test_bytes_are_the_same_for_oneof():
-
     message = Test(string="")
     message_reference = ReferenceTest(string="")
 
@@ -48,8 +50,23 @@ def test_bytes_are_the_same_for_oneof():
 
     # None of these fields were explicitly set BUT they should not actually be null
     # themselves
-    assert isinstance(message.foo, Foo)
+    assert not hasattr(message, "foo")
-    assert isinstance(message2.foo, Foo)
+    assert object.__getattribute__(message, "foo") == betterproto.PLACEHOLDER
+    assert not hasattr(message2, "foo")
+    assert object.__getattribute__(message2, "foo") == betterproto.PLACEHOLDER
 
     assert isinstance(message_reference.foo, ReferenceFoo)
     assert isinstance(message_reference2.foo, ReferenceFoo)
+
+
+def test_empty_message_field():
+    message = Request()
+    reference_message = ReferenceRequest()
+
+    message.foo = Empty()
+    reference_message.foo.CopyFrom(ReferenceEmpty())
+
+    assert betterproto.serialized_on_wire(message.foo)
+    assert reference_message.HasField("foo")
+
+    assert bytes(message) == reference_message.SerializeToString()

tests/inputs/oneof_enum/test_oneof_enum.py

@@ -18,9 +18,8 @@ def test_which_one_of_returns_enum_with_default_value():
         get_test_case_json_data("oneof_enum", "oneof_enum-enum-0.json")[0].json
     )
 
-    assert message.move == Move(
+    assert not hasattr(message, "move")
-        x=0, y=0
+    assert object.__getattribute__(message, "move") == betterproto.PLACEHOLDER
-    )  # Proto3 will default this as there is no null
     assert message.signal == Signal.PASS
     assert betterproto.which_one_of(message, "action") == ("signal", Signal.PASS)
 
@@ -33,9 +32,8 @@ def test_which_one_of_returns_enum_with_non_default_value():
     message.from_json(
         get_test_case_json_data("oneof_enum", "oneof_enum-enum-1.json")[0].json
     )
-    assert message.move == Move(
+    assert not hasattr(message, "move")
-        x=0, y=0
+    assert object.__getattribute__(message, "move") == betterproto.PLACEHOLDER
-    )  # Proto3 will default this as there is no null
     assert message.signal == Signal.RESIGN
     assert betterproto.which_one_of(message, "action") == ("signal", Signal.RESIGN)
 
@@ -44,5 +42,6 @@ def test_which_one_of_returns_second_field_when_set():
     message = Test()
     message.from_json(get_test_case_json_data("oneof_enum")[0].json)
     assert message.move == Move(x=2, y=3)
-    assert message.signal == Signal.PASS
+    assert not hasattr(message, "signal")
+    assert object.__getattribute__(message, "signal") == betterproto.PLACEHOLDER
     assert betterproto.which_one_of(message, "action") == ("move", Move(x=2, y=3))

tests/oneof_pattern_matching.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass
+
+import pytest
+
+import betterproto
+
+
+def test_oneof_pattern_matching():
+    @dataclass
+    class Sub(betterproto.Message):
+        val: int = betterproto.int32_field(1)
+
+    @dataclass
+    class Foo(betterproto.Message):
+        bar: int = betterproto.int32_field(1, group="group1")
+        baz: str = betterproto.string_field(2, group="group1")
+        sub: Sub = betterproto.message_field(3, group="group2")
+        abc: str = betterproto.string_field(4, group="group2")
+
+    foo = Foo(baz="test1", abc="test2")
+
+    match foo:
+        case Foo(bar=_):
+            pytest.fail("Matched 'bar' instead of 'baz'")
+        case Foo(baz=v):
+            assert v == "test1"
+        case _:
+            pytest.fail("Matched neither 'bar' nor 'baz'")
+
+    match foo:
+        case Foo(sub=_):
+            pytest.fail("Matched 'sub' instead of 'abc'")
+        case Foo(abc=v):
+            assert v == "test2"
+        case _:
+            pytest.fail("Matched neither 'sub' nor 'abc'")
+
+    foo.sub = Sub(val=1)
+
+    match foo:
+        case Foo(sub=Sub(val=v)):
+            assert v == 1
+        case Foo(abc=v):
+            pytest.fail("Matched 'abc' instead of 'sub'")
+        case _:
+            pytest.fail("Matched neither 'sub' nor 'abc'")

tests/test_features.py

@@ -1,4 +1,5 @@
 import json
+import sys
 from copy import (
     copy,
     deepcopy,
@@ -18,6 +19,8 @@ from typing import (
     Optional,
 )
 
+import pytest
+
 import betterproto
 
 
@@ -151,17 +154,18 @@ def test_oneof_support():
     foo.baz = "test"
 
     # Other oneof fields should now be unset
-    assert foo.bar == 0
+    assert not hasattr(foo, "bar")
+    assert object.__getattribute__(foo, "bar") == betterproto.PLACEHOLDER
     assert betterproto.which_one_of(foo, "group1")[0] == "baz"
 
-    foo.sub.val = 1
+    foo.sub = Sub(val=1)
     assert betterproto.serialized_on_wire(foo.sub)
 
     foo.abc = "test"
 
     # Group 1 shouldn't be touched, group 2 should have reset
-    assert foo.sub.val == 0
+    assert not hasattr(foo, "sub")
-    assert betterproto.serialized_on_wire(foo.sub) is False
+    assert object.__getattribute__(foo, "sub") == betterproto.PLACEHOLDER
     assert betterproto.which_one_of(foo, "group2")[0] == "abc"
 
     # Zero value should always serialize for one-of
@@ -176,6 +180,16 @@ def test_oneof_support():
     assert betterproto.which_one_of(foo2, "group2")[0] == ""
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 10),
+    reason="pattern matching is only supported in python3.10+",
+)
+def test_oneof_pattern_matching():
+    from .oneof_pattern_matching import test_oneof_pattern_matching
+
+    test_oneof_pattern_matching()
+
+
 def test_json_casing():
     @dataclass
     class CasingTest(betterproto.Message):