From d34b16993d73d49ae87849dd3fd881b22621fb70 Mon Sep 17 00:00:00 2001 From: Erik Friese <64597681+124C41p@users.noreply.github.com> Date: Thu, 7 Dec 2023 01:21:29 +0100 Subject: [PATCH] Use external package `betterproto-rust-codec` for better (de-)serialization performance (#545) * optionally use betterproto-rust-codec * monkey patch `parse` and `__bytes__` --- benchmarks/benchmarks.py | 53 +++++++++++++++---------------------- poetry.lock | 23 +++++++++++++++- pyproject.toml | 2 ++ src/betterproto/__init__.py | 17 ++++++++++++ 4 files changed, 62 insertions(+), 33 deletions(-) diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index 07b2360..5768cbf 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -6,32 +6,32 @@ import betterproto @dataclass class TestMessage(betterproto.Message): - foo: int = betterproto.uint32_field(0) - bar: str = betterproto.string_field(1) - baz: float = betterproto.float_field(2) + foo: int = betterproto.uint32_field(1) + bar: str = betterproto.string_field(2) + baz: float = betterproto.float_field(3) @dataclass class TestNestedChildMessage(betterproto.Message): - str_key: str = betterproto.string_field(0) - bytes_key: bytes = betterproto.bytes_field(1) - bool_key: bool = betterproto.bool_field(2) - float_key: float = betterproto.float_field(3) - int_key: int = betterproto.uint64_field(4) + str_key: str = betterproto.string_field(1) + bytes_key: bytes = betterproto.bytes_field(2) + bool_key: bool = betterproto.bool_field(3) + float_key: float = betterproto.float_field(4) + int_key: int = betterproto.uint64_field(5) @dataclass class TestNestedMessage(betterproto.Message): - foo: TestNestedChildMessage = betterproto.message_field(0) - bar: TestNestedChildMessage = betterproto.message_field(1) - baz: TestNestedChildMessage = betterproto.message_field(2) + foo: TestNestedChildMessage = betterproto.message_field(1) + bar: TestNestedChildMessage = betterproto.message_field(2) + baz: TestNestedChildMessage = betterproto.message_field(3) @dataclass class TestRepeatedMessage(betterproto.Message): - foo_repeat: List[str] = betterproto.string_field(0) - bar_repeat: List[int] = betterproto.int64_field(1) - baz_repeat: List[bool] = betterproto.bool_field(2) + foo_repeat: List[str] = betterproto.string_field(1) + bar_repeat: List[int] = betterproto.int64_field(2) + baz_repeat: List[bool] = betterproto.bool_field(3) class BenchMessage: @@ -44,25 +44,14 @@ class BenchMessage: self.instance_filled_bytes = bytes(self.instance_filled) self.instance_filled_nested = TestNestedMessage( TestNestedChildMessage("foo", bytearray(b"test1"), True, 0.1234, 500), - TestNestedChildMessage("bar", bytearray(b"test2"), True, 3.1415, -302), + TestNestedChildMessage("bar", bytearray(b"test2"), True, 3.1415, 302), TestNestedChildMessage("baz", bytearray(b"test3"), False, 1e5, 300), ) self.instance_filled_nested_bytes = bytes(self.instance_filled_nested) self.instance_filled_repeated = TestRepeatedMessage( - [ - "test1", - "test2", - "test3", - "test4", - "test5", - "test6", - "test7", - "test8", - "test9", - "test10", - ], - [2, -100, 0, 500000, 600, -425678, 1000000000, -300, 1, -694214214466], - [True, False, False, False, True, True, False, True, False, False], + [f"test{i}" for i in range(1_000)], + [(i - 500) ** 3 for i in range(1_000)], + [i % 2 == 0 for i in range(1_000)], ) self.instance_filled_repeated_bytes = bytes(self.instance_filled_repeated) @@ -71,9 +60,9 @@ class BenchMessage: @dataclass class Message(betterproto.Message): - foo: int = betterproto.uint32_field(0) - bar: str = betterproto.string_field(1) - baz: float = betterproto.float_field(2) + foo: int = betterproto.uint32_field(1) + bar: str = betterproto.string_field(2) + baz: float = betterproto.float_field(3) def time_instantiation(self): """Time instantiation""" diff --git a/poetry.lock b/poetry.lock index b4af79f..c20859a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,6 +97,26 @@ files = [ {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, ] +[[package]] +name = "betterproto-rust-codec" +version = "0.1.0" +description = "Fast conversion between betterproto messages and Protobuf wire format." +optional = true +python-versions = ">=3.7" +files = [ + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-macosx_10_7_x86_64.whl", hash = "sha256:3e7bea088f5f630cf123b56b1e1da7b35a22a295561825dd7c1446fa841876c3"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:d1706da37a162d72de557ae3e285d04bb66cf129584d380439266e6e0d60c70a"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07aeb3471e9ea3f142f79673b1057810a133c8285aa0a606c53a43a3e28c37da"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b905251be0eae0e7605ecf4a2d60f8c54f3a3c5e6d1d732a0060a18d69cf5e4"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c82353cd2a53f0ced0546d94dd5b18026eed1231a7658e6d5876cc09b1cfdae8"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b881d5a54e8ac7a898fc7f27971e91a6596d344faf342a262cecfa5b1f0dc1a"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fc2d2d11d62a91b75d94783518563673972ad2c99bee9fd16176d3cd4c15d76"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f9b99f776ce1da52f83715b926b61f2ad52fdf1e6d7dec0aea5fb471cd468b1"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-win32.whl", hash = "sha256:2f98155920ff409551208820224ac62564e8dd89d7caed86f8cd519fc7e05568"}, + {file = "betterproto_rust_codec-0.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:8e32f75cf59d5faaa5efb00c268fa22f47def0855c3395e5accccf296b2243be"}, + {file = "betterproto_rust_codec-0.1.0.tar.gz", hash = "sha256:090c23accb23e8f167dabab6fab70d30ae20f587e677c2da4a88982c5fafc5d1"}, +] + [[package]] name = "black" version = "23.3.0" @@ -1904,8 +1924,9 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [extras] compiler = ["black", "isort", "jinja2"] +rust-codec = ["betterproto-rust-codec"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "a522aa8f9c81809b07db9458b493302642836cb300e87efaeed441a451fb0558" +content-hash = "0e0d9fe748a60b22a7147e05c7940ffcad6facb7554cc1ad791e58f4a4c5c22d" diff --git a/pyproject.toml b/pyproject.toml index 8e77da4..dc85cd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ jinja2 = { version = ">=3.0.3", optional = true } python-dateutil = "^2.8" isort = {version = "^5.11.5", optional = true} typing-extensions = "^4.7.1" +betterproto-rust-codec = { version = "0.1.0", optional = true } [tool.poetry.group.dev.dependencies] asv = "^0.4.2" @@ -48,6 +49,7 @@ protoc-gen-python_betterproto = "betterproto.plugin:main" [tool.poetry.extras] compiler = ["black", "isort", "jinja2"] +rust-codec = ["betterproto-rust-codec"] # Dev workflow tasks diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index b2a63d8..da6893d 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1868,6 +1868,23 @@ class Message(ABC): Message.__annotations__ = {} # HACK to avoid typing.get_type_hints breaking :) +# monkey patch (de-)serialization functions of class `Message` +# with functions from `betterproto-rust-codec` if available +try: + import betterproto_rust_codec + + def __parse_patch(self: T, data: bytes) -> T: + betterproto_rust_codec.deserialize(self, data) + return self + + def __bytes_patch(self) -> bytes: + return betterproto_rust_codec.serialize(self) + + Message.parse = __parse_patch + Message.__bytes__ = __bytes_patch +except ModuleNotFoundError: + pass + def serialized_on_wire(message: Message) -> bool: """