diff --git a/betterproto-extras/.gitignore b/betterproto-extras/.gitignore new file mode 100644 index 0000000..af3ca5e --- /dev/null +++ b/betterproto-extras/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version \ No newline at end of file diff --git a/betterproto-extras/Cargo.lock b/betterproto-extras/Cargo.lock new file mode 100644 index 0000000..6d6ce22 --- /dev/null +++ b/betterproto-extras/Cargo.lock @@ -0,0 +1,383 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "betterproto-extras" +version = "0.1.0" +dependencies = [ + "indoc 2.0.3", + "prost-reflect", + "pyo3", + "thiserror", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "indoc" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-reflect" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b823de344848e011658ac981009100818b322421676740546f8b52ed5249428" +dependencies = [ + "once_cell", + "prost", + "prost-types", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "pyo3" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +dependencies = [ + "cfg-if", + "indoc 1.0.9", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" + +[[package]] +name = "thiserror" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/betterproto-extras/Cargo.toml b/betterproto-extras/Cargo.toml new file mode 100644 index 0000000..205d1af --- /dev/null +++ b/betterproto-extras/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "betterproto-extras" +version = "0.1.0" +edition = "2021" + +[lib] +name = "betterproto_extras" +crate-type = ["cdylib"] + +[dependencies] +indoc = "2.0.3" +prost-reflect = "0.11.5" +pyo3 = { version = "0.19.2", features = ["abi3-py37", "extension-module"] } +thiserror = "1.0.47" diff --git a/betterproto-extras/betterproto_extras.pyi b/betterproto-extras/betterproto_extras.pyi new file mode 100644 index 0000000..1741626 --- /dev/null +++ b/betterproto-extras/betterproto_extras.pyi @@ -0,0 +1,5 @@ +def deserialize(msg, data: bytes): + """ + Parses the binary encoded Protobuf `data` with respect to the metadata + given by the betterproto message `msg`, and merges the result into `msg`. + """ \ No newline at end of file diff --git a/betterproto-extras/pyproject.toml b/betterproto-extras/pyproject.toml new file mode 100644 index 0000000..e73b0e2 --- /dev/null +++ b/betterproto-extras/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["maturin>=1.2,<2.0"] +build-backend = "maturin" + +[project] +name = "betterproto-extras" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/betterproto-extras/src/descriptor_pool.rs b/betterproto-extras/src/descriptor_pool.rs new file mode 100644 index 0000000..88d1dfc --- /dev/null +++ b/betterproto-extras/src/descriptor_pool.rs @@ -0,0 +1,289 @@ +use crate::{ + error::{Error, Result}, + py_any_extras::PyAnyExtras, +}; +use prost_reflect::{ + prost_types::{ + field_descriptor_proto::{Label, Type}, + DescriptorProto, EnumDescriptorProto, EnumValueDescriptorProto, FieldDescriptorProto, + FileDescriptorProto, MessageOptions, OneofDescriptorProto, + }, + DescriptorPool, MessageDescriptor, +}; +use pyo3::PyAny; +use std::sync::{Mutex, OnceLock}; + +pub fn create_cached_descriptor(obj: &PyAny) -> Result { + static DESCRIPTOR_POOL: OnceLock> = OnceLock::new(); + let mut pool = DESCRIPTOR_POOL + .get_or_init(|| Mutex::new(DescriptorPool::global())) + .lock() + .unwrap(); + + let cls = obj.getattr("__class__")?; + let name = format!("{}_{}", cls.qualified_name()?, cls.py_identifier()); + if let Some(desc) = pool.get_message_by_name(&name) { + return Ok(desc); + } + + let mut file = FileDescriptorProto { + name: Some(name.clone()), + ..Default::default() + }; + + add_message_to_file(name.clone(), obj, &pool, &mut file)?; + pool.add_file_descriptor_proto(file)?; + Ok(pool.get_message_by_name(&name).expect("Just registered...")) +} + +fn add_message_to_file( + message_name: String, + obj: &PyAny, + pool: &DescriptorPool, + file: &mut FileDescriptorProto, +) -> Result<()> { + let mut messages_to_add = vec![(message_name, obj)]; + + while let Some((message_name, obj)) = messages_to_add.pop() { + let meta = obj.get_proto_meta()?; + let mut message = DescriptorProto { + name: Some(message_name.to_string()), + ..Default::default() + }; + + for item in meta + .getattr("meta_by_field_name")? + .call_method0("items")? + .iter()? + { + let (field_name, field_meta) = item?.extract::<(&str, &PyAny)>()?; + message.field.push({ + let mut field = FieldDescriptorProto { + name: Some(field_name.to_string()), + number: Some(field_meta.getattr("number")?.extract::()?), + ..Default::default() + }; + let proto_type = field_meta.getattr("proto_type")?.extract::<&str>()?; + + if proto_type == "map" { + field.set_type(Type::Message); + let (key, val) = field_meta.getattr("map_types")?.extract::<(&str, &str)>()?; + let key = map_type(key)?; + let val = map_type(val)?; + + if matches!( + key, + Type::Float | Type::Double | Type::Bytes | Type::Message | Type::Enum + ) { + return Err(Error::UnsupportedMapKeyType(key)); + } + + let map_entry_name = format!("{field_name}Entry"); + field.type_name = Some(format!("{message_name}.{map_entry_name}")); + field.set_label(Label::Repeated); + message.nested_type.push(DescriptorProto { + name: Some(map_entry_name), + field: vec![ + { + let mut proto = FieldDescriptorProto { + name: Some("key".to_string()), + number: Some(1), + ..Default::default() + }; + proto.set_type(key); + proto + }, + { + let mut proto = FieldDescriptorProto { + name: Some("value".to_string()), + number: Some(2), + ..Default::default() + }; + proto.set_type(val); + if val == Type::Message { + set_type_name( + &message_name, + meta.get_class(&format!("{field_name}.value"))?, + &mut proto, + file, + &mut messages_to_add, + pool, + )?; + } + proto + }, + ], + options: Some(MessageOptions { + map_entry: Some(true), + ..Default::default() + }), + ..Default::default() + }) + } else { + field.set_type(map_type(proto_type)?); + match field.r#type() { + Type::Message => match field_meta + .getattr("wraps")? + .extract::>()? + .map(map_type) + .transpose()? + { + Some(Type::Bool) => { + field.type_name = Some("google.protobuf.BoolValue".to_string()); + } + Some(Type::Double) => { + field.type_name = Some("google.protobuf.DoubleValue".to_string()); + } + Some(Type::Float) => { + field.type_name = Some("google.protobuf.FloatValue".to_string()); + } + Some(Type::Int64) => { + field.type_name = Some("google.protobuf.Int64Value".to_string()); + } + Some(Type::Uint64) => { + field.type_name = Some("google.protobuf.UInt64Value".to_string()); + } + Some(Type::Int32) => { + field.type_name = Some("google.protobuf.Int32Value".to_string()); + } + Some(Type::Uint32) => { + field.type_name = Some("google.protobuf.UInt32Value".to_string()); + } + Some(Type::String) => { + field.type_name = Some("google.protobuf.StringValue".to_string()); + } + Some(Type::Bytes) => { + field.type_name = Some("google.protobuf.BytesValue".to_string()); + } + Some(t) => return Err(Error::UnsupportedWrapperType(t)), + None => { + set_type_name( + &message_name, + meta.get_class(field_name)?, + &mut field, + file, + &mut messages_to_add, + pool, + )?; + } + }, + Type::Enum => { + let cls = meta.get_class(field_name)?; + let cls_name = + format!("{}_{}", cls.qualified_name()?, cls.py_identifier()); + field.type_name = Some(cls_name.to_string()); + + if pool.get_enum_by_name(&cls_name).is_none() + && !file.enum_type.iter().any(|item| item.name() == cls_name) + { + let mut proto = EnumDescriptorProto { + name: Some(cls_name.clone()), + ..Default::default() + }; + + for item in cls.iter()? { + let item = item?; + proto.value.push(EnumValueDescriptorProto { + number: Some(item.getattr("value")?.extract()?), + name: Some(format!( + "{}_{}", + cls_name, + item.getattr("name")?.extract::<&str>()? + )), + ..Default::default() + }); + } + + file.enum_type.push(proto); + } + } + _ => {} + } + + if meta.is_list_field(field_name)? { + field.set_label(Label::Repeated); + } else if field_meta.getattr("optional")?.extract::()? { + field.proto3_optional = Some(true); + } + } + + if let Some(grp) = meta.oneof_group(field_name)? { + let oneof_index = message.oneof_decl.iter().position(|x| x.name() == grp); + + match oneof_index { + Some(i) => field.oneof_index = Some(i as i32), + None => { + message.oneof_decl.push(OneofDescriptorProto { + name: Some(grp), + ..Default::default() + }); + field.oneof_index = Some((message.oneof_decl.len() - 1) as i32) + } + } + } + + field + }); + } + + file.message_type.push(message); + } + Ok(()) +} + +fn map_type(str: &str) -> Result { + match str { + "enum" => Ok(Type::Enum), + "bool" => Ok(Type::Bool), + "int32" => Ok(Type::Int32), + "int64" => Ok(Type::Int64), + "uint32" => Ok(Type::Uint32), + "uint64" => Ok(Type::Uint64), + "sint32" => Ok(Type::Sint32), + "sint64" => Ok(Type::Sint64), + "float" => Ok(Type::Float), + "double" => Ok(Type::Double), + "fixed32" => Ok(Type::Fixed32), + "sfixed32" => Ok(Type::Sfixed32), + "fixed64" => Ok(Type::Fixed64), + "sfixed64" => Ok(Type::Sfixed64), + "string" => Ok(Type::String), + "bytes" => Ok(Type::Bytes), + "message" => Ok(Type::Message), + _ => Err(Error::UnsupportedType(str.to_string())), + } +} + +fn set_type_name<'py>( + message_name: &str, + field_cls: &'py PyAny, + field: &mut FieldDescriptorProto, + file: &FileDescriptorProto, + messages_to_add: &mut Vec<(String, &'py PyAny)>, + pool: &DescriptorPool, +) -> Result<()> { + let cls_name = field_cls.qualified_name()?; + + match cls_name.as_str() { + "datetime.datetime" => { + field.type_name = Some("google.protobuf.Timestamp".to_string()); + } + "datetime.timedelta" => { + field.type_name = Some("google.protobuf.Duration".to_string()); + } + _ => { + let cls_name = format!("{}_{}", cls_name, field_cls.py_identifier()); + field.type_name = Some(cls_name.clone()); + + if message_name != cls_name + && pool.get_message_by_name(&cls_name).is_none() + && !file.message_type.iter().any(|item| item.name() == cls_name) + && !messages_to_add.iter().any(|item| item.0 == cls_name) + { + messages_to_add.push((cls_name, field_cls.call0()?)); + } + } + } + + Ok(()) +} diff --git a/betterproto-extras/src/error.rs b/betterproto-extras/src/error.rs new file mode 100644 index 0000000..6cc0885 --- /dev/null +++ b/betterproto-extras/src/error.rs @@ -0,0 +1,29 @@ +use prost_reflect::{ + prost::DecodeError, prost_types::field_descriptor_proto::Type, DescriptorError, +}; +use pyo3::{exceptions::PyRuntimeError, PyErr}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Given object is not a valid betterproto message.")] + NoBetterprotoMessage(#[from] PyErr), + #[error("Unsupported type `{0}`.")] + UnsupportedType(String), + #[error("Unsupported map key type `{0:?}`.")] + UnsupportedMapKeyType(Type), + #[error("Unsupported wrapper type `{0:?}`.")] + UnsupportedWrapperType(Type), + #[error("Error on proto registration")] + FailedToRegisterDescriptor(#[from] DescriptorError), + #[error("The given binary data does not match the protobuf schema.")] + FailedToDecode(#[from] DecodeError), +} + +pub type Result = core::result::Result; + +impl From for PyErr { + fn from(value: Error) -> Self { + PyRuntimeError::new_err(value.to_string()) + } +} diff --git a/betterproto-extras/src/lib.rs b/betterproto-extras/src/lib.rs new file mode 100644 index 0000000..e7a0b39 --- /dev/null +++ b/betterproto-extras/src/lib.rs @@ -0,0 +1,24 @@ +mod descriptor_pool; +mod error; +mod merging; +mod py_any_extras; + +use descriptor_pool::create_cached_descriptor; +use error::Result; +use merging::merge_msg_into_pyobj; +use prost_reflect::DynamicMessage; +use pyo3::prelude::*; + +#[pyfunction] +fn deserialize(obj: &PyAny, buf: &[u8]) -> Result<()> { + let desc = create_cached_descriptor(obj)?; + let msg = DynamicMessage::decode(desc, buf)?; + merge_msg_into_pyobj(obj, msg)?; + Ok(()) +} + +#[pymodule] +fn betterproto_extras(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(deserialize, m)?)?; + Ok(()) +} diff --git a/betterproto-extras/src/merging.rs b/betterproto-extras/src/merging.rs new file mode 100644 index 0000000..a326958 --- /dev/null +++ b/betterproto-extras/src/merging.rs @@ -0,0 +1,182 @@ +use crate::{error::Result, py_any_extras::PyAnyExtras}; +use indoc::indoc; +use prost_reflect::{ + prost_types::{Duration, Timestamp}, + DynamicMessage, MapKey, ReflectMessage, Value, +}; +use pyo3::{ + sync::GILOnceCell, + types::{IntoPyDict, PyBytes, PyModule}, + Py, PyAny, PyObject, Python, ToPyObject, +}; + +pub fn merge_msg_into_pyobj(obj: &PyAny, mut msg: DynamicMessage) -> Result<()> { + for field in msg.take_fields() { + let field_name = field.0.name(); + let proto_meta = obj.get_proto_meta()?; + obj.setattr( + field_name, + map_field_value(field_name, field.1, proto_meta)?, + )?; + } + + let mut buf = vec![]; + for field in msg.unknown_fields() { + field.encode(&mut buf); + } + if !buf.is_empty() { + let mut unknown_fields = obj.getattr("_unknown_fields")?.extract::>()?; + unknown_fields.append(&mut buf); + obj.setattr("_unknown_fields", PyBytes::new(obj.py(), &unknown_fields))?; + } + + obj.setattr("_serialized_on_wire", true)?; + + Ok(()) +} + +fn map_field_value(field_name: &str, field_value: Value, proto_meta: &PyAny) -> Result { + let py = proto_meta.py(); + match field_value { + Value::Bool(x) => Ok(x.to_object(py)), + Value::Bytes(x) => Ok(PyBytes::new(py, &x).to_object(py)), + Value::F32(x) => Ok(x.to_object(py)), + Value::F64(x) => Ok(x.to_object(py)), + Value::I32(x) => Ok(x.to_object(py)), + Value::I64(x) => Ok(x.to_object(py)), + Value::String(x) => Ok(x.to_object(py)), + Value::U32(x) => Ok(x.to_object(py)), + Value::U64(x) => Ok(x.to_object(py)), + Value::Message(msg) => match msg.descriptor().full_name() { + "google.protobuf.BoolValue" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_bool()) + .to_object(py)), + "google.protobuf.DoubleValue" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_f64()) + .to_object(py)), + "google.protobuf.FloatValue" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_f32()) + .to_object(py)), + "google.protobuf.Int64Value" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_i64()) + .to_object(py)), + "google.protobuf.UInt64Value" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_u64()) + .to_object(py)), + "google.protobuf.Int32Value" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_i32()) + .to_object(py)), + "google.protobuf.UInt32Value" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_u32()) + .to_object(py)), + "google.protobuf.StringValue" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_str().map(|s| s.to_string())) + .to_object(py)), + "google.protobuf.BytesValue" => Ok(msg + .get_field_by_number(1) + .and_then(|val| val.as_bytes().map(|b| PyBytes::new(py, b))) + .to_object(py)), + "google.protobuf.Timestamp" => { + let msg = msg.transcode_to::()?; + Ok(create_py_datetime(&msg, py)) + } + "google.protobuf.Duration" => { + let msg = msg.transcode_to::()?; + Ok(create_py_timedelta(&msg, py)) + } + _ => { + let obj = proto_meta.create_instance(field_name)?; + merge_msg_into_pyobj(obj, msg)?; + Ok(obj.to_object(py)) + } + }, + Value::List(ls) => Ok(ls + .into_iter() + .map(|x| map_field_value(field_name, x, proto_meta)) + .collect::>>()? + .to_object(py)), + Value::EnumNumber(x) => { + let cls = proto_meta.get_class(field_name)?; + Ok(cls.call1((x,))?.to_object(py)) + } + Value::Map(map) => { + let res: Result> = map + .into_iter() + .map(|(k, v)| { + let key = map_key(k, py); + let val = map_field_value(&format!("{field_name}.value"), v, proto_meta)?; + Ok((key, val)) + }) + .collect(); + Ok(res?.into_py_dict(py).to_object(py)) + } + } +} + +fn map_key(key: MapKey, py: Python) -> PyObject { + match key { + MapKey::Bool(x) => x.to_object(py), + MapKey::I32(x) => x.to_object(py), + MapKey::I64(x) => x.to_object(py), + MapKey::U32(x) => x.to_object(py), + MapKey::U64(x) => x.to_object(py), + MapKey::String(x) => x.to_object(py), + } +} + +fn create_py_datetime(ts: &Timestamp, py: Python) -> PyObject { + static CONSTRUCTOR_CACHE: GILOnceCell> = GILOnceCell::new(); + let constructor = CONSTRUCTOR_CACHE.get_or_init(py, || { + let constructor = PyModule::from_code( + py, + indoc! {" + from datetime import datetime, timezone + + def constructor(ts): + return datetime.fromtimestamp(ts, tz=timezone.utc) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("constructor") + .expect("Attribute exists"); + Py::from(constructor) + }); + let ts = (ts.seconds as f64) + (ts.nanos as f64) / 1e9; + constructor + .call1(py, (ts,)) + .expect("static function will not fail") +} + +fn create_py_timedelta(duration: &Duration, py: Python) -> PyObject { + static CONSTRUCTOR_CACHE: GILOnceCell> = GILOnceCell::new(); + let constructor = CONSTRUCTOR_CACHE.get_or_init(py, || { + let constructor = PyModule::from_code( + py, + indoc! {" + from datetime import timedelta + + def constructor(s, ms): + return timedelta(seconds=s, microseconds=ms) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("constructor") + .expect("Attribute exists"); + Py::from(constructor) + }); + constructor + .call1(py, (duration.seconds as f64, (duration.nanos as f64) / 1e3)) + .expect("static function will not fail") +} diff --git a/betterproto-extras/src/py_any_extras.rs b/betterproto-extras/src/py_any_extras.rs new file mode 100644 index 0000000..998031b --- /dev/null +++ b/betterproto-extras/src/py_any_extras.rs @@ -0,0 +1,68 @@ +use crate::error::Result; +use pyo3::{PyAny, Py, sync::GILOnceCell}; + +pub trait PyAnyExtras { + fn qualified_name(&self) -> Result; + fn qualified_class_name(&self) -> Result; + fn get_proto_meta(&self) -> Result<&PyAny>; + fn get_class(&self, field_name: &str) -> Result<&PyAny>; + fn create_instance(&self, field_name: &str) -> Result<&PyAny>; + fn is_list_field(&self, field_name: &str) -> Result; + fn oneof_group(&self, field_name: &str) -> Result>; + fn py_identifier(&self) -> u64; +} + +impl PyAnyExtras for PyAny { + fn qualified_name(&self) -> Result { + let module = self.getattr("__module__")?; + let name = self.getattr("__name__")?; + Ok(format!("{module}.{name}")) + } + + fn qualified_class_name(&self) -> Result { + self.getattr("__class__")?.qualified_name() + } + + fn get_proto_meta(&self) -> Result<&PyAny> { + Ok(self.getattr("_betterproto")?) + } + + fn get_class(&self, field_name: &str) -> Result<&PyAny> { + let cls = self.getattr("cls_by_field")?.get_item(field_name)?; + Ok(cls) + } + + fn create_instance(&self, field_name: &str) -> Result<&PyAny> { + Ok(self.get_class(field_name)?.call0()?) + } + + fn is_list_field(&self, field_name: &str) -> Result { + let cls = self.getattr("default_gen")?.get_item(field_name)?; + let module = cls.getattr("__module__")?; + let name = cls.getattr("__name__")?; + Ok(module.to_string() == "builtins" && name.to_string() == "list") + } + + fn oneof_group(&self, field_name: &str) -> Result> { + let opt = self + .getattr("oneof_group_by_field")? + .call_method1("get", (field_name,))? + .extract()?; + Ok(opt) + } + + fn py_identifier(&self) -> u64 { + static FUN_CACHE: GILOnceCell> = GILOnceCell::new(); + let py = self.py(); + let fun = FUN_CACHE.get_or_init(py, || { + let fun = py + .eval("id", None, None) + .expect("This is a valid Python expression"); + Py::from(fun) + }); + fun.call1(py, (self,)) + .expect("Identity function is callable") + .extract::(py) + .expect("Identity function always returns an integer") + } +} diff --git a/example.py b/example.py new file mode 100644 index 0000000..86f1d9b --- /dev/null +++ b/example.py @@ -0,0 +1,55 @@ +# dev tests +# to be deleted later + +import betterproto +from dataclasses import dataclass +from typing import Dict, List, Optional + +@dataclass(repr=False) +class Baz(betterproto.Message): + a: float = betterproto.float_field(1, group = "x") + b: int = betterproto.int64_field(2, group = "x") + c: float = betterproto.float_field(3, group = "y") + d: int = betterproto.int64_field(4, group = "y") + e: Optional[int] = betterproto.int32_field(5, group = "_e", optional = True) + +@dataclass(repr=False) +class Foo(betterproto.Message): + x: int = betterproto.int32_field(1) + y: float = betterproto.double_field(2) + z: List[Baz] = betterproto.message_field(3) + +class Enm(betterproto.Enum): + A = 0 + B = 1 + C = 2 + +@dataclass(repr=False) +class Bar(betterproto.Message): + foo1: Foo = betterproto.message_field(1) + foo2: Foo = betterproto.message_field(2) + packed: List[int] = betterproto.int64_field(3) + enm: Enm = betterproto.enum_field(4) + map: Dict[int, bool] = betterproto.map_field(5, betterproto.TYPE_INT64, betterproto.TYPE_BOOL) + maybe: Optional[bool] = betterproto.message_field(6, wraps=betterproto.TYPE_BOOL) + bts: bytes = betterproto.bytes_field(7) + +# Serialization has not been changed yet. So nothing unusual here +buffer = bytes( + Bar( + foo1=Foo(1, 2.34), + foo2=Foo(3, 4.56, [Baz(a = 1.234), Baz(b = 5, e=1), Baz(b = 2, d = 3)]), + packed=[5, 3, 1], + enm=Enm.B, + map={ + 1: True, + 42: False + }, + maybe=True, + bts=b'Hi There!' + ) +) + +# Native deserialization happening here +bar = Bar().parse(buffer) +print(bar) diff --git a/poetry.lock b/poetry.lock index 00d1966..d3d0838 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "ansicon" version = "1.89.0" description = "Python wrapper for loading Jason Hood's ANSICON" -category = "dev" optional = false python-versions = "*" files = [ @@ -28,7 +26,6 @@ files = [ name = "asv" version = "0.4.2" description = "Airspeed Velocity: A simple Python history benchmarking tool" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -45,7 +42,6 @@ hg = ["python-hglib (>=1.5)"] name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -56,7 +52,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -78,7 +73,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -93,7 +87,6 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "backports-cached-property" version = "1.0.2" description = "cached_property() - computed once per instance, cached as attribute" -category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -101,11 +94,23 @@ files = [ {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, ] +[[package]] +name = "betterproto-extras" +version = "0.1.0" +description = "" +optional = false +python-versions = ">=3.7" +files = [] +develop = false + +[package.source] +type = "directory" +url = "betterproto-extras" + [[package]] name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -156,7 +161,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "blessed" version = "1.20.0" description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -173,7 +177,6 @@ wcwidth = ">=0.1.4" name = "bpython" version = "0.19" description = "Fancy Interface to the Python Interpreter" -category = "dev" optional = false python-versions = "*" files = [ @@ -197,7 +200,6 @@ watch = ["watchdog"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -209,7 +211,6 @@ files = [ name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -221,7 +222,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -306,7 +306,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -322,7 +321,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -334,7 +332,6 @@ files = [ name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -407,7 +404,6 @@ toml = ["tomli"] name = "curtsies" version = "0.4.1" description = "Curses-like terminal wrapper, with colored strings!" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -423,7 +419,6 @@ cwcwidth = "*" name = "cwcwidth" version = "0.1.8" description = "Python bindings for wc(s)width" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -469,7 +464,6 @@ files = [ name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -481,7 +475,6 @@ files = [ name = "docutils" version = "0.20.1" description = "Docutils -- Python Documentation Utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -493,7 +486,6 @@ files = [ name = "filelock" version = "3.12.0" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -509,7 +501,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -583,7 +574,6 @@ test = ["objgraph", "psutil"] name = "grpcio" version = "1.54.2" description = "HTTP/2-based RPC framework" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -641,7 +631,6 @@ protobuf = ["grpcio-tools (>=1.54.2)"] name = "grpcio-tools" version = "1.54.2" description = "Protobuf code generator for gRPC" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -701,7 +690,6 @@ setuptools = "*" name = "grpclib" version = "0.4.4" description = "Pure-Python gRPC implementation for asyncio" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -719,7 +707,6 @@ protobuf = ["protobuf (>=3.15.0)"] name = "h2" version = "4.1.0" description = "HTTP/2 State-Machine based protocol implementation" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -735,7 +722,6 @@ hyperframe = ">=6.0,<7" name = "hpack" version = "4.0.0" description = "Pure-Python HPACK header compression" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -747,7 +733,6 @@ files = [ name = "hyperframe" version = "6.0.1" description = "HTTP/2 framing layer for Python" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -759,7 +744,6 @@ files = [ name = "identify" version = "2.5.24" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -774,7 +758,6 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -786,7 +769,6 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -798,7 +780,6 @@ files = [ name = "importlib-metadata" version = "6.6.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -819,7 +800,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -831,7 +811,6 @@ files = [ name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -849,7 +828,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -867,7 +845,6 @@ i18n = ["Babel (>=2.7)"] name = "jinxed" version = "1.2.0" description = "Jinxed Terminal Library" -category = "dev" optional = false python-versions = "*" files = [ @@ -882,7 +859,6 @@ ansicon = {version = "*", markers = "platform_system == \"Windows\""} name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -942,7 +918,6 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1026,7 +1001,6 @@ files = [ name = "mypy" version = "0.930" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1066,7 +1040,6 @@ python2 = ["typed-ast (>=1.4.0,<2)"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1078,7 +1051,6 @@ files = [ name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1093,7 +1065,6 @@ setuptools = "*" name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1105,7 +1076,6 @@ files = [ name = "pastel" version = "0.2.1" description = "Bring colors to your terminal." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1117,7 +1087,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1129,7 +1098,6 @@ files = [ name = "platformdirs" version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1148,7 +1116,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1167,7 +1134,6 @@ testing = ["pytest", "pytest-benchmark"] name = "poethepoet" version = "0.19.0" description = "A task runner that works well with poetry." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1186,7 +1152,6 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] name = "pre-commit" version = "2.21.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1206,7 +1171,6 @@ virtualenv = ">=20.10.0" name = "protobuf" version = "4.23.2" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1229,7 +1193,6 @@ files = [ name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1241,7 +1204,6 @@ files = [ name = "pydantic" version = "1.10.8" description = "Data validation and settings management using python type hints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1294,7 +1256,6 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1309,7 +1270,6 @@ plugins = ["importlib-metadata"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1335,7 +1295,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "pytest-asyncio" version = "0.12.0" description = "Pytest support for asyncio." -category = "dev" optional = false python-versions = ">= 3.5" files = [ @@ -1352,7 +1311,6 @@ testing = ["async_generator (>=1.3)", "coverage", "hypothesis (>=5.7.1)"] name = "pytest-cov" version = "2.12.1" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1372,7 +1330,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-mock" version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1390,7 +1347,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1405,7 +1361,6 @@ six = ">=1.5" name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "dev" optional = false python-versions = "*" files = [ @@ -1417,7 +1372,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1467,7 +1421,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1489,7 +1442,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "setuptools" version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1506,7 +1458,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1518,7 +1469,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "dev" optional = false python-versions = "*" files = [ @@ -1530,7 +1480,6 @@ files = [ name = "sphinx" version = "3.1.2" description = "Python documentation generator" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1566,7 +1515,6 @@ test = ["cython", "html5lib", "pytest", "pytest-cov", "typed-ast"] name = "sphinx-rtd-theme" version = "0.5.0" description = "Read the Docs theme for Sphinx" -category = "dev" optional = false python-versions = "*" files = [ @@ -1584,7 +1532,6 @@ dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client"] name = "sphinxcontrib-applehelp" version = "1.0.2" description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1600,7 +1547,6 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1616,7 +1562,6 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1632,7 +1577,6 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1647,7 +1591,6 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1663,7 +1606,6 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1679,7 +1621,6 @@ test = ["pytest"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1691,7 +1632,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1703,7 +1643,6 @@ files = [ name = "tomlkit" version = "0.7.2" description = "Style preserving TOML library" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1715,7 +1654,6 @@ files = [ name = "tox" version = "3.28.0" description = "tox is a generic virtualenv management and test command line tool" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -1742,7 +1680,6 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psu name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1776,7 +1713,6 @@ files = [ name = "typing-extensions" version = "4.6.2" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1788,7 +1724,6 @@ files = [ name = "urllib3" version = "2.0.2" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1806,7 +1741,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "virtualenv" version = "20.23.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1828,7 +1762,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -1840,7 +1773,6 @@ files = [ name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1858,4 +1790,4 @@ compiler = ["black", "isort", "jinja2"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "8f733a72705d31633a7f198a7a7dd6e3170876a1ccb8ca75b7d94b6379384a8f" +content-hash = "f3f3c42f9d20b60b53b7b4639dcd6ccfb945c63a55cb7485de72b5a298d0e618" diff --git a/pyproject.toml b/pyproject.toml index 526573d..4183cf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ importlib-metadata = { version = ">=1.6.0", python = "<3.8" } jinja2 = { version = ">=3.0.3", optional = true } python-dateutil = "^2.8" isort = {version = "^5.11.5", optional = true} +betterproto-extras = { path = "betterproto-extras" } [tool.poetry.dev-dependencies] asv = "^0.4.2" diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index dabc082..29819b5 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1330,6 +1330,13 @@ class Message(ABC): :class:`Message` The initialized message. """ + if True: + # TODO: Make native deserialization optional + + import betterproto_extras + betterproto_extras.deserialize(self, data) + return self + with BytesIO(data) as stream: return self.load(stream) diff --git a/src/betterproto/plugin/models.py b/src/betterproto/plugin/models.py index ea819d4..479ecee 100644 --- a/src/betterproto/plugin/models.py +++ b/src/betterproto/plugin/models.py @@ -142,12 +142,12 @@ def monkey_patch_oneof_index(): "betterproto" ], "group", - "oneof_index", + "_oneof_index", ) object.__setattr__( Field.__dataclass_fields__["oneof_index"].metadata["betterproto"], "group", - "oneof_index", + "_oneof_index", ) @@ -385,7 +385,7 @@ def is_oneof(proto_field_obj: FieldDescriptorProto) -> bool: us to tell whether it was set, via the which_one_of interface. """ - return which_one_of(proto_field_obj, "oneof_index")[0] == "oneof_index" + return which_one_of(proto_field_obj, "_oneof_index")[0] == "oneof_index" @dataclass