From 7fe64ad8fe386899ffa5336f0a73370c53af192a Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Fri, 18 Oct 2019 21:07:12 -0700 Subject: [PATCH] Doc updates, refactor code layout, python package --- .gitignore | 3 + MANIFEST.in | 2 + Pipfile | 2 +- README.md | 216 +++++++++++++++++- betterproto/__init__.py | 4 +- .../plugin.py | 25 +- .../templates/{main.py => template.py} | 5 +- betterproto/tests/generate.py | 15 +- setup.py | 20 ++ 9 files changed, 270 insertions(+), 22 deletions(-) create mode 100644 MANIFEST.in rename protoc-gen-betterpy.py => betterproto/plugin.py (94%) rename betterproto/templates/{main.py => template.py} (94%) create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 5d5b945..070814d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ betterproto/tests/*.py !betterproto/tests/generate.py !betterproto/tests/test_*.py **/__pycache__ +dist +**/*.egg-info +output diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0f7f9de --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-exclude tests * +exclude output diff --git a/Pipfile b/Pipfile index aae4234..53a24fc 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,6 @@ grpclib = "*" python_version = "3.7" [scripts] -plugin = "protoc --plugin=protoc-gen-custom=protoc-gen-betterpy.py --custom_out=output" +plugin = "protoc --plugin=protoc-gen-custom=betterproto/plugin.py --custom_out=output" generate = "python betterproto/tests/generate.py" test = "pytest ./betterproto/tests" diff --git a/README.md b/README.md index b71f6f6..24e0e27 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Better Protobuf / gRPC Support for Python -This project aims to provide an improved experience when using Protobuf / gRPC in a modern Python environment by making use of modern language features and generating readable, understandable code. It will not support legacy features or environments. The following are supported: +This project aims to provide an improved experience when using Protobuf / gRPC in a modern Python environment by making use of modern language features and generating readable, understandable, idiomatic Python code. It will not support legacy features or environments. The following are supported: - Protobuf 3 & gRPC code generation - Both binary & JSON serialization is built-in -- Python 3.7+ +- Python 3.7+ making use of: - Enums - Dataclasses - `async`/`await` @@ -17,7 +17,198 @@ This project is heavily inspired by, and borrows functionality from: - https://github.com/eigenein/protobuf/ - https://github.com/vmagamedov/grpclib -## TODO +## Motivation + +This project exists because I am unhappy with the state of the official Google protoc plugin for Python. + +- No `async` support (requires additional `grpclib` plugin) +- No typing support or code completion/intelligence (requires additional `mypy` plugin) +- No `__init__.py` module files get generated +- Output is not importable + - Import paths break in Python 3 unless you mess with `sys.path` +- Bugs when names clash (e.g. `codecs` package) +- Generated code is not idiomatic + - Completely unreadable runtime code-generation + - Much code looks like C++ or Java ported 1:1 to Python + - Capitalized function names like `HasField()` and `SerializeToString()` + - Uses `SerializeToString()` rather than the built-in `__bytes__()` + +This project is a reimplementation from the ground up focused on idiomatic modern Python to help fix some of the above. While it may not be a 1:1 drop-in replacement due to changed method names and call patterns, the wire format is identical. + +## Installation & Getting Started + +First, install the package: + +```sh +$ pip install betterproto +``` + +Now, given a proto file, e.g `example.proto`: + +```protobuf +syntax = "proto3"; + +package hello; + +// Greeting represents a message you can tell a user. +message Greeting { + string message = 1; +} +``` + +You can run the following: + +```sh +$ protoc -I . --python_betterproto_out=. example.proto +``` + +This will generate `hello.py` which looks like: + +```py +# Generated by the protocol buffer compiler. DO NOT EDIT! +# sources: hello.proto +# plugin: python-betterproto +from dataclasses import dataclass + +import betterproto + + +@dataclass +class Hello(betterproto.Message): + """Greeting represents a message you can tell a user.""" + + message: str = betterproto.string_field(1) +``` + +Now you can use it! + +```py +>>> from hello import Hello +>>> test = Hello() +>>> test +Hello(message='') + +>>> test.message = "Hey!" +>>> test +Hello(message="Hey!") + +>>> serialized = bytes(test) +>>> serialized +b'\n\x04Hey!' + +>>> another = Hello().parse(serialized) +>>> another +Hello(message="Hey!") + +>>> another.to_dict() +{"message": "Hey!"} +>>> another.to_json(indent=2) +'{\n "message": "Hey!"\n}' +``` + +### Async gRPC Support + +The generated Protobuf `Message` classes are compatible with [grpclib](https://github.com/vmagamedov/grpclib) so you are free to use it if you like. That said, this project also includes support for async gRPC stub generation with better static type checking and code completion support. It is enabled by default. + +Given an example like: + +```protobuf +syntax = "proto3"; + +package echo; + +message EchoRequest { + string value = 1; + // Number of extra times to echo + uint32 extra_times = 2; +} + +message EchoResponse { + repeated string values = 1; +} + +message EchoStreamResponse { + string value = 1; +} + +service Echo { + rpc Echo(EchoRequest) returns (EchoResponse); + rpc EchoStream(EchoRequest) returns (stream EchoStreamResponse); +} +``` + +You can use it like so (enable async in the interactive shell first): + +```py +>>> import echo +>>> from grpclib.client import Channel + +>>> channel = Channel(host="127.0.0.1", port=1234) +>>> service = echo.EchoStub(channel) +>>> await service.echo(value="hello", extra_times=1) +EchoResponse(values=["hello", "hello"]) + +>>> async for response in service.echo_stream(value="hello", extra_times=1) + print(response) + +EchoStreamResponse(value="hello") +EchoStreamResponse(value="hello") +``` + +### JSON + +Both serializing and parsing are supported to/from JSON and Python dictionaries using the following methods: + +- Dicts: `Message().to_dict()`, `Message().from_dict(...)` +- JSON: `Message().to_json()`, `Message().from_json(...)` + +### Determining if a message was sent + +Sometimes it is useful to be able to determine whether a message has been sent on the wire. This is how the Google wrapper types work to let you know whether a value is unset, set as the default (zero value), or set as something else, for example. + +Use `Message().serialized_on_wire` to determine if it was sent. This is a little bit different from the official Google generated Python code: + +```py +# Old way +>>> mymessage.HasField('myfield') + +# New way +>>> mymessage.myfield.serialized_on_wire +``` + +## Development + +First, make sure you have Python 3.7+ and `pipenv` installed: + +```sh +# Get set up with the virtual env & dependencies +$ pipenv install --dev + +# Link the local package +$ pipenv shell +$ pip install -e . +``` + +### Tests + +There are two types of tests: + +1. Manually-written tests for some behavior of the library +2. Proto files and JSON inputs for automated tests + +For #2, you can add a new `*.proto` file into the `betterproto/tests` directory along with a sample `*.json` input and it will get automatically picked up. + +Here's how to run the tests. + +```sh +# Generate assets from sample .proto files +$ pipenv run generate + +# Run the tests +$ pipenv run tests +``` + +### TODO - [x] Fixed length fields - [x] Packed fixed-length @@ -30,11 +221,26 @@ This project is heavily inspired by, and borrows functionality from: - [ ] Support passthrough of unknown fields - [x] Refs to nested types - [x] Imports in proto files -- [ ] Well-known Google types +- [x] Well-known Google types - [ ] JSON that isn't completely naive. + - [x] 64-bit ints as strings + - [x] Maps + - [x] Lists + - [ ] Bytes as base64 + - [ ] Any support + - [ ] Well known types support (timestamp, duration, wrappers) - [ ] Async service stubs - [x] Unary-unary - [x] Server streaming response - [ ] Client streaming request -- [ ] Python package +- [ ] Renaming messages and fields to conform to Python name standards +- [ ] Renaming clashes with language keywords and standard library top-level packages +- [x] Python package +- [ ] Automate running tests - [ ] Cleanup! + +## License + +Copyright © 2019 Daniel G. Taylor + +http://dgt.mit-license.org/ diff --git a/betterproto/__init__.py b/betterproto/__init__.py index 5e9fa3c..7a1bb3a 100644 --- a/betterproto/__init__.py +++ b/betterproto/__init__.py @@ -633,9 +633,9 @@ class Message(ABC): setattr(self, field.name, v) return self - def to_json(self) -> str: + def to_json(self, indent: Union[None, int, str] = None) -> str: """Returns the encoded JSON representation of this message instance.""" - return json.dumps(self.to_dict()) + return json.dumps(self.to_dict(), indent=indent) def from_json(self: T, value: Union[str, bytes]) -> T: """ diff --git a/protoc-gen-betterpy.py b/betterproto/plugin.py similarity index 94% rename from protoc-gen-betterpy.py rename to betterproto/plugin.py index 2d3e470..c96a1ed 100755 --- a/protoc-gen-betterpy.py +++ b/betterproto/plugin.py @@ -8,7 +8,7 @@ import sys import textwrap from typing import Any, List, Tuple -from jinja2 import Environment, PackageLoader +import jinja2 from google.protobuf.compiler import plugin_pb2 as plugin from google.protobuf.descriptor_pb2 import ( @@ -130,12 +130,12 @@ def get_comment(proto_file, path: List[int]) -> str: def generate_code(request, response): - env = Environment( + env = jinja2.Environment( trim_blocks=True, lstrip_blocks=True, - loader=PackageLoader("betterproto", "templates"), + loader=jinja2.FileSystemLoader("%s/templates/" % os.path.dirname(__file__)), ) - template = env.get_template("main.py") + template = env.get_template("template.py") output_map = {} for proto_file in request.proto_file: @@ -157,6 +157,7 @@ def generate_code(request, response): "package": package, "files": [f.name for f in options["files"]], "imports": set(), + "typing_imports": set(), "messages": [], "enums": [], "services": [], @@ -229,12 +230,14 @@ def generate_code(request, response): f.Type.Name(nested.field[0].type), f.Type.Name(nested.field[1].type), ) + output["typing_imports"].add("Dict") if f.label == 3 and field_type != "map": # Repeated field repeated = True t = f"List[{t}]" zero = "[]" + output["typing_imports"].add("List") if f.type in [1, 2, 3, 4, 5, 6, 7, 8, 13, 15, 16, 17, 18]: packed = True @@ -292,6 +295,9 @@ def generate_code(request, response): for msg in output["messages"]: if msg["name"] == input_type: input_message = msg + for field in msg["properties"]: + if field["zero"] == "None": + output["typing_imports"].add("Optional") break data["methods"].append( @@ -311,9 +317,13 @@ def generate_code(request, response): } ) + if method.server_streaming: + output["typing_imports"].add("AsyncGenerator") + output["services"].append(data) output["imports"] = sorted(output["imports"]) + output["typing_imports"] = sorted(output["typing_imports"]) # Fill response f = response.file.add() @@ -341,7 +351,8 @@ def generate_code(request, response): init.content = b"" -if __name__ == "__main__": +def main(): + """The plugin's main entry point.""" # Read request message from stdin data = sys.stdin.buffer.read() @@ -360,3 +371,7 @@ if __name__ == "__main__": # Write to stdout sys.stdout.buffer.write(output) + + +if __name__ == "__main__": + main() diff --git a/betterproto/templates/main.py b/betterproto/templates/template.py similarity index 94% rename from betterproto/templates/main.py rename to betterproto/templates/template.py index 7782707..ab4a9fa 100644 --- a/betterproto/templates/main.py +++ b/betterproto/templates/template.py @@ -4,7 +4,10 @@ {% if description.enums %}import enum {% endif %} from dataclasses import dataclass -from typing import AsyncGenerator, Dict, List, Optional +{% if description.typing_imports %} +from typing import {% for i in description.typing_imports %}{{ i }}{% if not loop.last %}, {% endif %}{% endfor %} + +{% endif %} import betterproto {% if description.services %} diff --git a/betterproto/tests/generate.py b/betterproto/tests/generate.py index 5fa9e55..49ef870 100644 --- a/betterproto/tests/generate.py +++ b/betterproto/tests/generate.py @@ -1,7 +1,12 @@ #!/usr/bin/env python +import os + +# Force pure-python implementation instead of C++, otherwise imports +# break things because we can't properly reset the symbol database. +os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + import importlib import json -import os # isort: skip import subprocess import sys from typing import Generator, Tuple @@ -10,12 +15,6 @@ from google.protobuf import symbol_database from google.protobuf.descriptor_pool import DescriptorPool from google.protobuf.json_format import MessageToJson, Parse -# Force pure-python implementation instead of C++, otherwise imports -# break things because we can't properly reset the symbol database. -os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" - - - root = os.path.dirname(os.path.realpath(__file__)) @@ -55,7 +54,7 @@ if __name__ == "__main__": f"protoc --python_out=. {os.path.basename(filename)}", shell=True ) subprocess.run( - f"protoc --plugin=protoc-gen-custom=../../protoc-gen-betterpy.py --custom_out=. {os.path.basename(filename)}", + f"protoc --plugin=protoc-gen-custom=../plugin.py --custom_out=. {os.path.basename(filename)}", shell=True, ) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ad1ee29 --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +from setuptools import setup, find_packages + +setup( + name="betterproto", + version="1.0", + description="A better Protobuf / gRPC generator & library", + url="http://github.com/danielgtaylor/python-betterproto", + author="Daniel G. Taylor", + author_email="danielgtaylor@gmail.com", + license="MIT", + entry_points={ + "console_scripts": ["protoc-gen-python_betterproto=betterproto.plugin:main"] + }, + packages=find_packages( + exclude=["tests", "*.tests", "*.tests.*", "output", "output.*"] + ), + package_data={"betterproto": ["py.typed", "templates"]}, + install_requires=["grpclib", "protobuf"], + zip_safe=False, +)