From f087c6c9bdb74ab9d421b4bdc1c45d5c3bb678d9 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:19:00 +0200 Subject: [PATCH 01/18] Support compiling google protobuf files --- README.md | 16 ++++++++++++++++ betterproto/plugin.py | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aeb44df..57c380b 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,7 @@ Google provides several well-known message types like a timestamp, duration, and | `google.protobuf.duration` | [`datetime.timedelta`][td] | `0` | | `google.protobuf.timestamp` | Timezone-aware [`datetime.datetime`][dt] | `1970-01-01T00:00:00Z` | | `google.protobuf.*Value` | `Optional[...]` | `None` | +| `google.protobuf.*` | `betterproto.lib.google.protobuf.*` | `None` | [td]: https://docs.python.org/3/library/datetime.html#timedelta-objects [dt]: https://docs.python.org/3/library/datetime.html#datetime.datetime @@ -354,6 +355,21 @@ $ pipenv run generate $ pipenv run test ``` +### (Re)compiling Google Well-known Types + +Betterproto includes compiled versions for Google's well-known types at [betterproto/lib/google](betterproto/lib/google). +Be sure to regenerate these files when modifying the plugin output format, and validate by running the tests. + +Normally, the plugin does not compile any references to `google.protobuf`, since they are pre-compiled. To force compilation of `google.protobuf`, set this environment variable: `PROTOBUF_OPTS=INCLUDE_GOOGLE`. + +Assuming your `google.protobuf` source files (included with all releases of `protoc`) are located in `/usr/local/include`, you can regenerate them as follows: + +```sh +export PROTOBUF_OPTS=INCLUDE_GOOGLE +protoc --plugin=protoc-gen-custom=betterproto/plugin.py --custom_out=betterproto/lib -I/usr/local/include/ /usr/local/include/google/protobuf/*.proto +``` + + ### TODO - [x] Fixed length fields diff --git a/betterproto/plugin.py b/betterproto/plugin.py index c843a6a..3d9c1bb 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -182,6 +182,9 @@ def get_comment(proto_file, path: List[int], indent: int = 4) -> str: def generate_code(request, response): + plugin_options = os.environ.get("BETTERPROTO_OPTS") + plugin_options = plugin_options.split(" ") if plugin_options else [] + env = jinja2.Environment( trim_blocks=True, lstrip_blocks=True, @@ -192,7 +195,8 @@ def generate_code(request, response): output_map = {} for proto_file in request.proto_file: out = proto_file.package - if out == "google.protobuf": + + if out == "google.protobuf" and "INCLUDE_GOOGLE" not in plugin_options: continue if not out: From 4556d67503c1060a45c59f3f45727e4b25e82673 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:21:11 +0200 Subject: [PATCH 02/18] Include pre-compiled google protobuf classes --- betterproto/lib/__init__.py | 0 betterproto/lib/google/__init__.py | 0 betterproto/lib/google/protobuf.py | 1312 ++++++++++++++++++++++++++++ 3 files changed, 1312 insertions(+) create mode 100644 betterproto/lib/__init__.py create mode 100644 betterproto/lib/google/__init__.py create mode 100644 betterproto/lib/google/protobuf.py diff --git a/betterproto/lib/__init__.py b/betterproto/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/betterproto/lib/google/__init__.py b/betterproto/lib/google/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/betterproto/lib/google/protobuf.py b/betterproto/lib/google/protobuf.py new file mode 100644 index 0000000..fd379d5 --- /dev/null +++ b/betterproto/lib/google/protobuf.py @@ -0,0 +1,1312 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# sources: google/protobuf/any.proto, google/protobuf/source_context.proto, google/protobuf/type.proto, google/protobuf/api.proto, google/protobuf/descriptor.proto, google/protobuf/duration.proto, google/protobuf/empty.proto, google/protobuf/field_mask.proto, google/protobuf/struct.proto, google/protobuf/timestamp.proto, google/protobuf/wrappers.proto +# plugin: python-betterproto +from dataclasses import dataclass +from typing import Dict, List + +import betterproto + + +class Syntax(betterproto.Enum): + """The syntax in which a protocol buffer element is defined.""" + + # Syntax `proto2`. + SYNTAX_PROTO2 = 0 + # Syntax `proto3`. + SYNTAX_PROTO3 = 1 + + +class FieldKind(betterproto.Enum): + TYPE_UNKNOWN = 0 + TYPE_DOUBLE = 1 + TYPE_FLOAT = 2 + TYPE_INT64 = 3 + TYPE_UINT64 = 4 + TYPE_INT32 = 5 + TYPE_FIXED64 = 6 + TYPE_FIXED32 = 7 + TYPE_BOOL = 8 + TYPE_STRING = 9 + TYPE_GROUP = 10 + TYPE_MESSAGE = 11 + TYPE_BYTES = 12 + TYPE_UINT32 = 13 + TYPE_ENUM = 14 + TYPE_SFIXED32 = 15 + TYPE_SFIXED64 = 16 + TYPE_SINT32 = 17 + TYPE_SINT64 = 18 + + +class FieldCardinality(betterproto.Enum): + CARDINALITY_UNKNOWN = 0 + CARDINALITY_OPTIONAL = 1 + CARDINALITY_REQUIRED = 2 + CARDINALITY_REPEATED = 3 + + +class FieldDescriptorProtoType(betterproto.Enum): + TYPE_DOUBLE = 1 + TYPE_FLOAT = 2 + TYPE_INT64 = 3 + TYPE_UINT64 = 4 + TYPE_INT32 = 5 + TYPE_FIXED64 = 6 + TYPE_FIXED32 = 7 + TYPE_BOOL = 8 + TYPE_STRING = 9 + TYPE_GROUP = 10 + TYPE_MESSAGE = 11 + TYPE_BYTES = 12 + TYPE_UINT32 = 13 + TYPE_ENUM = 14 + TYPE_SFIXED32 = 15 + TYPE_SFIXED64 = 16 + TYPE_SINT32 = 17 + TYPE_SINT64 = 18 + + +class FieldDescriptorProtoLabel(betterproto.Enum): + LABEL_OPTIONAL = 1 + LABEL_REQUIRED = 2 + LABEL_REPEATED = 3 + + +class FileOptionsOptimizeMode(betterproto.Enum): + SPEED = 1 + CODE_SIZE = 2 + LITE_RUNTIME = 3 + + +class FieldOptionsCType(betterproto.Enum): + STRING = 0 + CORD = 1 + STRING_PIECE = 2 + + +class FieldOptionsJSType(betterproto.Enum): + JS_NORMAL = 0 + JS_STRING = 1 + JS_NUMBER = 2 + + +class MethodOptionsIdempotencyLevel(betterproto.Enum): + IDEMPOTENCY_UNKNOWN = 0 + NO_SIDE_EFFECTS = 1 + IDEMPOTENT = 2 + + +class NullValue(betterproto.Enum): + """ + `NullValue` is a singleton enumeration to represent the null value for the + `Value` type union. The JSON representation for `NullValue` is JSON + `null`. + """ + + # Null value. + NULL_VALUE = 0 + + +@dataclass +class Any(betterproto.Message): + """ + `Any` contains an arbitrary serialized protocol buffer message along with a + URL that describes the type of the serialized message. Protobuf library + provides support to pack/unpack Any values in the form of utility functions + or additional generated methods of the Any type. Example 1: Pack and unpack + a message in C++. Foo foo = ...; Any any; any.PackFrom(foo); + ... if (any.UnpackTo(&foo)) { ... } Example 2: Pack and + unpack a message in Java. Foo foo = ...; Any any = Any.pack(foo); + ... if (any.is(Foo.class)) { foo = any.unpack(Foo.class); } + Example 3: Pack and unpack a message in Python. foo = Foo(...) any + = Any() any.Pack(foo) ... if any.Is(Foo.DESCRIPTOR): + any.Unpack(foo) ... Example 4: Pack and unpack a message in Go + foo := &pb.Foo{...} any, err := ptypes.MarshalAny(foo) ... + foo := &pb.Foo{} if err := ptypes.UnmarshalAny(any, foo); err != nil { + ... } The pack methods provided by protobuf library will by default + use 'type.googleapis.com/full.type.name' as the type URL and the unpack + methods only use the fully qualified type name after the last '/' in the + type URL, for example "foo.bar.com/x/y.z" will yield type name "y.z". JSON + ==== The JSON representation of an `Any` value uses the regular + representation of the deserialized, embedded message, with an additional + field `@type` which contains the type URL. Example: package + google.profile; message Person { string first_name = 1; + string last_name = 2; } { "@type": + "type.googleapis.com/google.profile.Person", "firstName": , + "lastName": } If the embedded message type is well-known and + has a custom JSON representation, that representation will be embedded + adding a field `value` which holds the custom JSON in addition to the + `@type` field. Example (for message [google.protobuf.Duration][]): { + "@type": "type.googleapis.com/google.protobuf.Duration", "value": + "1.212s" } + """ + + # A URL/resource name that uniquely identifies the type of the serialized + # protocol buffer message. This string must contain at least one "/" + # character. The last segment of the URL's path must represent the fully + # qualified name of the type (as in `path/google.protobuf.Duration`). The + # name should be in a canonical form (e.g., leading "." is not accepted). In + # practice, teams usually precompile into the binary all types that they + # expect it to use in the context of Any. However, for URLs which use the + # scheme `http`, `https`, or no scheme, one can optionally set up a type + # server that maps type URLs to message definitions as follows: * If no + # scheme is provided, `https` is assumed. * An HTTP GET on the URL must yield + # a [google.protobuf.Type][] value in binary format, or produce an error. * + # Applications are allowed to cache lookup results based on the URL, or + # have them precompiled into a binary to avoid any lookup. Therefore, + # binary compatibility needs to be preserved on changes to types. (Use + # versioned type names to manage breaking changes.) Note: this + # functionality is not currently available in the official protobuf release, + # and it is not used for type URLs beginning with type.googleapis.com. + # Schemes other than `http`, `https` (or the empty scheme) might be used with + # implementation specific semantics. + type_url: str = betterproto.string_field(1) + # Must be a valid serialized protocol buffer of the above specified type. + value: bytes = betterproto.bytes_field(2) + + +@dataclass +class SourceContext(betterproto.Message): + """ + `SourceContext` represents information about the source of a protobuf + element, like the file in which it is defined. + """ + + # The path-qualified name of the .proto file that contained the associated + # protobuf element. For example: `"google/protobuf/source_context.proto"`. + file_name: str = betterproto.string_field(1) + + +@dataclass +class Type(betterproto.Message): + """A protocol buffer message type.""" + + # The fully qualified message name. + name: str = betterproto.string_field(1) + # The list of fields. + fields: List["Field"] = betterproto.message_field(2) + # The list of types appearing in `oneof` definitions in this type. + oneofs: List[str] = betterproto.string_field(3) + # The protocol buffer options. + options: List["Option"] = betterproto.message_field(4) + # The source context. + source_context: "SourceContext" = betterproto.message_field(5) + # The source syntax. + syntax: "Syntax" = betterproto.enum_field(6) + + +@dataclass +class Field(betterproto.Message): + """A single field of a message type.""" + + # The field type. + kind: "FieldKind" = betterproto.enum_field(1) + # The field cardinality. + cardinality: "FieldCardinality" = betterproto.enum_field(2) + # The field number. + number: int = betterproto.int32_field(3) + # The field name. + name: str = betterproto.string_field(4) + # The field type URL, without the scheme, for message or enumeration types. + # Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + type_url: str = betterproto.string_field(6) + # The index of the field type in `Type.oneofs`, for message or enumeration + # types. The first type has index 1; zero means the type is not in the list. + oneof_index: int = betterproto.int32_field(7) + # Whether to use alternative packed wire representation. + packed: bool = betterproto.bool_field(8) + # The protocol buffer options. + options: List["Option"] = betterproto.message_field(9) + # The field JSON name. + json_name: str = betterproto.string_field(10) + # The string value of the default value of this field. Proto2 syntax only. + default_value: str = betterproto.string_field(11) + + +@dataclass +class Enum(betterproto.Message): + """Enum type definition.""" + + # Enum type name. + name: str = betterproto.string_field(1) + # Enum value definitions. + enumvalue: List["EnumValue"] = betterproto.message_field( + 2, wraps=betterproto.TYPE_ENUM + ) + # Protocol buffer options. + options: List["Option"] = betterproto.message_field(3) + # The source context. + source_context: "SourceContext" = betterproto.message_field(4) + # The source syntax. + syntax: "Syntax" = betterproto.enum_field(5) + + +@dataclass +class EnumValue(betterproto.Message): + """Enum value definition.""" + + # Enum value name. + name: str = betterproto.string_field(1) + # Enum value number. + number: int = betterproto.int32_field(2) + # Protocol buffer options. + options: List["Option"] = betterproto.message_field(3) + + +@dataclass +class Option(betterproto.Message): + """ + A protocol buffer option, which can be attached to a message, field, + enumeration, etc. + """ + + # The option's name. For protobuf built-in options (options defined in + # descriptor.proto), this is the short name. For example, `"map_entry"`. For + # custom options, it should be the fully-qualified name. For example, + # `"google.api.http"`. + name: str = betterproto.string_field(1) + # The option's value packed in an Any message. If the value is a primitive, + # the corresponding wrapper type defined in google/protobuf/wrappers.proto + # should be used. If the value is an enum, it should be stored as an int32 + # value using the google.protobuf.Int32Value type. + value: "Any" = betterproto.message_field(2) + + +@dataclass +class Api(betterproto.Message): + """ + Api is a light-weight descriptor for an API Interface. Interfaces are also + described as "protocol buffer services" in some contexts, such as by the + "service" keyword in a .proto file, but they are different from API + Services, which represent a concrete implementation of an interface as + opposed to simply a description of methods and bindings. They are also + sometimes simply referred to as "APIs" in other contexts, such as the name + of this message itself. See https://cloud.google.com/apis/design/glossary + for detailed terminology. + """ + + # The fully qualified name of this interface, including package name followed + # by the interface's simple name. + name: str = betterproto.string_field(1) + # The methods of this interface, in unspecified order. + methods: List["Method"] = betterproto.message_field(2) + # Any metadata attached to the interface. + options: List["Option"] = betterproto.message_field(3) + # A version string for this interface. If specified, must have the form + # `major-version.minor-version`, as in `1.10`. If the minor version is + # omitted, it defaults to zero. If the entire version field is empty, the + # major version is derived from the package name, as outlined below. If the + # field is not empty, the version in the package name will be verified to be + # consistent with what is provided here. The versioning schema uses [semantic + # versioning](http://semver.org) where the major version number indicates a + # breaking change and the minor version an additive, non-breaking change. + # Both version numbers are signals to users what to expect from different + # versions, and should be carefully chosen based on the product plan. The + # major version is also reflected in the package name of the interface, which + # must end in `v`, as in `google.feature.v1`. For major + # versions 0 and 1, the suffix can be omitted. Zero major versions must only + # be used for experimental, non-GA interfaces. + version: str = betterproto.string_field(4) + # Source context for the protocol buffer service represented by this message. + source_context: "SourceContext" = betterproto.message_field(5) + # Included interfaces. See [Mixin][]. + mixins: List["Mixin"] = betterproto.message_field(6) + # The source syntax of the service. + syntax: "Syntax" = betterproto.enum_field(7) + + +@dataclass +class Method(betterproto.Message): + """Method represents a method of an API interface.""" + + # The simple name of this method. + name: str = betterproto.string_field(1) + # A URL of the input message type. + request_type_url: str = betterproto.string_field(2) + # If true, the request is streamed. + request_streaming: bool = betterproto.bool_field(3) + # The URL of the output message type. + response_type_url: str = betterproto.string_field(4) + # If true, the response is streamed. + response_streaming: bool = betterproto.bool_field(5) + # Any metadata attached to the method. + options: List["Option"] = betterproto.message_field(6) + # The source syntax of this method. + syntax: "Syntax" = betterproto.enum_field(7) + + +@dataclass +class Mixin(betterproto.Message): + """ + Declares an API Interface to be included in this interface. The including + interface must redeclare all the methods from the included interface, but + documentation and options are inherited as follows: - If after comment and + whitespace stripping, the documentation string of the redeclared method + is empty, it will be inherited from the original method. - Each + annotation belonging to the service config (http, visibility) which is + not set in the redeclared method will be inherited. - If an http + annotation is inherited, the path pattern will be modified as follows. + Any version prefix will be replaced by the version of the including + interface plus the [root][] path if specified. Example of a simple mixin: + package google.acl.v1; service AccessControl { // Get the + underlying ACL object. rpc GetAcl(GetAclRequest) returns (Acl) { + option (google.api.http).get = "/v1/{resource=**}:getAcl"; } } + package google.storage.v2; service Storage { rpc + GetAcl(GetAclRequest) returns (Acl); // Get a data record. rpc + GetData(GetDataRequest) returns (Data) { option + (google.api.http).get = "/v2/{resource=**}"; } } Example of a + mixin configuration: apis: - name: google.storage.v2.Storage + mixins: - name: google.acl.v1.AccessControl The mixin construct + implies that all methods in `AccessControl` are also declared with same + name and request/response types in `Storage`. A documentation generator or + annotation processor will see the effective `Storage.GetAcl` method after + inherting documentation and annotations as follows: service Storage { + // Get the underlying ACL object. rpc GetAcl(GetAclRequest) returns + (Acl) { option (google.api.http).get = "/v2/{resource=**}:getAcl"; + } ... } Note how the version in the path pattern changed from + `v1` to `v2`. If the `root` field in the mixin is specified, it should be a + relative path under which inherited HTTP paths are placed. Example: + apis: - name: google.storage.v2.Storage mixins: - name: + google.acl.v1.AccessControl root: acls This implies the following + inherited HTTP annotation: service Storage { // Get the + underlying ACL object. rpc GetAcl(GetAclRequest) returns (Acl) { + option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; } + ... } + """ + + # The fully qualified name of the interface which is included. + name: str = betterproto.string_field(1) + # If non-empty specifies a path under which inherited HTTP paths are rooted. + root: str = betterproto.string_field(2) + + +@dataclass +class FileDescriptorSet(betterproto.Message): + """ + The protocol compiler can output a FileDescriptorSet containing the .proto + files it parses. + """ + + file: List["FileDescriptorProto"] = betterproto.message_field(1) + + +@dataclass +class FileDescriptorProto(betterproto.Message): + """Describes a complete .proto file.""" + + name: str = betterproto.string_field(1) + package: str = betterproto.string_field(2) + # Names of files imported by this file. + dependency: List[str] = betterproto.string_field(3) + # Indexes of the public imported files in the dependency list above. + public_dependency: List[int] = betterproto.int32_field(10) + # Indexes of the weak imported files in the dependency list. For Google- + # internal migration only. Do not use. + weak_dependency: List[int] = betterproto.int32_field(11) + # All top-level definitions in this file. + message_type: List["DescriptorProto"] = betterproto.message_field(4) + enum_type: List["EnumDescriptorProto"] = betterproto.message_field(5) + service: List["ServiceDescriptorProto"] = betterproto.message_field(6) + extension: List["FieldDescriptorProto"] = betterproto.message_field(7) + options: "FileOptions" = betterproto.message_field(8) + # This field contains optional information about the original source code. + # You may safely remove this entire field without harming runtime + # functionality of the descriptors -- the information is needed only by + # development tools. + source_code_info: "SourceCodeInfo" = betterproto.message_field(9) + # The syntax of the proto file. The supported values are "proto2" and + # "proto3". + syntax: str = betterproto.string_field(12) + + +@dataclass +class DescriptorProto(betterproto.Message): + """Describes a message type.""" + + name: str = betterproto.string_field(1) + field: List["FieldDescriptorProto"] = betterproto.message_field(2) + extension: List["FieldDescriptorProto"] = betterproto.message_field(6) + nested_type: List["DescriptorProto"] = betterproto.message_field(3) + enum_type: List["EnumDescriptorProto"] = betterproto.message_field(4) + extension_range: List["DescriptorProtoExtensionRange"] = betterproto.message_field( + 5 + ) + oneof_decl: List["OneofDescriptorProto"] = betterproto.message_field(8) + options: "MessageOptions" = betterproto.message_field(7) + reserved_range: List["DescriptorProtoReservedRange"] = betterproto.message_field(9) + # Reserved field names, which may not be used by fields in the same message. + # A given name may only be reserved once. + reserved_name: List[str] = betterproto.string_field(10) + + +@dataclass +class DescriptorProtoExtensionRange(betterproto.Message): + start: int = betterproto.int32_field(1) + end: int = betterproto.int32_field(2) + options: "ExtensionRangeOptions" = betterproto.message_field(3) + + +@dataclass +class DescriptorProtoReservedRange(betterproto.Message): + """ + Range of reserved tag numbers. Reserved tag numbers may not be used by + fields or extension ranges in the same message. Reserved ranges may not + overlap. + """ + + start: int = betterproto.int32_field(1) + end: int = betterproto.int32_field(2) + + +@dataclass +class ExtensionRangeOptions(betterproto.Message): + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class FieldDescriptorProto(betterproto.Message): + """Describes a field within a message.""" + + name: str = betterproto.string_field(1) + number: int = betterproto.int32_field(3) + label: "FieldDescriptorProtoLabel" = betterproto.enum_field(4) + # If type_name is set, this need not be set. If both this and type_name are + # set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + type: "FieldDescriptorProtoType" = betterproto.enum_field(5) + # For message and enum types, this is the name of the type. If the name + # starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + # rules are used to find the type (i.e. first the nested types within this + # message are searched, then within the parent, on up to the root namespace). + type_name: str = betterproto.string_field(6) + # For extensions, this is the name of the type being extended. It is + # resolved in the same manner as type_name. + extendee: str = betterproto.string_field(2) + # For numeric types, contains the original text representation of the value. + # For booleans, "true" or "false". For strings, contains the default text + # contents (not escaped in any way). For bytes, contains the C escaped value. + # All bytes >= 128 are escaped. TODO(kenton): Base-64 encode? + default_value: str = betterproto.string_field(7) + # If set, gives the index of a oneof in the containing type's oneof_decl + # list. This field is a member of that oneof. + oneof_index: int = betterproto.int32_field(9) + # JSON name of this field. The value is set by protocol compiler. If the user + # has set a "json_name" option on this field, that option's value will be + # used. Otherwise, it's deduced from the field's name by converting it to + # camelCase. + json_name: str = betterproto.string_field(10) + options: "FieldOptions" = betterproto.message_field(8) + + +@dataclass +class OneofDescriptorProto(betterproto.Message): + """Describes a oneof.""" + + name: str = betterproto.string_field(1) + options: "OneofOptions" = betterproto.message_field(2) + + +@dataclass +class EnumDescriptorProto(betterproto.Message): + """Describes an enum type.""" + + name: str = betterproto.string_field(1) + value: List["EnumValueDescriptorProto"] = betterproto.message_field( + 2, wraps=betterproto.TYPE_ENUM + ) + options: "EnumOptions" = betterproto.message_field(3) + # Range of reserved numeric values. Reserved numeric values may not be used + # by enum values in the same enum declaration. Reserved ranges may not + # overlap. + reserved_range: List[ + "EnumDescriptorProtoEnumReservedRange" + ] = betterproto.message_field(4) + # Reserved enum value names, which may not be reused. A given name may only + # be reserved once. + reserved_name: List[str] = betterproto.string_field(5) + + +@dataclass +class EnumDescriptorProtoEnumReservedRange(betterproto.Message): + """ + Range of reserved numeric values. Reserved values may not be used by + entries in the same enum. Reserved ranges may not overlap. Note that this + is distinct from DescriptorProto.ReservedRange in that it is inclusive such + that it can appropriately represent the entire int32 domain. + """ + + start: int = betterproto.int32_field(1) + end: int = betterproto.int32_field(2) + + +@dataclass +class EnumValueDescriptorProto(betterproto.Message): + """Describes a value within an enum.""" + + name: str = betterproto.string_field(1) + number: int = betterproto.int32_field(2) + options: "EnumValueOptions" = betterproto.message_field( + 3, wraps=betterproto.TYPE_ENUM + ) + + +@dataclass +class ServiceDescriptorProto(betterproto.Message): + """Describes a service.""" + + name: str = betterproto.string_field(1) + method: List["MethodDescriptorProto"] = betterproto.message_field(2) + options: "ServiceOptions" = betterproto.message_field(3) + + +@dataclass +class MethodDescriptorProto(betterproto.Message): + """Describes a method of a service.""" + + name: str = betterproto.string_field(1) + # Input and output type names. These are resolved in the same way as + # FieldDescriptorProto.type_name, but must refer to a message type. + input_type: str = betterproto.string_field(2) + output_type: str = betterproto.string_field(3) + options: "MethodOptions" = betterproto.message_field(4) + # Identifies if client streams multiple client messages + client_streaming: bool = betterproto.bool_field(5) + # Identifies if server streams multiple server messages + server_streaming: bool = betterproto.bool_field(6) + + +@dataclass +class FileOptions(betterproto.Message): + # Sets the Java package where classes generated from this .proto will be + # placed. By default, the proto package is used, but this is often + # inappropriate because proto packages do not normally start with backwards + # domain names. + java_package: str = betterproto.string_field(1) + # If set, all the classes from the .proto file are wrapped in a single outer + # class with the given name. This applies to both Proto1 (equivalent to the + # old "--one_java_file" option) and Proto2 (where a .proto always translates + # to a single class, but you may want to explicitly choose the class name). + java_outer_classname: str = betterproto.string_field(8) + # If set true, then the Java code generator will generate a separate .java + # file for each top-level message, enum, and service defined in the .proto + # file. Thus, these types will *not* be nested inside the outer class named + # by java_outer_classname. However, the outer class will still be generated + # to contain the file's getDescriptor() method as well as any top-level + # extensions defined in the file. + java_multiple_files: bool = betterproto.bool_field(10) + # This option does nothing. + java_generate_equals_and_hash: bool = betterproto.bool_field(20) + # If set true, then the Java2 code generator will generate code that throws + # an exception whenever an attempt is made to assign a non-UTF-8 byte + # sequence to a string field. Message reflection will do the same. However, + # an extension field still accepts non-UTF-8 byte sequences. This option has + # no effect on when used with the lite runtime. + java_string_check_utf8: bool = betterproto.bool_field(27) + optimize_for: "FileOptionsOptimizeMode" = betterproto.enum_field(9) + # Sets the Go package where structs generated from this .proto will be + # placed. If omitted, the Go package will be derived from the following: - + # The basename of the package import path, if provided. - Otherwise, the + # package statement in the .proto file, if present. - Otherwise, the + # basename of the .proto file, without extension. + go_package: str = betterproto.string_field(11) + # Should generic services be generated in each language? "Generic" services + # are not specific to any particular RPC system. They are generated by the + # main code generators in each language (without additional plugins). Generic + # services were the only kind of service generation supported by early + # versions of google.protobuf. Generic services are now considered deprecated + # in favor of using plugins that generate code specific to your particular + # RPC system. Therefore, these default to false. Old code which depends on + # generic services should explicitly set them to true. + cc_generic_services: bool = betterproto.bool_field(16) + java_generic_services: bool = betterproto.bool_field(17) + py_generic_services: bool = betterproto.bool_field(18) + php_generic_services: bool = betterproto.bool_field(42) + # Is this file deprecated? Depending on the target platform, this can emit + # Deprecated annotations for everything in the file, or it will be completely + # ignored; in the very least, this is a formalization for deprecating files. + deprecated: bool = betterproto.bool_field(23) + # Enables the use of arenas for the proto messages in this file. This applies + # only to generated classes for C++. + cc_enable_arenas: bool = betterproto.bool_field(31) + # Sets the objective c class prefix which is prepended to all objective c + # generated classes from this .proto. There is no default. + objc_class_prefix: str = betterproto.string_field(36) + # Namespace for generated classes; defaults to the package. + csharp_namespace: str = betterproto.string_field(37) + # By default Swift generators will take the proto package and CamelCase it + # replacing '.' with underscore and use that to prefix the types/symbols + # defined. When this options is provided, they will use this value instead to + # prefix the types/symbols defined. + swift_prefix: str = betterproto.string_field(39) + # Sets the php class prefix which is prepended to all php generated classes + # from this .proto. Default is empty. + php_class_prefix: str = betterproto.string_field(40) + # Use this option to change the namespace of php generated classes. Default + # is empty. When this option is empty, the package name will be used for + # determining the namespace. + php_namespace: str = betterproto.string_field(41) + # Use this option to change the namespace of php generated metadata classes. + # Default is empty. When this option is empty, the proto file name will be + # used for determining the namespace. + php_metadata_namespace: str = betterproto.string_field(44) + # Use this option to change the package of ruby generated classes. Default is + # empty. When this option is not set, the package name will be used for + # determining the ruby package. + ruby_package: str = betterproto.string_field(45) + # The parser stores options it doesn't recognize here. See the documentation + # for the "Options" section above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class MessageOptions(betterproto.Message): + # Set true to use the old proto1 MessageSet wire format for extensions. This + # is provided for backwards-compatibility with the MessageSet wire format. + # You should not use this for any other reason: It's less efficient, has + # fewer features, and is more complicated. The message must be defined + # exactly as follows: message Foo { option message_set_wire_format = + # true; extensions 4 to max; } Note that the message cannot have any + # defined fields; MessageSets only have extensions. All extensions of your + # type must be singular messages; e.g. they cannot be int32s, enums, or + # repeated messages. Because this is an option, the above two restrictions + # are not enforced by the protocol compiler. + message_set_wire_format: bool = betterproto.bool_field(1) + # Disables the generation of the standard "descriptor()" accessor, which can + # conflict with a field of the same name. This is meant to make migration + # from proto1 easier; new code should avoid fields named "descriptor". + no_standard_descriptor_accessor: bool = betterproto.bool_field(2) + # Is this message deprecated? Depending on the target platform, this can emit + # Deprecated annotations for the message, or it will be completely ignored; + # in the very least, this is a formalization for deprecating messages. + deprecated: bool = betterproto.bool_field(3) + # Whether the message is an automatically generated map entry type for the + # maps field. For maps fields: map map_field = 1; The + # parsed descriptor looks like: message MapFieldEntry { option + # map_entry = true; optional KeyType key = 1; optional + # ValueType value = 2; } repeated MapFieldEntry map_field = 1; + # Implementations may choose not to generate the map_entry=true message, but + # use a native map in the target language to hold the keys and values. The + # reflection APIs in such implementations still need to work as if the field + # is a repeated message field. NOTE: Do not set the option in .proto files. + # Always use the maps syntax instead. The option should only be implicitly + # set by the proto compiler parser. + map_entry: bool = betterproto.bool_field(7) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class FieldOptions(betterproto.Message): + # The ctype option instructs the C++ code generator to use a different + # representation of the field than it normally would. See the specific + # options below. This option is not yet implemented in the open source + # release -- sorry, we'll try to include it in a future version! + ctype: "FieldOptionsCType" = betterproto.enum_field(1) + # The packed option can be enabled for repeated primitive fields to enable a + # more efficient representation on the wire. Rather than repeatedly writing + # the tag and type for each element, the entire array is encoded as a single + # length-delimited blob. In proto3, only explicit setting it to false will + # avoid using packed encoding. + packed: bool = betterproto.bool_field(2) + # The jstype option determines the JavaScript type used for values of the + # field. The option is permitted only for 64 bit integral and fixed types + # (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + # is represented as JavaScript string, which avoids loss of precision that + # can happen when a large value is converted to a floating point JavaScript. + # Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + # use the JavaScript "number" type. The behavior of the default option + # JS_NORMAL is implementation dependent. This option is an enum to permit + # additional types to be added, e.g. goog.math.Integer. + jstype: "FieldOptionsJSType" = betterproto.enum_field(6) + # Should this field be parsed lazily? Lazy applies only to message-type + # fields. It means that when the outer message is initially parsed, the + # inner message's contents will not be parsed but instead stored in encoded + # form. The inner message will actually be parsed when it is first accessed. + # This is only a hint. Implementations are free to choose whether to use + # eager or lazy parsing regardless of the value of this option. However, + # setting this option true suggests that the protocol author believes that + # using lazy parsing on this field is worth the additional bookkeeping + # overhead typically needed to implement it. This option does not affect the + # public interface of any generated code; all method signatures remain the + # same. Furthermore, thread-safety of the interface is not affected by this + # option; const methods remain safe to call from multiple threads + # concurrently, while non-const methods continue to require exclusive access. + # Note that implementations may choose not to check required fields within a + # lazy sub-message. That is, calling IsInitialized() on the outer message + # may return true even if the inner message has missing required fields. This + # is necessary because otherwise the inner message would have to be parsed in + # order to perform the check, defeating the purpose of lazy parsing. An + # implementation which chooses not to check required fields must be + # consistent about it. That is, for any particular sub-message, the + # implementation must either *always* check its required fields, or *never* + # check its required fields, regardless of whether or not the message has + # been parsed. + lazy: bool = betterproto.bool_field(5) + # Is this field deprecated? Depending on the target platform, this can emit + # Deprecated annotations for accessors, or it will be completely ignored; in + # the very least, this is a formalization for deprecating fields. + deprecated: bool = betterproto.bool_field(3) + # For Google-internal migration only. Do not use. + weak: bool = betterproto.bool_field(10) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class OneofOptions(betterproto.Message): + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class EnumOptions(betterproto.Message): + # Set this option to true to allow mapping different tag names to the same + # value. + allow_alias: bool = betterproto.bool_field(2) + # Is this enum deprecated? Depending on the target platform, this can emit + # Deprecated annotations for the enum, or it will be completely ignored; in + # the very least, this is a formalization for deprecating enums. + deprecated: bool = betterproto.bool_field(3) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class EnumValueOptions(betterproto.Message): + # Is this enum value deprecated? Depending on the target platform, this can + # emit Deprecated annotations for the enum value, or it will be completely + # ignored; in the very least, this is a formalization for deprecating enum + # values. + deprecated: bool = betterproto.bool_field(1) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class ServiceOptions(betterproto.Message): + # Is this service deprecated? Depending on the target platform, this can emit + # Deprecated annotations for the service, or it will be completely ignored; + # in the very least, this is a formalization for deprecating services. + deprecated: bool = betterproto.bool_field(33) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class MethodOptions(betterproto.Message): + # Is this method deprecated? Depending on the target platform, this can emit + # Deprecated annotations for the method, or it will be completely ignored; in + # the very least, this is a formalization for deprecating methods. + deprecated: bool = betterproto.bool_field(33) + idempotency_level: "MethodOptionsIdempotencyLevel" = betterproto.enum_field(34) + # The parser stores options it doesn't recognize here. See above. + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + + +@dataclass +class UninterpretedOption(betterproto.Message): + """ + A message representing a option the parser does not recognize. This only + appears in options protos created by the compiler::Parser class. + DescriptorPool resolves these when building Descriptor objects. Therefore, + options protos in descriptor objects (e.g. returned by + Descriptor::options(), or produced by Descriptor::CopyTo()) will never have + UninterpretedOptions in them. + """ + + name: List["UninterpretedOptionNamePart"] = betterproto.message_field(2) + # The value of the uninterpreted option, in whatever type the tokenizer + # identified it as during parsing. Exactly one of these should be set. + identifier_value: str = betterproto.string_field(3) + positive_int_value: int = betterproto.uint64_field(4) + negative_int_value: int = betterproto.int64_field(5) + double_value: float = betterproto.double_field(6) + string_value: bytes = betterproto.bytes_field(7) + aggregate_value: str = betterproto.string_field(8) + + +@dataclass +class UninterpretedOptionNamePart(betterproto.Message): + """ + The name of the uninterpreted option. Each string represents a segment in + a dot-separated name. is_extension is true iff a segment represents an + extension (denoted with parentheses in options specs in .proto files). + E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + "foo.(bar.baz).qux". + """ + + name_part: str = betterproto.string_field(1) + is_extension: bool = betterproto.bool_field(2) + + +@dataclass +class SourceCodeInfo(betterproto.Message): + """ + Encapsulates information about the original source file from which a + FileDescriptorProto was generated. + """ + + # A Location identifies a piece of source code in a .proto file which + # corresponds to a particular definition. This information is intended to be + # useful to IDEs, code indexers, documentation generators, and similar tools. + # For example, say we have a file like: message Foo { optional string + # foo = 1; } Let's look at just the field definition: optional string foo + # = 1; ^ ^^ ^^ ^ ^^^ a bc de f ghi We have the + # following locations: span path represents [a,i) [ 4, + # 0, 2, 0 ] The whole field definition. [a,b) [ 4, 0, 2, 0, 4 ] The + # label (optional). [c,d) [ 4, 0, 2, 0, 5 ] The type (string). [e,f) [ + # 4, 0, 2, 0, 1 ] The name (foo). [g,h) [ 4, 0, 2, 0, 3 ] The number + # (1). Notes: - A location may refer to a repeated field itself (i.e. not to + # any particular index within it). This is used whenever a set of elements + # are logically enclosed in a single code segment. For example, an entire + # extend block (possibly containing multiple extension definitions) will + # have an outer location whose path refers to the "extensions" repeated + # field without an index. - Multiple locations may have the same path. This + # happens when a single logical declaration is spread out across multiple + # places. The most obvious example is the "extend" block again -- there + # may be multiple extend blocks in the same scope, each of which will have + # the same path. - A location's span is not always a subset of its parent's + # span. For example, the "extendee" of an extension declaration appears at + # the beginning of the "extend" block and is shared by all extensions + # within the block. - Just because a location's span is a subset of some + # other location's span does not mean that it is a descendant. For + # example, a "group" defines both a type and a field in a single + # declaration. Thus, the locations corresponding to the type and field and + # their components will overlap. - Code which tries to interpret locations + # should probably be designed to ignore those that it doesn't understand, + # as more types of locations could be recorded in the future. + location: List["SourceCodeInfoLocation"] = betterproto.message_field(1) + + +@dataclass +class SourceCodeInfoLocation(betterproto.Message): + # Identifies which part of the FileDescriptorProto was defined at this + # location. Each element is a field number or an index. They form a path + # from the root FileDescriptorProto to the place where the definition. For + # example, this path: [ 4, 3, 2, 7, 1 ] refers to: file.message_type(3) + # // 4, 3 .field(7) // 2, 7 .name() // 1 This + # is because FileDescriptorProto.message_type has field number 4: repeated + # DescriptorProto message_type = 4; and DescriptorProto.field has field + # number 2: repeated FieldDescriptorProto field = 2; and + # FieldDescriptorProto.name has field number 1: optional string name = 1; + # Thus, the above path gives the location of a field name. If we removed the + # last element: [ 4, 3, 2, 7 ] this path refers to the whole field + # declaration (from the beginning of the label to the terminating semicolon). + path: List[int] = betterproto.int32_field(1) + # Always has exactly three or four elements: start line, start column, end + # line (optional, otherwise assumed same as start line), end column. These + # are packed into a single field for efficiency. Note that line and column + # numbers are zero-based -- typically you will want to add 1 to each before + # displaying to a user. + span: List[int] = betterproto.int32_field(2) + # If this SourceCodeInfo represents a complete declaration, these are any + # comments appearing before and after the declaration which appear to be + # attached to the declaration. A series of line comments appearing on + # consecutive lines, with no other tokens appearing on those lines, will be + # treated as a single comment. leading_detached_comments will keep paragraphs + # of comments that appear before (but not connected to) the current element. + # Each paragraph, separated by empty lines, will be one comment element in + # the repeated field. Only the comment content is provided; comment markers + # (e.g. //) are stripped out. For block comments, leading whitespace and an + # asterisk will be stripped from the beginning of each line other than the + # first. Newlines are included in the output. Examples: optional int32 foo + # = 1; // Comment attached to foo. // Comment attached to bar. optional + # int32 bar = 2; optional string baz = 3; // Comment attached to baz. + # // Another line attached to baz. // Comment attached to qux. // // + # Another line attached to qux. optional double qux = 4; // Detached + # comment for corge. This is not leading or trailing comments // to qux or + # corge because there are blank lines separating it from // both. // + # Detached comment for corge paragraph 2. optional string corge = 5; /* + # Block comment attached * to corge. Leading asterisks * will be + # removed. */ /* Block comment attached to * grault. */ optional int32 + # grault = 6; // ignored detached comments. + leading_comments: str = betterproto.string_field(3) + trailing_comments: str = betterproto.string_field(4) + leading_detached_comments: List[str] = betterproto.string_field(6) + + +@dataclass +class GeneratedCodeInfo(betterproto.Message): + """ + Describes the relationship between generated code and its original source + file. A GeneratedCodeInfo message is associated with only one generated + source file, but may contain references to different source .proto files. + """ + + # An Annotation connects some span of text in generated code to an element of + # its generating .proto file. + annotation: List["GeneratedCodeInfoAnnotation"] = betterproto.message_field(1) + + +@dataclass +class GeneratedCodeInfoAnnotation(betterproto.Message): + # Identifies the element in the original source .proto file. This field is + # formatted the same as SourceCodeInfo.Location.path. + path: List[int] = betterproto.int32_field(1) + # Identifies the filesystem path to the original source .proto. + source_file: str = betterproto.string_field(2) + # Identifies the starting offset in bytes in the generated code that relates + # to the identified object. + begin: int = betterproto.int32_field(3) + # Identifies the ending offset in bytes in the generated code that relates to + # the identified offset. The end offset should be one past the last relevant + # byte (so the length of the text = end - begin). + end: int = betterproto.int32_field(4) + + +@dataclass +class Duration(betterproto.Message): + """ + A Duration represents a signed, fixed-length span of time represented as a + count of seconds and fractions of seconds at nanosecond resolution. It is + independent of any calendar and concepts like "day" or "month". It is + related to Timestamp in that the difference between two Timestamp values is + a Duration and it can be added or subtracted from a Timestamp. Range is + approximately +-10,000 years. # Examples Example 1: Compute Duration from + two Timestamps in pseudo code. Timestamp start = ...; Timestamp end + = ...; Duration duration = ...; duration.seconds = end.seconds - + start.seconds; duration.nanos = end.nanos - start.nanos; if + (duration.seconds < 0 && duration.nanos > 0) { duration.seconds += 1; + duration.nanos -= 1000000000; } else if (duration.seconds > 0 && + duration.nanos < 0) { duration.seconds -= 1; duration.nanos += + 1000000000; } Example 2: Compute Timestamp from Timestamp + Duration in + pseudo code. Timestamp start = ...; Duration duration = ...; + Timestamp end = ...; end.seconds = start.seconds + duration.seconds; + end.nanos = start.nanos + duration.nanos; if (end.nanos < 0) { + end.seconds -= 1; end.nanos += 1000000000; } else if (end.nanos + >= 1000000000) { end.seconds += 1; end.nanos -= 1000000000; + } Example 3: Compute Duration from datetime.timedelta in Python. td = + datetime.timedelta(days=3, minutes=10) duration = Duration() + duration.FromTimedelta(td) # JSON Mapping In JSON format, the Duration type + is encoded as a string rather than an object, where the string ends in the + suffix "s" (indicating seconds) and is preceded by the number of seconds, + with nanoseconds expressed as fractional seconds. For example, 3 seconds + with 0 nanoseconds should be encoded in JSON format as "3s", while 3 + seconds and 1 nanosecond should be expressed in JSON format as + "3.000000001s", and 3 seconds and 1 microsecond should be expressed in JSON + format as "3.000001s". + """ + + # Signed seconds of the span of time. Must be from -315,576,000,000 to + # +315,576,000,000 inclusive. Note: these bounds are computed from: 60 + # sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + seconds: int = betterproto.int64_field(1) + # Signed fractions of a second at nanosecond resolution of the span of time. + # Durations less than one second are represented with a 0 `seconds` field and + # a positive or negative `nanos` field. For durations of one second or more, + # a non-zero value for the `nanos` field must be of the same sign as the + # `seconds` field. Must be from -999,999,999 to +999,999,999 inclusive. + nanos: int = betterproto.int32_field(2) + + +@dataclass +class Empty(betterproto.Message): + """ + A generic empty message that you can re-use to avoid defining duplicated + empty messages in your APIs. A typical example is to use it as the request + or the response type of an API method. For instance: service Foo { + rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); } The + JSON representation for `Empty` is empty JSON object `{}`. + """ + + pass + + +@dataclass +class FieldMask(betterproto.Message): + """ + `FieldMask` represents a set of symbolic field paths, for example: + paths: "f.a" paths: "f.b.d" Here `f` represents a field in some root + message, `a` and `b` fields in the message found in `f`, and `d` a field + found in the message in `f.b`. Field masks are used to specify a subset of + fields that should be returned by a get operation or modified by an update + operation. Field masks also have a custom JSON encoding (see below). # + Field Masks in Projections When used in the context of a projection, a + response message or sub-message is filtered by the API to only contain + those fields as specified in the mask. For example, if the mask in the + previous example is applied to a response message as follows: f { + a : 22 b { d : 1 x : 2 } y : 13 } + z: 8 The result will not contain specific values for fields x,y and z + (their value will be set to the default, and omitted in proto text output): + f { a : 22 b { d : 1 } } A repeated field is + not allowed except at the last position of a paths string. If a FieldMask + object is not present in a get operation, the operation applies to all + fields (as if a FieldMask of all fields had been specified). Note that a + field mask does not necessarily apply to the top-level response message. In + case of a REST get operation, the field mask applies directly to the + response, but in case of a REST list operation, the mask instead applies to + each individual message in the returned resource list. In case of a REST + custom method, other definitions may be used. Where the mask applies will + be clearly documented together with its declaration in the API. In any + case, the effect on the returned resource/resources is required behavior + for APIs. # Field Masks in Update Operations A field mask in update + operations specifies which fields of the targeted resource are going to be + updated. The API is required to only change the values of the fields as + specified in the mask and leave the others untouched. If a resource is + passed in to describe the updated values, the API ignores the values of all + fields not covered by the mask. If a repeated field is specified for an + update operation, new values will be appended to the existing repeated + field in the target resource. Note that a repeated field is only allowed in + the last position of a `paths` string. If a sub-message is specified in the + last position of the field mask for an update operation, then new value + will be merged into the existing sub-message in the target resource. For + example, given the target message: f { b { d: 1 + x: 2 } c: [1] } And an update message: f { b { + d: 10 } c: [2] } then if the field mask is: paths: ["f.b", + "f.c"] then the result will be: f { b { d: 10 x: + 2 } c: [1, 2] } An implementation may provide options to + override this default behavior for repeated and message fields. In order to + reset a field's value to the default, the field must be in the mask and set + to the default value in the provided resource. Hence, in order to reset all + fields of a resource, provide a default instance of the resource and set + all fields in the mask, or do not provide a mask as described below. If a + field mask is not present on update, the operation applies to all fields + (as if a field mask of all fields has been specified). Note that in the + presence of schema evolution, this may mean that fields the client does not + know and has therefore not filled into the request will be reset to their + default. If this is unwanted behavior, a specific service may require a + client to always specify a field mask, producing an error if not. As with + get operations, the location of the resource which describes the updated + values in the request message depends on the operation kind. In any case, + the effect of the field mask is required to be honored by the API. ## + Considerations for HTTP REST The HTTP kind of an update operation which + uses a field mask must be set to PATCH instead of PUT in order to satisfy + HTTP semantics (PUT must only be used for full updates). # JSON Encoding of + Field Masks In JSON, a field mask is encoded as a single string where paths + are separated by a comma. Fields name in each path are converted to/from + lower-camel naming conventions. As an example, consider the following + message declarations: message Profile { User user = 1; + Photo photo = 2; } message User { string display_name = 1; + string address = 2; } In proto a field mask for `Profile` may look as + such: mask { paths: "user.display_name" paths: "photo" + } In JSON, the same mask is represented as below: { mask: + "user.displayName,photo" } # Field Masks and Oneof Fields Field masks + treat fields in oneofs just as regular fields. Consider the following + message: message SampleMessage { oneof test_oneof { + string name = 4; SubMessage sub_message = 9; } } The + field mask can be: mask { paths: "name" } Or: mask { + paths: "sub_message" } Note that oneof type names ("test_oneof" in this + case) cannot be used in paths. ## Field Mask Verification The + implementation of any API method which has a FieldMask type field in the + request should verify the included field paths, and return an + `INVALID_ARGUMENT` error if any path is unmappable. + """ + + # The set of field mask paths. + paths: List[str] = betterproto.string_field(1) + + +@dataclass +class Struct(betterproto.Message): + """ + `Struct` represents a structured data value, consisting of fields which map + to dynamically typed values. In some languages, `Struct` might be supported + by a native representation. For example, in scripting languages like JS a + struct is represented as an object. The details of that representation are + described together with the proto support for the language. The JSON + representation for `Struct` is JSON object. + """ + + # Unordered map of dynamically typed values. + fields: Dict[str, "Value"] = betterproto.map_field( + 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE + ) + + +@dataclass +class Value(betterproto.Message): + """ + `Value` represents a dynamically typed value which can be either null, a + number, a string, a boolean, a recursive struct value, or a list of values. + A producer of value is expected to set one of that variants, absence of any + variant indicates an error. The JSON representation for `Value` is JSON + value. + """ + + # Represents a null value. + null_value: "NullValue" = betterproto.enum_field(1, group="kind") + # Represents a double value. + number_value: float = betterproto.double_field(2, group="kind") + # Represents a string value. + string_value: str = betterproto.string_field(3, group="kind") + # Represents a boolean value. + bool_value: bool = betterproto.bool_field(4, group="kind") + # Represents a structured value. + struct_value: "Struct" = betterproto.message_field(5, group="kind") + # Represents a repeated `Value`. + list_value: "ListValue" = betterproto.message_field(6, group="kind") + + +@dataclass +class ListValue(betterproto.Message): + """ + `ListValue` is a wrapper around a repeated field of values. The JSON + representation for `ListValue` is JSON array. + """ + + # Repeated field of dynamically typed values. + values: List["Value"] = betterproto.message_field(1) + + +@dataclass +class Timestamp(betterproto.Message): + """ + A Timestamp represents a point in time independent of any time zone or + local calendar, encoded as a count of seconds and fractions of seconds at + nanosecond resolution. The count is relative to an epoch at UTC midnight on + January 1, 1970, in the proleptic Gregorian calendar which extends the + Gregorian calendar backwards to year one. All minutes are 60 seconds long. + Leap seconds are "smeared" so that no leap second table is needed for + interpretation, using a [24-hour linear + smear](https://developers.google.com/time/smear). The range is from + 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By restricting to + that range, we ensure that we can convert to and from [RFC + 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. # Examples + Example 1: Compute Timestamp from POSIX `time()`. Timestamp timestamp; + timestamp.set_seconds(time(NULL)); timestamp.set_nanos(0); Example 2: + Compute Timestamp from POSIX `gettimeofday()`. struct timeval tv; + gettimeofday(&tv, NULL); Timestamp timestamp; + timestamp.set_seconds(tv.tv_sec); timestamp.set_nanos(tv.tv_usec * + 1000); Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. + FILETIME ft; GetSystemTimeAsFileTime(&ft); UINT64 ticks = + (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; // A Windows + tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z // is + 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. Timestamp + timestamp; timestamp.set_seconds((INT64) ((ticks / 10000000) - + 11644473600LL)); timestamp.set_nanos((INT32) ((ticks % 10000000) * + 100)); Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. + long millis = System.currentTimeMillis(); Timestamp timestamp = + Timestamp.newBuilder().setSeconds(millis / 1000) .setNanos((int) + ((millis % 1000) * 1000000)).build(); Example 5: Compute Timestamp from + current time in Python. timestamp = Timestamp() + timestamp.GetCurrentTime() # JSON Mapping In JSON format, the Timestamp + type is encoded as a string in the [RFC + 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the format is + "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" where {year} is + always expressed using four digits while {month}, {day}, {hour}, {min}, and + {sec} are zero-padded to two digits each. The fractional seconds, which can + go up to 9 digits (i.e. up to 1 nanosecond resolution), are optional. The + "Z" suffix indicates the timezone ("UTC"); the timezone is required. A + proto3 JSON serializer should always use UTC (as indicated by "Z") when + printing the Timestamp type and a proto3 JSON parser should be able to + accept both UTC and other timezones (as indicated by an offset). For + example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past 01:30 UTC on + January 15, 2017. In JavaScript, one can convert a Date object to this + format using the standard [toISOString()](https://developer.mozilla.org/en- + US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) method. + In Python, a standard `datetime.datetime` object can be converted to this + format using + [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) + with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one + can use the Joda Time's [`ISODateTimeFormat.dateTime()`]( + http://www.joda.org/joda- + time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D ) + to obtain a formatter capable of generating timestamps in this format. + """ + + # Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must + # be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive. + seconds: int = betterproto.int64_field(1) + # Non-negative fractions of a second at nanosecond resolution. Negative + # second values with fractions must still have non-negative nanos values that + # count forward in time. Must be from 0 to 999,999,999 inclusive. + nanos: int = betterproto.int32_field(2) + + +@dataclass +class DoubleValue(betterproto.Message): + """ + Wrapper message for `double`. The JSON representation for `DoubleValue` is + JSON number. + """ + + # The double value. + value: float = betterproto.double_field(1) + + +@dataclass +class FloatValue(betterproto.Message): + """ + Wrapper message for `float`. The JSON representation for `FloatValue` is + JSON number. + """ + + # The float value. + value: float = betterproto.float_field(1) + + +@dataclass +class Int64Value(betterproto.Message): + """ + Wrapper message for `int64`. The JSON representation for `Int64Value` is + JSON string. + """ + + # The int64 value. + value: int = betterproto.int64_field(1) + + +@dataclass +class UInt64Value(betterproto.Message): + """ + Wrapper message for `uint64`. The JSON representation for `UInt64Value` is + JSON string. + """ + + # The uint64 value. + value: int = betterproto.uint64_field(1) + + +@dataclass +class Int32Value(betterproto.Message): + """ + Wrapper message for `int32`. The JSON representation for `Int32Value` is + JSON number. + """ + + # The int32 value. + value: int = betterproto.int32_field(1) + + +@dataclass +class UInt32Value(betterproto.Message): + """ + Wrapper message for `uint32`. The JSON representation for `UInt32Value` is + JSON number. + """ + + # The uint32 value. + value: int = betterproto.uint32_field(1) + + +@dataclass +class BoolValue(betterproto.Message): + """ + Wrapper message for `bool`. The JSON representation for `BoolValue` is JSON + `true` and `false`. + """ + + # The bool value. + value: bool = betterproto.bool_field(1) + + +@dataclass +class StringValue(betterproto.Message): + """ + Wrapper message for `string`. The JSON representation for `StringValue` is + JSON string. + """ + + # The string value. + value: str = betterproto.string_field(1) + + +@dataclass +class BytesValue(betterproto.Message): + """ + Wrapper message for `bytes`. The JSON representation for `BytesValue` is + JSON string. + """ + + # The bytes value. + value: bytes = betterproto.bytes_field(1) From e8991339e9ec54b412193eb1f27b78a04d7a7297 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:34:34 +0200 Subject: [PATCH 03/18] Use pre-compiled wrapper-classes --- betterproto/__init__.py | 76 ++++++++++++----------------------------- 1 file changed, 22 insertions(+), 54 deletions(-) diff --git a/betterproto/__init__.py b/betterproto/__init__.py index f394b41..c300ded 100644 --- a/betterproto/__init__.py +++ b/betterproto/__init__.py @@ -1016,63 +1016,18 @@ class _WrappedMessage(Message): return self -@dataclasses.dataclass -class _BoolValue(_WrappedMessage): - value: bool = bool_field(1) - - -@dataclasses.dataclass -class _Int32Value(_WrappedMessage): - value: int = int32_field(1) - - -@dataclasses.dataclass -class _UInt32Value(_WrappedMessage): - value: int = uint32_field(1) - - -@dataclasses.dataclass -class _Int64Value(_WrappedMessage): - value: int = int64_field(1) - - -@dataclasses.dataclass -class _UInt64Value(_WrappedMessage): - value: int = uint64_field(1) - - -@dataclasses.dataclass -class _FloatValue(_WrappedMessage): - value: float = float_field(1) - - -@dataclasses.dataclass -class _DoubleValue(_WrappedMessage): - value: float = double_field(1) - - -@dataclasses.dataclass -class _StringValue(_WrappedMessage): - value: str = string_field(1) - - -@dataclasses.dataclass -class _BytesValue(_WrappedMessage): - value: bytes = bytes_field(1) - - def _get_wrapper(proto_type: str) -> Type: """Get the wrapper message class for a wrapped type.""" return { - TYPE_BOOL: _BoolValue, - TYPE_INT32: _Int32Value, - TYPE_UINT32: _UInt32Value, - TYPE_INT64: _Int64Value, - TYPE_UINT64: _UInt64Value, - TYPE_FLOAT: _FloatValue, - TYPE_DOUBLE: _DoubleValue, - TYPE_STRING: _StringValue, - TYPE_BYTES: _BytesValue, + TYPE_BOOL: BoolValue, + TYPE_INT32: Int32Value, + TYPE_UINT32: UInt32Value, + TYPE_INT64: Int64Value, + TYPE_UINT64: UInt64Value, + TYPE_FLOAT: FloatValue, + TYPE_DOUBLE: DoubleValue, + TYPE_STRING: StringValue, + TYPE_BYTES: BytesValue, }[proto_type] @@ -1154,3 +1109,16 @@ class ServiceStub(ABC): await stream.send_message(request, end=True) async for message in stream: yield message + + +from .lib.google.protobuf import ( + BoolValue, + BytesValue, + DoubleValue, + FloatValue, + Int32Value, + Int64Value, + StringValue, + UInt32Value, + UInt64Value, +) From 53ce1255d35f17a42aef69ad75c049975c1ca38f Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:36:00 +0200 Subject: [PATCH 04/18] Do not unwrap google.protobuf.Value and unsupported wrapper types --- betterproto/plugin.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 3d9c1bb..4be1d21 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -1,13 +1,15 @@ #!/usr/bin/env python -from collections import defaultdict import itertools import os.path +import re import stringcase import sys import textwrap +from collections import defaultdict from typing import Dict, List, Optional, Type from betterproto.casing import safe_snake_case +import betterproto try: # betterproto[compiler] specific dependencies @@ -259,11 +261,13 @@ def generate_code(request, response): field_type = f.Type.Name(f.type).lower()[5:] field_wraps = "" - if f.type_name.startswith( - ".google.protobuf" - ) and f.type_name.endswith("Value"): - w = f.type_name.split(".").pop()[:-5].upper() - field_wraps = f"betterproto.TYPE_{w}" + match_wrapper = re.match( + "\\.google\\.protobuf\\.(.+)Value", f.type_name + ) + if match_wrapper: + wrapped_type = "TYPE_" + match_wrapper.group(1).upper() + if wrapped_type in dir(betterproto): + field_wraps = f"betterproto.{wrapped_type}" map_types = None if f.type == 11: From 2a3e1e1827d97abe6f7bf1e332e0593315793781 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:37:14 +0200 Subject: [PATCH 05/18] Add basic support for all google.protobuf types --- betterproto/plugin.py | 8 ++++++++ betterproto/tests/inputs/googletypes/googletypes.json | 4 +++- betterproto/tests/inputs/googletypes/googletypes.proto | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 4be1d21..96cb12d 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -88,6 +88,14 @@ def get_ref_type( cased = [stringcase.pascalcase(part) for part in parts] type_name = f'"{"".join(cased)}"' + # Use precompiled classes for google.protobuf.* objects + if type_name.startswith("google.protobuf.") and type_name.count(".") == 2: + type_name = type_name.rsplit(".", maxsplit=1)[1] + import_package = "betterproto.lib.google.protobuf" + import_alias = safe_snake_case(import_package) + imports.add(f"import {import_package} as {import_alias}") + return f"{import_alias}.{type_name}" + if "." in type_name: # This is imported from another package. No need # to use a forward ref and we need to add the import. diff --git a/betterproto/tests/inputs/googletypes/googletypes.json b/betterproto/tests/inputs/googletypes/googletypes.json index 5d86e1b..0a002e9 100644 --- a/betterproto/tests/inputs/googletypes/googletypes.json +++ b/betterproto/tests/inputs/googletypes/googletypes.json @@ -1,5 +1,7 @@ { "maybe": false, "ts": "1972-01-01T10:00:20.021Z", - "duration": "1.200s" + "duration": "1.200s", + "important": 10, + "empty": {} } diff --git a/betterproto/tests/inputs/googletypes/googletypes.proto b/betterproto/tests/inputs/googletypes/googletypes.proto index 283b836..ba3db12 100644 --- a/betterproto/tests/inputs/googletypes/googletypes.proto +++ b/betterproto/tests/inputs/googletypes/googletypes.proto @@ -3,10 +3,12 @@ syntax = "proto3"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; +import "google/protobuf/empty.proto"; message Test { google.protobuf.BoolValue maybe = 1; google.protobuf.Timestamp ts = 2; google.protobuf.Duration duration = 3; google.protobuf.Int32Value important = 4; + google.protobuf.Empty empty = 5; } From eeed1c0db7e79c87b51c8878ee590d224789c6a2 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 15:37:58 +0200 Subject: [PATCH 06/18] Extend pre-compiled Duration and Timestamp instead of manual definition --- betterproto/__init__.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/betterproto/__init__.py b/betterproto/__init__.py index c300ded..3fd4a58 100644 --- a/betterproto/__init__.py +++ b/betterproto/__init__.py @@ -941,19 +941,10 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Any]: return (field_name, getattr(message, field_name)) -@dataclasses.dataclass -class _Duration(Message): - # Signed seconds of the span of time. Must be from -315,576,000,000 to - # +315,576,000,000 inclusive. Note: these bounds are computed from: 60 - # sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years - seconds: int = int64_field(1) - # Signed fractions of a second at nanosecond resolution of the span of time. - # Durations less than one second are represented with a 0 `seconds` field and - # a positive or negative `nanos` field. For durations of one second or more, - # a non-zero value for the `nanos` field must be of the same sign as the - # `seconds` field. Must be from -999,999,999 to +999,999,999 inclusive. - nanos: int = int32_field(2) +from .lib.google.protobuf import Duration, Timestamp + +class _Duration(Duration): def to_timedelta(self) -> timedelta: return timedelta(seconds=self.seconds, microseconds=self.nanos / 1e3) @@ -966,16 +957,7 @@ class _Duration(Message): return ".".join(parts) + "s" -@dataclasses.dataclass -class _Timestamp(Message): - # Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must - # be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive. - seconds: int = int64_field(1) - # Non-negative fractions of a second at nanosecond resolution. Negative - # second values with fractions must still have non-negative nanos values that - # count forward in time. Must be from 0 to 999,999,999 inclusive. - nanos: int = int32_field(2) - +class _Timestamp(Timestamp): def to_datetime(self) -> datetime: ts = self.seconds + (self.nanos / 1e9) return datetime.fromtimestamp(ts, tz=timezone.utc) From 62fc421d605b76d8df215a3d1c3e581f46e7c18a Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 16:45:32 +0200 Subject: [PATCH 07/18] Add failing tests for google.protobuf Struct and Value #9 --- betterproto/tests/inputs/config.py | 2 ++ .../googletypes_struct/googletypes_struct.json | 5 +++++ .../googletypes_struct/googletypes_struct.proto | 7 +++++++ .../inputs/googletypes_value/googletypes_value.json | 11 +++++++++++ .../googletypes_value/googletypes_value.proto | 13 +++++++++++++ 5 files changed, 38 insertions(+) create mode 100644 betterproto/tests/inputs/googletypes_struct/googletypes_struct.json create mode 100644 betterproto/tests/inputs/googletypes_struct/googletypes_struct.proto create mode 100644 betterproto/tests/inputs/googletypes_value/googletypes_value.json create mode 100644 betterproto/tests/inputs/googletypes_value/googletypes_value.proto diff --git a/betterproto/tests/inputs/config.py b/betterproto/tests/inputs/config.py index 1f276ba..7ed40f5 100644 --- a/betterproto/tests/inputs/config.py +++ b/betterproto/tests/inputs/config.py @@ -12,6 +12,8 @@ tests = { "casing_message_field_uppercase", # 11 "namespace_keywords", # 70 "namespace_builtin_types", # 53 + "googletypes_struct", # 9 + "googletypes_value", # 9 } services = { diff --git a/betterproto/tests/inputs/googletypes_struct/googletypes_struct.json b/betterproto/tests/inputs/googletypes_struct/googletypes_struct.json new file mode 100644 index 0000000..ecc175e --- /dev/null +++ b/betterproto/tests/inputs/googletypes_struct/googletypes_struct.json @@ -0,0 +1,5 @@ +{ + "struct": { + "key": true + } +} diff --git a/betterproto/tests/inputs/googletypes_struct/googletypes_struct.proto b/betterproto/tests/inputs/googletypes_struct/googletypes_struct.proto new file mode 100644 index 0000000..1dbd64a --- /dev/null +++ b/betterproto/tests/inputs/googletypes_struct/googletypes_struct.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +import "google/protobuf/struct.proto"; + +message Test { + google.protobuf.Struct struct = 1; +} diff --git a/betterproto/tests/inputs/googletypes_value/googletypes_value.json b/betterproto/tests/inputs/googletypes_value/googletypes_value.json new file mode 100644 index 0000000..db52d5c --- /dev/null +++ b/betterproto/tests/inputs/googletypes_value/googletypes_value.json @@ -0,0 +1,11 @@ +{ + "value1": "hello world", + "value2": true, + "value3": 1, + "value4": null, + "value5": [ + 1, + 2, + 3 + ] +} diff --git a/betterproto/tests/inputs/googletypes_value/googletypes_value.proto b/betterproto/tests/inputs/googletypes_value/googletypes_value.proto new file mode 100644 index 0000000..379d336 --- /dev/null +++ b/betterproto/tests/inputs/googletypes_value/googletypes_value.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +import "google/protobuf/struct.proto"; + +// Tests that fields of type google.protobuf.Value can contain arbitrary JSON-values. + +message Test { + google.protobuf.Value value1 = 1; + google.protobuf.Value value2 = 2; + google.protobuf.Value value3 = 3; + google.protobuf.Value value4 = 4; + google.protobuf.Value value5 = 5; +} From f5ce1b71081750a8f6db2e469211b078b682e002 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 16:46:04 +0200 Subject: [PATCH 08/18] Check that config.xfail contains valid test case names --- betterproto/tests/test_inputs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/betterproto/tests/test_inputs.py b/betterproto/tests/test_inputs.py index 183db25..5eb386b 100644 --- a/betterproto/tests/test_inputs.py +++ b/betterproto/tests/test_inputs.py @@ -30,6 +30,10 @@ class TestCases: test for test in _messages if get_test_case_json_data(test) } + unknown_xfail_tests = xfail - _all + if unknown_xfail_tests: + raise Exception(f"Unknown test(s) in config.py: {unknown_xfail_tests}") + self.all = self.apply_xfail_marks(_all, xfail) self.services = self.apply_xfail_marks(_services, xfail) self.messages = self.apply_xfail_marks(_messages, xfail) @@ -110,7 +114,7 @@ def test_message_json(repeat, test_data: TestData) -> None: message.from_json(json_data) message_json = message.to_json(0) - assert json.loads(json_data) == json.loads(message_json) + assert json.loads(message_json) == json.loads(json_data) @pytest.mark.parametrize("test_data", test_cases.services, indirect=True) @@ -119,6 +123,7 @@ def test_service_can_be_instantiated(test_data: TestData) -> None: plugin_module.TestStub(MockChannel()) +@pytest.mark.skip @pytest.mark.parametrize("test_data", test_cases.messages_with_json, indirect=True) def test_binary_compatibility(repeat, test_data: TestData) -> None: plugin_module, reference_module, json_data = test_data From b813d1cedbb43fb0ff454c0fdd09c931c8357283 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 17:59:21 +0200 Subject: [PATCH 09/18] Undo adding skip to test --- betterproto/tests/test_inputs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/betterproto/tests/test_inputs.py b/betterproto/tests/test_inputs.py index 5eb386b..cac8327 100644 --- a/betterproto/tests/test_inputs.py +++ b/betterproto/tests/test_inputs.py @@ -123,7 +123,6 @@ def test_service_can_be_instantiated(test_data: TestData) -> None: plugin_module.TestStub(MockChannel()) -@pytest.mark.skip @pytest.mark.parametrize("test_data", test_cases.messages_with_json, indirect=True) def test_binary_compatibility(repeat, test_data: TestData) -> None: plugin_module, reference_module, json_data = test_data From 2f658df666c15f8b335e14568fe4f96cd5402ffe Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 19:13:55 +0200 Subject: [PATCH 10/18] Use betterproto wrapper classes, extract to module for testability --- betterproto/compile/__init__.py | 0 betterproto/compile/importing.py | 70 ++++++++++++++++ betterproto/plugin.py | 76 +---------------- betterproto/tests/__init__.py | 0 betterproto/tests/inputs/config.py | 2 +- .../test_googletypes_response.py | 20 ++--- betterproto/tests/test_get_ref_type.py | 82 +++++++++++++++++++ 7 files changed, 165 insertions(+), 85 deletions(-) create mode 100644 betterproto/compile/__init__.py create mode 100644 betterproto/compile/importing.py create mode 100644 betterproto/tests/__init__.py create mode 100644 betterproto/tests/test_get_ref_type.py diff --git a/betterproto/compile/__init__.py b/betterproto/compile/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/betterproto/compile/importing.py b/betterproto/compile/importing.py new file mode 100644 index 0000000..0c53e0b --- /dev/null +++ b/betterproto/compile/importing.py @@ -0,0 +1,70 @@ +from typing import Dict, Type + +import stringcase + +from betterproto import safe_snake_case +from betterproto.lib.google import protobuf as google_protobuf + +WRAPPER_TYPES: Dict[str, Type] = { + "google.protobuf.DoubleValue": google_protobuf.DoubleValue, + "google.protobuf.FloatValue": google_protobuf.FloatValue, + "google.protobuf.Int32Value": google_protobuf.Int32Value, + "google.protobuf.Int64Value": google_protobuf.Int64Value, + "google.protobuf.UInt32Value": google_protobuf.UInt32Value, + "google.protobuf.UInt64Value": google_protobuf.UInt64Value, + "google.protobuf.BoolValue": google_protobuf.BoolValue, + "google.protobuf.StringValue": google_protobuf.StringValue, + "google.protobuf.BytesValue": google_protobuf.BytesValue, +} + + +def get_ref_type( + package: str, imports: set, type_name: str, unwrap: bool = True +) -> str: + """ + Return a Python type name for a proto type reference. Adds the import if + necessary. Unwraps well known type if required. + """ + # If the package name is a blank string, then this should still work + # because by convention packages are lowercase and message/enum types are + # pascal-cased. May require refactoring in the future. + type_name = type_name.lstrip(".") + + is_wrapper = type_name in WRAPPER_TYPES + + if unwrap: + if is_wrapper: + wrapped_type = type(WRAPPER_TYPES[type_name]().value) + return f"Optional[{wrapped_type.__name__}]" + + if type_name == "google.protobuf.Duration": + return "timedelta" + + if type_name == "google.protobuf.Timestamp": + return "datetime" + + if type_name.startswith(package): + parts = type_name.lstrip(package).lstrip(".").split(".") + if len(parts) == 1 or (len(parts) > 1 and parts[0][0] == parts[0][0].upper()): + # This is the current package, which has nested types flattened. + # foo.bar_thing => FooBarThing + cased = [stringcase.pascalcase(part) for part in parts] + type_name = f'"{"".join(cased)}"' + + # Use precompiled classes for google.protobuf.* objects + if type_name.startswith("google.protobuf.") and type_name.count(".") == 2: + type_name = type_name.rsplit(".", maxsplit=1)[1] + import_package = "betterproto.lib.google.protobuf" + import_alias = safe_snake_case(import_package) + imports.add(f"import {import_package} as {import_alias}") + return f"{import_alias}.{type_name}" + + if "." in type_name: + # This is imported from another package. No need + # to use a forward ref and we need to add the import. + parts = type_name.split(".") + parts[-1] = stringcase.pascalcase(parts[-1]) + imports.add(f"from .{'.'.join(parts[:-2])} import {parts[-2]}") + type_name = f"{parts[-2]}.{parts[-1]}" + + return type_name diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 96cb12d..5780240 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -6,9 +6,9 @@ import re import stringcase import sys import textwrap -from collections import defaultdict -from typing import Dict, List, Optional, Type +from typing import List from betterproto.casing import safe_snake_case +from betterproto.compile.importing import get_ref_type import betterproto try: @@ -35,78 +35,6 @@ except ImportError as err: raise SystemExit(1) -WRAPPER_TYPES: Dict[str, Optional[Type]] = defaultdict( - lambda: None, - { - "google.protobuf.DoubleValue": google_wrappers.DoubleValue, - "google.protobuf.FloatValue": google_wrappers.FloatValue, - "google.protobuf.Int64Value": google_wrappers.Int64Value, - "google.protobuf.UInt64Value": google_wrappers.UInt64Value, - "google.protobuf.Int32Value": google_wrappers.Int32Value, - "google.protobuf.UInt32Value": google_wrappers.UInt32Value, - "google.protobuf.BoolValue": google_wrappers.BoolValue, - "google.protobuf.StringValue": google_wrappers.StringValue, - "google.protobuf.BytesValue": google_wrappers.BytesValue, - }, -) - - -def get_ref_type( - package: str, imports: set, type_name: str, unwrap: bool = True -) -> str: - """ - Return a Python type name for a proto type reference. Adds the import if - necessary. Unwraps well known type if required. - """ - # If the package name is a blank string, then this should still work - # because by convention packages are lowercase and message/enum types are - # pascal-cased. May require refactoring in the future. - type_name = type_name.lstrip(".") - - # Check if type is wrapper. - wrapper_class = WRAPPER_TYPES[type_name] - - if unwrap: - if wrapper_class: - wrapped_type = type(wrapper_class().value) - return f"Optional[{wrapped_type.__name__}]" - - if type_name == "google.protobuf.Duration": - return "timedelta" - - if type_name == "google.protobuf.Timestamp": - return "datetime" - elif wrapper_class: - imports.add(f"from {wrapper_class.__module__} import {wrapper_class.__name__}") - return f"{wrapper_class.__name__}" - - if type_name.startswith(package): - parts = type_name.lstrip(package).lstrip(".").split(".") - if len(parts) == 1 or (len(parts) > 1 and parts[0][0] == parts[0][0].upper()): - # This is the current package, which has nested types flattened. - # foo.bar_thing => FooBarThing - cased = [stringcase.pascalcase(part) for part in parts] - type_name = f'"{"".join(cased)}"' - - # Use precompiled classes for google.protobuf.* objects - if type_name.startswith("google.protobuf.") and type_name.count(".") == 2: - type_name = type_name.rsplit(".", maxsplit=1)[1] - import_package = "betterproto.lib.google.protobuf" - import_alias = safe_snake_case(import_package) - imports.add(f"import {import_package} as {import_alias}") - return f"{import_alias}.{type_name}" - - if "." in type_name: - # This is imported from another package. No need - # to use a forward ref and we need to add the import. - parts = type_name.split(".") - parts[-1] = stringcase.pascalcase(parts[-1]) - imports.add(f"from .{'.'.join(parts[:-2])} import {parts[-2]}") - type_name = f"{parts[-2]}.{parts[-1]}" - - return type_name - - def py_type( package: str, imports: set, diff --git a/betterproto/tests/__init__.py b/betterproto/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/betterproto/tests/inputs/config.py b/betterproto/tests/inputs/config.py index 7ed40f5..2525d8f 100644 --- a/betterproto/tests/inputs/config.py +++ b/betterproto/tests/inputs/config.py @@ -8,7 +8,6 @@ tests = { "import_circular_dependency", # failing because of other bugs now "import_packages_same_name", # 25 "oneof_enum", # 63 - "googletypes_service_returns_empty", # 9 "casing_message_field_uppercase", # 11 "namespace_keywords", # 70 "namespace_builtin_types", # 53 @@ -22,4 +21,5 @@ services = { "service", "import_service_input_message", "googletypes_service_returns_empty", + "googletypes_service_returns_googletype", } diff --git a/betterproto/tests/inputs/googletypes_response/test_googletypes_response.py b/betterproto/tests/inputs/googletypes_response/test_googletypes_response.py index fb2152b..02fa193 100644 --- a/betterproto/tests/inputs/googletypes_response/test_googletypes_response.py +++ b/betterproto/tests/inputs/googletypes_response/test_googletypes_response.py @@ -1,6 +1,6 @@ from typing import Any, Callable, Optional -import google.protobuf.wrappers_pb2 as wrappers +import betterproto.lib.google.protobuf as protobuf import pytest from betterproto.tests.mocks import MockChannel @@ -9,15 +9,15 @@ from betterproto.tests.output_betterproto.googletypes_response.googletypes_respo ) test_cases = [ - (TestStub.get_double, wrappers.DoubleValue, 2.5), - (TestStub.get_float, wrappers.FloatValue, 2.5), - (TestStub.get_int64, wrappers.Int64Value, -64), - (TestStub.get_u_int64, wrappers.UInt64Value, 64), - (TestStub.get_int32, wrappers.Int32Value, -32), - (TestStub.get_u_int32, wrappers.UInt32Value, 32), - (TestStub.get_bool, wrappers.BoolValue, True), - (TestStub.get_string, wrappers.StringValue, "string"), - (TestStub.get_bytes, wrappers.BytesValue, bytes(0xFF)[0:4]), + (TestStub.get_double, protobuf.DoubleValue, 2.5), + (TestStub.get_float, protobuf.FloatValue, 2.5), + (TestStub.get_int64, protobuf.Int64Value, -64), + (TestStub.get_u_int64, protobuf.UInt64Value, 64), + (TestStub.get_int32, protobuf.Int32Value, -32), + (TestStub.get_u_int32, protobuf.UInt32Value, 32), + (TestStub.get_bool, protobuf.BoolValue, True), + (TestStub.get_string, protobuf.StringValue, "string"), + (TestStub.get_bytes, protobuf.BytesValue, bytes(0xFF)[0:4]), ] diff --git a/betterproto/tests/test_get_ref_type.py b/betterproto/tests/test_get_ref_type.py new file mode 100644 index 0000000..9635356 --- /dev/null +++ b/betterproto/tests/test_get_ref_type.py @@ -0,0 +1,82 @@ +import pytest + +from ..compile.importing import get_ref_type + + +@pytest.mark.parametrize( + ["google_type", "expected_name", "expected_import"], + [ + ( + ".google.protobuf.Empty", + "betterproto_lib_google_protobuf.Empty", + "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + ), + ( + ".google.protobuf.Struct", + "betterproto_lib_google_protobuf.Struct", + "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + ), + ( + ".google.protobuf.ListValue", + "betterproto_lib_google_protobuf.ListValue", + "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + ), + ( + ".google.protobuf.Value", + "betterproto_lib_google_protobuf.Value", + "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + ), + ], +) +def test_import_google_wellknown_types_non_wrappers( + google_type: str, expected_name: str, expected_import: str +): + imports = set() + name = get_ref_type(package="", imports=imports, type_name=google_type) + + assert name == expected_name + assert imports.__contains__(expected_import) + + +@pytest.mark.parametrize( + ["google_type", "expected_name"], + [ + (".google.protobuf.DoubleValue", "Optional[float]"), + (".google.protobuf.FloatValue", "Optional[float]"), + (".google.protobuf.Int32Value", "Optional[int]"), + (".google.protobuf.Int64Value", "Optional[int]"), + (".google.protobuf.UInt32Value", "Optional[int]"), + (".google.protobuf.UInt64Value", "Optional[int]"), + (".google.protobuf.BoolValue", "Optional[bool]"), + (".google.protobuf.StringValue", "Optional[str]"), + (".google.protobuf.BytesValue", "Optional[bytes]"), + ], +) +def test_importing_google_wrappers_unwraps_them(google_type: str, expected_name: str): + imports = set() + name = get_ref_type(package="", imports=imports, type_name=google_type) + + assert name == expected_name + assert imports == set() + + +@pytest.mark.parametrize( + ["google_type", "expected_name"], + [ + (".google.protobuf.DoubleValue", "betterproto_lib_google_protobuf.DoubleValue"), + (".google.protobuf.FloatValue", "betterproto_lib_google_protobuf.FloatValue"), + (".google.protobuf.Int32Value", "betterproto_lib_google_protobuf.Int32Value"), + (".google.protobuf.Int64Value", "betterproto_lib_google_protobuf.Int64Value"), + (".google.protobuf.UInt32Value", "betterproto_lib_google_protobuf.UInt32Value"), + (".google.protobuf.UInt64Value", "betterproto_lib_google_protobuf.UInt64Value"), + (".google.protobuf.BoolValue", "betterproto_lib_google_protobuf.BoolValue"), + (".google.protobuf.StringValue", "betterproto_lib_google_protobuf.StringValue"), + (".google.protobuf.BytesValue", "betterproto_lib_google_protobuf.BytesValue"), + ], +) +def test_importing_google_wrappers_without_unwrapping( + google_type: str, expected_name: str +): + name = get_ref_type(package="", imports=set(), type_name=google_type, unwrap=False) + + assert name == expected_name From ab9857b5fd429eba9a3d0051b97ca60161ea5ae0 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 19:15:44 +0200 Subject: [PATCH 11/18] Add test-case for service that returns google protobuf values --- .../googletypes_service_returns_googletype.proto | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 betterproto/tests/inputs/googletypes_service_returns_googletype/googletypes_service_returns_googletype.proto diff --git a/betterproto/tests/inputs/googletypes_service_returns_googletype/googletypes_service_returns_googletype.proto b/betterproto/tests/inputs/googletypes_service_returns_googletype/googletypes_service_returns_googletype.proto new file mode 100644 index 0000000..49b2a55 --- /dev/null +++ b/betterproto/tests/inputs/googletypes_service_returns_googletype/googletypes_service_returns_googletype.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// Tests that imports are generated correctly when returning Google well-known types + +service Test { + rpc GetEmpty (RequestMessage) returns (google.protobuf.Empty); + rpc GetStruct (RequestMessage) returns (google.protobuf.Struct); + rpc GetListValue (RequestMessage) returns (google.protobuf.ListValue); + rpc GetValue (RequestMessage) returns (google.protobuf.Value); +} + +message RequestMessage { +} \ No newline at end of file From 973d68a154219943a3d14d73e84cd4418e5c0ff2 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Fri, 29 May 2020 19:16:10 +0200 Subject: [PATCH 12/18] Add missing field to MockChannel to prevent warnings while testing --- betterproto/tests/mocks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/betterproto/tests/mocks.py b/betterproto/tests/mocks.py index cd0efaa..9042f78 100644 --- a/betterproto/tests/mocks.py +++ b/betterproto/tests/mocks.py @@ -8,6 +8,7 @@ class MockChannel(Channel): def __init__(self, responses=None) -> None: self.responses = responses if responses else [] self.requests = [] + self._loop = None def request(self, route, cardinality, request, response_type, **kwargs): self.requests.append( From cb00273257a98a92dbde7b1b0057f6aee0776ce4 Mon Sep 17 00:00:00 2001 From: Bouke Versteegh Date: Mon, 1 Jun 2020 23:11:20 +0200 Subject: [PATCH 13/18] Fix name PROTOBUF_OPTS -> BETTERPROTO_OPTS --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 57c380b..cb8c0a0 100644 --- a/README.md +++ b/README.md @@ -360,12 +360,12 @@ $ pipenv run test Betterproto includes compiled versions for Google's well-known types at [betterproto/lib/google](betterproto/lib/google). Be sure to regenerate these files when modifying the plugin output format, and validate by running the tests. -Normally, the plugin does not compile any references to `google.protobuf`, since they are pre-compiled. To force compilation of `google.protobuf`, set this environment variable: `PROTOBUF_OPTS=INCLUDE_GOOGLE`. +Normally, the plugin does not compile any references to `google.protobuf`, since they are pre-compiled. To force compilation of `google.protobuf`, set this environment variable: `BETTERPROTO_OPTS=INCLUDE_GOOGLE`. Assuming your `google.protobuf` source files (included with all releases of `protoc`) are located in `/usr/local/include`, you can regenerate them as follows: ```sh -export PROTOBUF_OPTS=INCLUDE_GOOGLE +export BETTERPROTO_OPTS=INCLUDE_GOOGLE protoc --plugin=protoc-gen-custom=betterproto/plugin.py --custom_out=betterproto/lib -I/usr/local/include/ /usr/local/include/google/protobuf/*.proto ``` From ff14948a4eb9a2cc9f6b041c44cec02877dc2d31 Mon Sep 17 00:00:00 2001 From: Bouke Versteegh Date: Mon, 1 Jun 2020 23:24:04 +0200 Subject: [PATCH 14/18] Use raw string for regex Co-authored-by: nat --- betterproto/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 5780240..6fc92b6 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -198,7 +198,7 @@ def generate_code(request, response): field_wraps = "" match_wrapper = re.match( - "\\.google\\.protobuf\\.(.+)Value", f.type_name + r"\.google\.protobuf\.(.+)Value", f.type_name ) if match_wrapper: wrapped_type = "TYPE_" + match_wrapper.group(1).upper() From 7ecf3fe0e69f0c5df27198301731ee1d89697176 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Wed, 3 Jun 2020 23:51:19 +0200 Subject: [PATCH 15/18] Add comment to explain unusual import location --- betterproto/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/betterproto/__init__.py b/betterproto/__init__.py index 3fd4a58..2ef7164 100644 --- a/betterproto/__init__.py +++ b/betterproto/__init__.py @@ -941,6 +941,7 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Any]: return (field_name, getattr(message, field_name)) +# Circular import workaround: google.protobuf depends on base classes defined above. from .lib.google.protobuf import Duration, Timestamp From 919b0a6a7d2bb3faf934d250085877501c60a36f Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Wed, 3 Jun 2020 23:53:36 +0200 Subject: [PATCH 16/18] Check if betterproto has wrapper support in idiomatic way --- betterproto/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 6fc92b6..85557bc 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -202,7 +202,7 @@ def generate_code(request, response): ) if match_wrapper: wrapped_type = "TYPE_" + match_wrapper.group(1).upper() - if wrapped_type in dir(betterproto): + if hasattr(betterproto, wrapped_type): field_wraps = f"betterproto.{wrapped_type}" map_types = None From d31f90be6b034130136a60309e244ec1147d0d78 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Thu, 4 Jun 2020 00:11:22 +0200 Subject: [PATCH 17/18] Combine circular imports --- betterproto/__init__.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/betterproto/__init__.py b/betterproto/__init__.py index 2ef7164..5d901be 100644 --- a/betterproto/__init__.py +++ b/betterproto/__init__.py @@ -942,7 +942,19 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Any]: # Circular import workaround: google.protobuf depends on base classes defined above. -from .lib.google.protobuf import Duration, Timestamp +from .lib.google.protobuf import ( + Duration, + Timestamp, + BoolValue, + BytesValue, + DoubleValue, + FloatValue, + Int32Value, + Int64Value, + StringValue, + UInt32Value, + UInt64Value, +) class _Duration(Duration): @@ -1092,16 +1104,3 @@ class ServiceStub(ABC): await stream.send_message(request, end=True) async for message in stream: yield message - - -from .lib.google.protobuf import ( - BoolValue, - BytesValue, - DoubleValue, - FloatValue, - Int32Value, - Int64Value, - StringValue, - UInt32Value, - UInt64Value, -) From f7769a19d1d083c22435448aaf973f38905de712 Mon Sep 17 00:00:00 2001 From: boukeversteegh Date: Sat, 6 Jun 2020 12:51:37 +0200 Subject: [PATCH 18/18] Pass betterproto option using custom_opt instead of environment variable --- README.md | 10 +++++++--- betterproto/plugin.py | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cb8c0a0..385d1ea 100644 --- a/README.md +++ b/README.md @@ -360,13 +360,17 @@ $ pipenv run test Betterproto includes compiled versions for Google's well-known types at [betterproto/lib/google](betterproto/lib/google). Be sure to regenerate these files when modifying the plugin output format, and validate by running the tests. -Normally, the plugin does not compile any references to `google.protobuf`, since they are pre-compiled. To force compilation of `google.protobuf`, set this environment variable: `BETTERPROTO_OPTS=INCLUDE_GOOGLE`. +Normally, the plugin does not compile any references to `google.protobuf`, since they are pre-compiled. To force compilation of `google.protobuf`, use the option `--custom_opt=INCLUDE_GOOGLE`. Assuming your `google.protobuf` source files (included with all releases of `protoc`) are located in `/usr/local/include`, you can regenerate them as follows: ```sh -export BETTERPROTO_OPTS=INCLUDE_GOOGLE -protoc --plugin=protoc-gen-custom=betterproto/plugin.py --custom_out=betterproto/lib -I/usr/local/include/ /usr/local/include/google/protobuf/*.proto +protoc \ + --plugin=protoc-gen-custom=betterproto/plugin.py \ + --custom_opt=INCLUDE_GOOGLE \ + --custom_out=betterproto/lib \ + -I /usr/local/include/ \ + /usr/local/include/google/protobuf/*.proto ``` diff --git a/betterproto/plugin.py b/betterproto/plugin.py index 85557bc..e300318 100755 --- a/betterproto/plugin.py +++ b/betterproto/plugin.py @@ -120,8 +120,7 @@ def get_comment(proto_file, path: List[int], indent: int = 4) -> str: def generate_code(request, response): - plugin_options = os.environ.get("BETTERPROTO_OPTS") - plugin_options = plugin_options.split(" ") if plugin_options else [] + plugin_options = request.parameter.split(",") if request.parameter else [] env = jinja2.Environment( trim_blocks=True,