Merge branch 'master' into michael-sayapin/master

This commit is contained in:
Bouke Versteegh
2020-07-04 11:23:42 +02:00
committed by GitHub
41 changed files with 870 additions and 238 deletions

View File

@@ -12,12 +12,12 @@ inputs/
## Test case directory structure
Each testcase has a `<name>.proto` file with a message called `Test`, a matching `.json` file and optionally a custom test file called `test_*.py`.
Each testcase has a `<name>.proto` file with a message called `Test`, and optionally a matching `.json` file and a custom test called `test_*.py`.
```bash
bool/
bool.proto
bool.json
bool.json # optional
test_bool.py # optional
```
@@ -61,21 +61,22 @@ def test_value():
The following tests are automatically executed for all cases:
- [x] Can the generated python code imported?
- [x] Can the generated python code be imported?
- [x] Can the generated message class be instantiated?
- [x] Is the generated code compatible with the Google's `grpc_tools.protoc` implementation?
- _when `.json` is present_
## Running the tests
- `pipenv run generate`
This generates
- `pipenv run generate`
This generates:
- `betterproto/tests/output_betterproto` &mdash; *the plugin generated python classes*
- `betterproto/tests/output_reference` &mdash; *reference implementation classes*
- `pipenv run test`
## Intentionally Failing tests
The standard test suite includes tests that fail by intention. These tests document known bugs and missing features that are intended to be corrented in the future.
The standard test suite includes tests that fail by intention. These tests document known bugs and missing features that are intended to be corrected in the future.
When running `pytest`, they show up as `x` or `X` in the test results.

View File

@@ -1,4 +1,4 @@
from betterproto.tests.output_betterproto.bool.bool import Test
from betterproto.tests.output_betterproto.bool import Test
def test_value():

View File

@@ -10,6 +10,7 @@ message Test {
int32 camelCase = 1;
my_enum snake_case = 2;
snake_case_message snake_case_message = 3;
int32 UPPERCASE = 4;
}
message snake_case_message {

View File

@@ -1,5 +1,5 @@
import betterproto.tests.output_betterproto.casing.casing as casing
from betterproto.tests.output_betterproto.casing.casing import Test
import betterproto.tests.output_betterproto.casing as casing
from betterproto.tests.output_betterproto.casing import Test
def test_message_attributes():
@@ -8,6 +8,7 @@ def test_message_attributes():
message, "snake_case_message"
), "snake_case field name is same in python"
assert hasattr(message, "camel_case"), "CamelCase field is snake_case in python"
assert hasattr(message, "uppercase"), "UPPERCASE field is lowercase in python"
def test_message_casing():

View File

@@ -1,5 +0,0 @@
{
"UPPERCASE": 10,
"UPPERCASE_V2": 10,
"UPPER_CAMEL_CASE": 10
}

View File

@@ -1,6 +1,4 @@
from betterproto.tests.output_betterproto.casing_message_field_uppercase.casing_message_field_uppercase import (
Test,
)
from betterproto.tests.output_betterproto.casing_message_field_uppercase import Test
def test_message_casing():

View File

@@ -1,19 +1,15 @@
# Test cases that are expected to fail, e.g. unimplemented features or bug-fixes.
# Remove from list when fixed.
tests = {
"import_root_sibling", # 61
"import_child_package_from_package", # 58
"import_root_package_from_child", # 60
"import_parent_package_from_child", # 59
"import_circular_dependency", # failing because of other bugs now
"import_packages_same_name", # 25
xfail = {
"import_circular_dependency",
"oneof_enum", # 63
"casing_message_field_uppercase", # 11
"namespace_keywords", # 70
"namespace_builtin_types", # 53
"googletypes_struct", # 9
"googletypes_value", # 9
"enum_skipped_value", # 93
"import_capitalized_package",
"example", # This is the example in the readme. Not a test.
}
services = {

View File

@@ -0,0 +1,8 @@
syntax = "proto3";
package hello;
// Greeting represents a message you can tell a user.
message Greeting {
string message = 1;
}

View File

@@ -4,9 +4,7 @@ import betterproto.lib.google.protobuf as protobuf
import pytest
from betterproto.tests.mocks import MockChannel
from betterproto.tests.output_betterproto.googletypes_response.googletypes_response import (
TestStub,
)
from betterproto.tests.output_betterproto.googletypes_response import TestStub
test_cases = [
(TestStub.get_double, protobuf.DoubleValue, 2.5),

View File

@@ -1,7 +1,7 @@
import pytest
from betterproto.tests.mocks import MockChannel
from betterproto.tests.output_betterproto.googletypes_response_embedded.googletypes_response_embedded import (
from betterproto.tests.output_betterproto.googletypes_response_embedded import (
Output,
TestStub,
)

View File

@@ -0,0 +1,8 @@
syntax = "proto3";
package Capitalized;
message Message {
}

View File

@@ -0,0 +1,9 @@
syntax = "proto3";
import "capitalized.proto";
// Tests that we can import from a package with a capital name, that looks like a nested type, but isn't.
message Test {
Capitalized.Message message = 1;
}

View File

@@ -3,9 +3,9 @@ syntax = "proto3";
import "root.proto";
import "other.proto";
// This test-case verifies that future implementations will support circular dependencies in the generated python files.
// This test-case verifies support for circular dependencies in the generated python files.
//
// This becomes important when generating 1 python file/module per package, rather than 1 file per proto file.
// This is important because we generate 1 python file/module per package, rather than 1 file per proto file.
//
// Scenario:
//
@@ -24,5 +24,5 @@ import "other.proto";
// (root: Test & RootPackageMessage) <-------> (other: OtherPackageMessage)
message Test {
RootPackageMessage message = 1;
other.OtherPackageMessage other =2;
other.OtherPackageMessage other = 2;
}

View File

@@ -0,0 +1,6 @@
syntax = "proto3";
package cousin.cousin_subpackage;
message CousinMessage {
}

View File

@@ -0,0 +1,11 @@
syntax = "proto3";
package test.subpackage;
import "cousin.proto";
// Verify that we can import message unrelated to us
message Test {
cousin.cousin_subpackage.CousinMessage message = 1;
}

View File

@@ -0,0 +1,6 @@
syntax = "proto3";
package cousin.subpackage;
message CousinMessage {
}

View File

@@ -0,0 +1,11 @@
syntax = "proto3";
package test.subpackage;
import "cousin.proto";
// Verify that we can import a message unrelated to us, in a subpackage with the same name as us.
message Test {
cousin.subpackage.CousinMessage message = 1;
}

View File

@@ -0,0 +1,11 @@
syntax = "proto3";
package child;
import "root.proto";
// Verify that we can import root message from child package
message Test {
RootMessage message = 1;
}

View File

@@ -1,11 +0,0 @@
syntax = "proto3";
import "root.proto";
package child;
// Tests generated imports when a message inside a child-package refers to a message defined in the root.
message Test {
RootMessage message = 1;
}

View File

@@ -1,7 +1,7 @@
import pytest
from betterproto.tests.mocks import MockChannel
from betterproto.tests.output_betterproto.import_service_input_message.import_service_input_message import (
from betterproto.tests.output_betterproto.import_service_input_message import (
RequestResponse,
TestStub,
)

View File

@@ -15,4 +15,4 @@ message Test {
message Sibling {
int32 foo = 1;
}
}

View File

@@ -0,0 +1,19 @@
syntax = "proto3";
import "package.proto";
message Game {
message Player {
enum Race {
human = 0;
orc = 1;
}
}
}
message Test {
Game game = 1;
Game.Player GamePlayer = 2;
Game.Player.Race GamePlayerRace = 3;
equipment.Weapon Weapon = 4;
}

View File

@@ -0,0 +1,7 @@
syntax = "proto3";
package equipment;
message Weapon {
}

View File

@@ -1,10 +1,10 @@
{
"root": {
"top": {
"name": "double-nested",
"parent": {
"child": [{"foo": "hello"}],
"enumChild": ["A"],
"rootParentChild": [{"a": "hello"}],
"middle": {
"bottom": [{"foo": "hello"}],
"enumBottom": ["A"],
"topMiddleBottom": [{"a": "hello"}],
"bar": true
}
}

View File

@@ -1,26 +1,26 @@
syntax = "proto3";
message Test {
message Root {
message Parent {
message RootParentChild {
message Top {
message Middle {
message TopMiddleBottom {
string a = 1;
}
enum EnumChild{
enum EnumBottom{
A = 0;
B = 1;
}
message Child {
message Bottom {
string foo = 1;
}
reserved 1;
repeated Child child = 2;
repeated EnumChild enumChild=3;
repeated RootParentChild rootParentChild=4;
repeated Bottom bottom = 2;
repeated EnumBottom enumBottom=3;
repeated TopMiddleBottom topMiddleBottom=4;
bool bar = 5;
}
string name = 1;
Parent parent = 2;
Middle middle = 2;
}
Root root = 1;
Top top = 1;
}

View File

@@ -1,5 +1,5 @@
import betterproto
from betterproto.tests.output_betterproto.oneof.oneof import Test
from betterproto.tests.output_betterproto.oneof import Test
from betterproto.tests.util import get_test_case_json_data

View File

@@ -1,7 +1,7 @@
import pytest
import betterproto
from betterproto.tests.output_betterproto.oneof_enum.oneof_enum import (
from betterproto.tests.output_betterproto.oneof_enum import (
Move,
Signal,
Test,

View File

@@ -1,7 +1,5 @@
syntax = "proto3";
package ref;
import "repeatedmessage.proto";
message Test {

View File

@@ -0,0 +1,125 @@
import pytest
from betterproto.casing import camel_case, pascal_case, snake_case
@pytest.mark.parametrize(
["value", "expected"],
[
("", ""),
("a", "A"),
("foobar", "Foobar"),
("fooBar", "FooBar"),
("FooBar", "FooBar"),
("foo.bar", "FooBar"),
("foo_bar", "FooBar"),
("FOOBAR", "Foobar"),
("FOOBar", "FooBar"),
("UInt32", "UInt32"),
("FOO_BAR", "FooBar"),
("FOOBAR1", "Foobar1"),
("FOOBAR_1", "Foobar1"),
("FOO1BAR2", "Foo1Bar2"),
("foo__bar", "FooBar"),
("_foobar", "Foobar"),
("foobaR", "FoobaR"),
("foo~bar", "FooBar"),
("foo:bar", "FooBar"),
("1foobar", "1Foobar"),
],
)
def test_pascal_case(value, expected):
actual = pascal_case(value, strict=True)
assert actual == expected, f"{value} => {expected} (actual: {actual})"
@pytest.mark.parametrize(
["value", "expected"],
[
("", ""),
("a", "a"),
("foobar", "foobar"),
("fooBar", "fooBar"),
("FooBar", "fooBar"),
("foo.bar", "fooBar"),
("foo_bar", "fooBar"),
("FOOBAR", "foobar"),
("FOO_BAR", "fooBar"),
("FOOBAR1", "foobar1"),
("FOOBAR_1", "foobar1"),
("FOO1BAR2", "foo1Bar2"),
("foo__bar", "fooBar"),
("_foobar", "foobar"),
("foobaR", "foobaR"),
("foo~bar", "fooBar"),
("foo:bar", "fooBar"),
("1foobar", "1Foobar"),
],
)
def test_camel_case_strict(value, expected):
actual = camel_case(value, strict=True)
assert actual == expected, f"{value} => {expected} (actual: {actual})"
@pytest.mark.parametrize(
["value", "expected"],
[
("foo_bar", "fooBar"),
("FooBar", "fooBar"),
("foo__bar", "foo_Bar"),
("foo__Bar", "foo__Bar"),
],
)
def test_camel_case_not_strict(value, expected):
actual = camel_case(value, strict=False)
assert actual == expected, f"{value} => {expected} (actual: {actual})"
@pytest.mark.parametrize(
["value", "expected"],
[
("", ""),
("a", "a"),
("foobar", "foobar"),
("fooBar", "foo_bar"),
("FooBar", "foo_bar"),
("foo.bar", "foo_bar"),
("foo_bar", "foo_bar"),
("foo_Bar", "foo_bar"),
("FOOBAR", "foobar"),
("FOOBar", "foo_bar"),
("UInt32", "u_int32"),
("FOO_BAR", "foo_bar"),
("FOOBAR1", "foobar1"),
("FOOBAR_1", "foobar_1"),
("FOOBAR_123", "foobar_123"),
("FOO1BAR2", "foo1_bar2"),
("foo__bar", "foo_bar"),
("_foobar", "foobar"),
("foobaR", "fooba_r"),
("foo~bar", "foo_bar"),
("foo:bar", "foo_bar"),
("1foobar", "1_foobar"),
("GetUInt64", "get_u_int64"),
],
)
def test_snake_case_strict(value, expected):
actual = snake_case(value)
assert actual == expected, f"{value} => {expected} (actual: {actual})"
@pytest.mark.parametrize(
["value", "expected"],
[
("fooBar", "foo_bar"),
("FooBar", "foo_bar"),
("foo_Bar", "foo__bar"),
("foo__bar", "foo__bar"),
("FOOBar", "foo_bar"),
("__foo", "__foo"),
("GetUInt64", "get_u_int64"),
],
)
def test_snake_case_not_strict(value, expected):
actual = snake_case(value, strict=False)
assert actual == expected, f"{value} => {expected} (actual: {actual})"

View File

@@ -1,6 +1,6 @@
import pytest
from ..compile.importing import get_ref_type
from ..compile.importing import get_type_reference, parse_source_type_name
@pytest.mark.parametrize(
@@ -28,14 +28,16 @@ from ..compile.importing import get_ref_type
),
],
)
def test_import_google_wellknown_types_non_wrappers(
def test_reference_google_wellknown_types_non_wrappers(
google_type: str, expected_name: str, expected_import: str
):
imports = set()
name = get_ref_type(package="", imports=imports, type_name=google_type)
name = get_type_reference(package="", imports=imports, source_type=google_type)
assert name == expected_name
assert imports.__contains__(expected_import)
assert imports.__contains__(
expected_import
), f"{expected_import} not found in {imports}"
@pytest.mark.parametrize(
@@ -52,9 +54,11 @@ def test_import_google_wellknown_types_non_wrappers(
(".google.protobuf.BytesValue", "Optional[bytes]"),
],
)
def test_importing_google_wrappers_unwraps_them(google_type: str, expected_name: str):
def test_referenceing_google_wrappers_unwraps_them(
google_type: str, expected_name: str
):
imports = set()
name = get_ref_type(package="", imports=imports, type_name=google_type)
name = get_type_reference(package="", imports=imports, source_type=google_type)
assert name == expected_name
assert imports == set()
@@ -74,9 +78,238 @@ def test_importing_google_wrappers_unwraps_them(google_type: str, expected_name:
(".google.protobuf.BytesValue", "betterproto_lib_google_protobuf.BytesValue"),
],
)
def test_importing_google_wrappers_without_unwrapping(
def test_referenceing_google_wrappers_without_unwrapping(
google_type: str, expected_name: str
):
name = get_ref_type(package="", imports=set(), type_name=google_type, unwrap=False)
name = get_type_reference(
package="", imports=set(), source_type=google_type, unwrap=False
)
assert name == expected_name
def test_reference_child_package_from_package():
imports = set()
name = get_type_reference(
package="package", imports=imports, source_type="package.child.Message"
)
assert imports == {"from . import child"}
assert name == "child.Message"
def test_reference_child_package_from_root():
imports = set()
name = get_type_reference(package="", imports=imports, source_type="child.Message")
assert imports == {"from . import child"}
assert name == "child.Message"
def test_reference_camel_cased():
imports = set()
name = get_type_reference(
package="", imports=imports, source_type="child_package.example_message"
)
assert imports == {"from . import child_package"}
assert name == "child_package.ExampleMessage"
def test_reference_nested_child_from_root():
imports = set()
name = get_type_reference(
package="", imports=imports, source_type="nested.child.Message"
)
assert imports == {"from .nested import child as nested_child"}
assert name == "nested_child.Message"
def test_reference_deeply_nested_child_from_root():
imports = set()
name = get_type_reference(
package="", imports=imports, source_type="deeply.nested.child.Message"
)
assert imports == {"from .deeply.nested import child as deeply_nested_child"}
assert name == "deeply_nested_child.Message"
def test_reference_deeply_nested_child_from_package():
imports = set()
name = get_type_reference(
package="package",
imports=imports,
source_type="package.deeply.nested.child.Message",
)
assert imports == {"from .deeply.nested import child as deeply_nested_child"}
assert name == "deeply_nested_child.Message"
def test_reference_root_sibling():
imports = set()
name = get_type_reference(package="", imports=imports, source_type="Message")
assert imports == set()
assert name == '"Message"'
def test_reference_nested_siblings():
imports = set()
name = get_type_reference(package="foo", imports=imports, source_type="foo.Message")
assert imports == set()
assert name == '"Message"'
def test_reference_deeply_nested_siblings():
imports = set()
name = get_type_reference(
package="foo.bar", imports=imports, source_type="foo.bar.Message"
)
assert imports == set()
assert name == '"Message"'
def test_reference_parent_package_from_child():
imports = set()
name = get_type_reference(
package="package.child", imports=imports, source_type="package.Message"
)
assert imports == {"from ... import package as __package__"}
assert name == "__package__.Message"
def test_reference_parent_package_from_deeply_nested_child():
imports = set()
name = get_type_reference(
package="package.deeply.nested.child",
imports=imports,
source_type="package.deeply.nested.Message",
)
assert imports == {"from ... import nested as __nested__"}
assert name == "__nested__.Message"
def test_reference_ancestor_package_from_nested_child():
imports = set()
name = get_type_reference(
package="package.ancestor.nested.child",
imports=imports,
source_type="package.ancestor.Message",
)
assert imports == {"from .... import ancestor as ___ancestor__"}
assert name == "___ancestor__.Message"
def test_reference_root_package_from_child():
imports = set()
name = get_type_reference(
package="package.child", imports=imports, source_type="Message"
)
assert imports == {"from ... import Message as __Message__"}
assert name == "__Message__"
def test_reference_root_package_from_deeply_nested_child():
imports = set()
name = get_type_reference(
package="package.deeply.nested.child", imports=imports, source_type="Message"
)
assert imports == {"from ..... import Message as ____Message__"}
assert name == "____Message__"
def test_reference_unrelated_package():
imports = set()
name = get_type_reference(package="a", imports=imports, source_type="p.Message")
assert imports == {"from .. import p as _p__"}
assert name == "_p__.Message"
def test_reference_unrelated_nested_package():
imports = set()
name = get_type_reference(package="a.b", imports=imports, source_type="p.q.Message")
assert imports == {"from ...p import q as __p_q__"}
assert name == "__p_q__.Message"
def test_reference_unrelated_deeply_nested_package():
imports = set()
name = get_type_reference(
package="a.b.c.d", imports=imports, source_type="p.q.r.s.Message"
)
assert imports == {"from .....p.q.r import s as ____p_q_r_s__"}
assert name == "____p_q_r_s__.Message"
def test_reference_cousin_package():
imports = set()
name = get_type_reference(package="a.x", imports=imports, source_type="a.y.Message")
assert imports == {"from .. import y as _y__"}
assert name == "_y__.Message"
def test_reference_cousin_package_different_name():
imports = set()
name = get_type_reference(
package="test.package1", imports=imports, source_type="cousin.package2.Message"
)
assert imports == {"from ...cousin import package2 as __cousin_package2__"}
assert name == "__cousin_package2__.Message"
def test_reference_cousin_package_same_name():
imports = set()
name = get_type_reference(
package="test.package", imports=imports, source_type="cousin.package.Message"
)
assert imports == {"from ...cousin import package as __cousin_package__"}
assert name == "__cousin_package__.Message"
def test_reference_far_cousin_package():
imports = set()
name = get_type_reference(
package="a.x.y", imports=imports, source_type="a.b.c.Message"
)
assert imports == {"from ...b import c as __b_c__"}
assert name == "__b_c__.Message"
def test_reference_far_far_cousin_package():
imports = set()
name = get_type_reference(
package="a.x.y.z", imports=imports, source_type="a.b.c.d.Message"
)
assert imports == {"from ....b.c import d as ___b_c_d__"}
assert name == "___b_c_d__.Message"
@pytest.mark.parametrize(
["full_name", "expected_output"],
[
("package.SomeMessage.NestedType", ("package", "SomeMessage.NestedType")),
(".package.SomeMessage.NestedType", ("package", "SomeMessage.NestedType")),
(".service.ExampleRequest", ("service", "ExampleRequest")),
(".package.lower_case_message", ("package", "lower_case_message")),
],
)
def test_parse_field_type_name(full_name, expected_output):
assert parse_source_type_name(full_name) == expected_output

View File

@@ -3,6 +3,7 @@ import json
import os
import sys
from collections import namedtuple
from types import ModuleType
from typing import Set
import pytest
@@ -10,7 +11,12 @@ import pytest
import betterproto
from betterproto.tests.inputs import config as test_input_config
from betterproto.tests.mocks import MockChannel
from betterproto.tests.util import get_directories, get_test_case_json_data, inputs_path
from betterproto.tests.util import (
find_module,
get_directories,
get_test_case_json_data,
inputs_path,
)
# Force pure-python implementation instead of C++, otherwise imports
# break things because we can't properly reset the symbol database.
@@ -50,14 +56,17 @@ class TestCases:
test_cases = TestCases(
path=inputs_path,
services=test_input_config.services,
xfail=test_input_config.tests,
xfail=test_input_config.xfail,
)
plugin_output_package = "betterproto.tests.output_betterproto"
reference_output_package = "betterproto.tests.output_reference"
TestData = namedtuple("TestData", ["plugin_module", "reference_module", "json_data"])
TestData = namedtuple("TestData", "plugin_module, reference_module, json_data")
def module_has_entry_point(module: ModuleType):
return any(hasattr(module, attr) for attr in ["Test", "TestStub"])
@pytest.fixture
@@ -75,11 +84,19 @@ def test_data(request):
sys.path.append(reference_module_root)
plugin_module = importlib.import_module(f"{plugin_output_package}.{test_case_name}")
plugin_module_entry_point = find_module(plugin_module, module_has_entry_point)
if not plugin_module_entry_point:
raise Exception(
f"Test case {repr(test_case_name)} has no entry point. "
"Please add a proto message or service called Test and recompile."
)
yield (
TestData(
plugin_module=importlib.import_module(
f"{plugin_output_package}.{test_case_name}.{test_case_name}"
),
plugin_module=plugin_module_entry_point,
reference_module=lambda: importlib.import_module(
f"{reference_output_package}.{test_case_name}.{test_case_name}_pb2"
),

View File

@@ -1,7 +1,10 @@
import asyncio
import importlib
import os
import pathlib
from pathlib import Path
from typing import Generator, IO, Optional
from types import ModuleType
from typing import Callable, Generator, Optional
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
@@ -55,3 +58,35 @@ def get_test_case_json_data(test_case_name: str, json_file_name: Optional[str] =
with test_data_file_path.open("r") as fh:
return fh.read()
def find_module(
module: ModuleType, predicate: Callable[[ModuleType], bool]
) -> Optional[ModuleType]:
"""
Recursively search module tree for a module that matches the search predicate.
Assumes that the submodules are directories containing __init__.py.
Example:
# find module inside foo that contains Test
import foo
test_module = find_module(foo, lambda m: hasattr(m, 'Test'))
"""
if predicate(module):
return module
module_path = pathlib.Path(*module.__path__)
for sub in list(sub.parent for sub in module_path.glob("**/__init__.py")):
if sub == module_path:
continue
sub_module_path = sub.relative_to(module_path)
sub_module_name = ".".join(sub_module_path.parts)
sub_module = importlib.import_module(f".{sub_module_name}", module.__name__)
if predicate(sub_module):
return sub_module
return None