Improve generate script

- Fix issue with __pycache__ dirs getting picked up
- parallelise code generation with asyncio for 3x speedup
- silence protoc output unless -v option is supplied
- Use pathlib ;)

betterproto/tests/util.py

@@ -1,23 +1,24 @@
+import asyncio
 import os
-import subprocess
-from typing import Generator
+from pathlib import Path
+from typing import Generator, IO, Optional
 
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 
-root_path = os.path.dirname(os.path.realpath(__file__))
-inputs_path = os.path.join(root_path, "inputs")
-output_path_reference = os.path.join(root_path, "output_reference")
-output_path_betterproto = os.path.join(root_path, "output_betterproto")
+root_path = Path(__file__).resolve().parent
+inputs_path = root_path.joinpath("inputs")
+output_path_reference = root_path.joinpath("output_reference")
+output_path_betterproto = root_path.joinpath("output_betterproto")
 
 if os.name == "nt":
-    plugin_path = os.path.join(root_path, "..", "plugin.bat")
+    plugin_path = root_path.joinpath("..", "plugin.bat").resolve()
 else:
-    plugin_path = os.path.join(root_path, "..", "plugin.py")
+    plugin_path = root_path.joinpath("..", "plugin.py").resolve()
 
 
-def get_files(path, end: str) -> Generator[str, None, None]:
+def get_files(path, suffix: str) -> Generator[str, None, None]:
     for r, dirs, files in os.walk(path):
-        for filename in [f for f in files if f.endswith(end)]:
+        for filename in [f for f in files if f.endswith(suffix)]:
             yield os.path.join(r, filename)
 
 
@@ -27,36 +28,30 @@ def get_directories(path):
             yield directory
 
 
-def relative(file: str, path: str):
-    return os.path.join(os.path.dirname(file), path)
-
-
-def read_relative(file: str, path: str):
-    with open(relative(file, path)) as fh:
-        return fh.read()
-
-
-def protoc_plugin(path: str, output_dir: str) -> subprocess.CompletedProcess:
-    return subprocess.run(
+async def protoc_plugin(path: str, output_dir: str):
+    proc = await asyncio.create_subprocess_shell(
         f"protoc --plugin=protoc-gen-custom={plugin_path} --custom_out={output_dir} --proto_path={path} {path}/*.proto",
-        shell=True,
-        check=True,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
     )
+    return (*(await proc.communicate()), proc.returncode)
 
 
-def protoc_reference(path: str, output_dir: str):
-    subprocess.run(
+async def protoc_reference(path: str, output_dir: str):
+    proc = await asyncio.create_subprocess_shell(
         f"protoc --python_out={output_dir} --proto_path={path} {path}/*.proto",
-        shell=True,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
     )
+    return (*(await proc.communicate()), proc.returncode)
 
 
-def get_test_case_json_data(test_case_name, json_file_name=None):
+def get_test_case_json_data(test_case_name: str, json_file_name: Optional[str] = None):
     test_data_file_name = json_file_name if json_file_name else f"{test_case_name}.json"
-    test_data_file_path = os.path.join(inputs_path, test_case_name, test_data_file_name)
+    test_data_file_path = inputs_path.joinpath(test_case_name, test_data_file_name)
 
-    if not os.path.exists(test_data_file_path):
+    if not test_data_file_path.exists():
         return None
 
-    with open(test_data_file_path) as fh:
+    with test_data_file_path.open("r") as fh:
         return fh.read()