Add support for streaming delimited messages (#529)
* Add support for streaming delimited messages This allows developers to easily dump and load multiple messages from a stream in a way that is compatible with official protobuf implementations (such as Java's `MessageLite#writeDelimitedTo(...)`). * Add Java compatibility tests for streaming These tests stream data such as messages to output files, have a Java binary read them and then write them back using the `protobuf-java` functions, and then read them back in on the Python side to check that the returned data is as expected. This checks that the official Java implementation (and so any other matching implementations) can properly parse outputs from Betterproto, and vice-versa, ensuring compatibility in these functions between the two. * Replace `xxxxableBuffer` with `SupportsXxxx`
This commit is contained in:
Joshua Leivers
GitHub
2
tests/streams/delimited_messages.in
Normal file
2
tests/streams/delimited_messages.in
Normal file
@@ -0,0 +1,2 @@
|
||||
<18><><EFBFBD>:bTesting<18><><EFBFBD>:bTesting
|
||||
|
||||
38
tests/streams/java/.gitignore
vendored
Normal file
38
tests/streams/java/.gitignore
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
### Output ###
|
||||
target/
|
||||
!.mvn/wrapper/maven-wrapper.jar
|
||||
!**/src/main/**/target/
|
||||
!**/src/test/**/target/
|
||||
dependency-reduced-pom.xml
|
||||
MANIFEST.MF
|
||||
|
||||
### IntelliJ IDEA ###
|
||||
.idea/
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
|
||||
### Eclipse ###
|
||||
.apt_generated
|
||||
.classpath
|
||||
.factorypath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.sts4-cache
|
||||
|
||||
### NetBeans ###
|
||||
/nbproject/private/
|
||||
/nbbuild/
|
||||
/dist/
|
||||
/nbdist/
|
||||
/.nb-gradle/
|
||||
build/
|
||||
!**/src/main/**/build/
|
||||
!**/src/test/**/build/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
94
tests/streams/java/pom.xml
Normal file
94
tests/streams/java/pom.xml
Normal file
@@ -0,0 +1,94 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>betterproto</groupId>
|
||||
<artifactId>compatibility-test</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<protobuf.version>3.23.4</protobuf.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<extensions>
|
||||
<extension>
|
||||
<groupId>kr.motd.maven</groupId>
|
||||
<artifactId>os-maven-plugin</artifactId>
|
||||
<version>1.7.1</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.5.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>betterproto.CompatibilityTest</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<addClasspath>true</addClasspath>
|
||||
<mainClass>betterproto.CompatibilityTest</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.xolstice.maven.plugins</groupId>
|
||||
<artifactId>protobuf-maven-plugin</artifactId>
|
||||
<version>0.6.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<protocArtifact>
|
||||
com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
|
||||
</protocArtifact>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<finalName>${project.artifactId}</finalName>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,41 @@
|
||||
package betterproto;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class CompatibilityTest {
|
||||
public static void main(String[] args) throws IOException {
|
||||
if (args.length < 2)
|
||||
throw new RuntimeException("Attempted to run without the required arguments.");
|
||||
else if (args.length > 2)
|
||||
throw new RuntimeException(
|
||||
"Attempted to run with more than the expected number of arguments (>1).");
|
||||
|
||||
Tests tests = new Tests(args[1]);
|
||||
|
||||
switch (args[0]) {
|
||||
case "single_varint":
|
||||
tests.testSingleVarint();
|
||||
break;
|
||||
|
||||
case "multiple_varints":
|
||||
tests.testMultipleVarints();
|
||||
break;
|
||||
|
||||
case "single_message":
|
||||
tests.testSingleMessage();
|
||||
break;
|
||||
|
||||
case "multiple_messages":
|
||||
tests.testMultipleMessages();
|
||||
break;
|
||||
|
||||
case "infinite_messages":
|
||||
tests.testInfiniteMessages();
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new RuntimeException(
|
||||
"Attempted to run with unknown argument '" + args[0] + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
115
tests/streams/java/src/main/java/betterproto/Tests.java
Normal file
115
tests/streams/java/src/main/java/betterproto/Tests.java
Normal file
@@ -0,0 +1,115 @@
|
||||
package betterproto;
|
||||
|
||||
import betterproto.nested.NestedOuterClass;
|
||||
import betterproto.oneof.Oneof;
|
||||
|
||||
import com.google.protobuf.CodedInputStream;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
public class Tests {
|
||||
String path;
|
||||
|
||||
public Tests(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public void testSingleVarint() throws IOException {
|
||||
// Read in the Python-generated single varint file
|
||||
FileInputStream inputStream = new FileInputStream(path + "/py_single_varint.out");
|
||||
CodedInputStream codedInput = CodedInputStream.newInstance(inputStream);
|
||||
|
||||
int value = codedInput.readUInt32();
|
||||
|
||||
inputStream.close();
|
||||
|
||||
// Write the value back to a file
|
||||
FileOutputStream outputStream = new FileOutputStream(path + "/java_single_varint.out");
|
||||
CodedOutputStream codedOutput = CodedOutputStream.newInstance(outputStream);
|
||||
|
||||
codedOutput.writeUInt32NoTag(value);
|
||||
|
||||
codedOutput.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
public void testMultipleVarints() throws IOException {
|
||||
// Read in the Python-generated multiple varints file
|
||||
FileInputStream inputStream = new FileInputStream(path + "/py_multiple_varints.out");
|
||||
CodedInputStream codedInput = CodedInputStream.newInstance(inputStream);
|
||||
|
||||
int value1 = codedInput.readUInt32();
|
||||
int value2 = codedInput.readUInt32();
|
||||
long value3 = codedInput.readUInt64();
|
||||
|
||||
inputStream.close();
|
||||
|
||||
// Write the values back to a file
|
||||
FileOutputStream outputStream = new FileOutputStream(path + "/java_multiple_varints.out");
|
||||
CodedOutputStream codedOutput = CodedOutputStream.newInstance(outputStream);
|
||||
|
||||
codedOutput.writeUInt32NoTag(value1);
|
||||
codedOutput.writeUInt64NoTag(value2);
|
||||
codedOutput.writeUInt64NoTag(value3);
|
||||
|
||||
codedOutput.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
public void testSingleMessage() throws IOException {
|
||||
// Read in the Python-generated single message file
|
||||
FileInputStream inputStream = new FileInputStream(path + "/py_single_message.out");
|
||||
CodedInputStream codedInput = CodedInputStream.newInstance(inputStream);
|
||||
|
||||
Oneof.Test message = Oneof.Test.parseFrom(codedInput);
|
||||
|
||||
inputStream.close();
|
||||
|
||||
// Write the message back to a file
|
||||
FileOutputStream outputStream = new FileOutputStream(path + "/java_single_message.out");
|
||||
CodedOutputStream codedOutput = CodedOutputStream.newInstance(outputStream);
|
||||
|
||||
message.writeTo(codedOutput);
|
||||
|
||||
codedOutput.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
public void testMultipleMessages() throws IOException {
|
||||
// Read in the Python-generated multi-message file
|
||||
FileInputStream inputStream = new FileInputStream(path + "/py_multiple_messages.out");
|
||||
|
||||
Oneof.Test oneof = Oneof.Test.parseDelimitedFrom(inputStream);
|
||||
NestedOuterClass.Test nested = NestedOuterClass.Test.parseDelimitedFrom(inputStream);
|
||||
|
||||
inputStream.close();
|
||||
|
||||
// Write the messages back to a file
|
||||
FileOutputStream outputStream = new FileOutputStream(path + "/java_multiple_messages.out");
|
||||
|
||||
oneof.writeDelimitedTo(outputStream);
|
||||
nested.writeDelimitedTo(outputStream);
|
||||
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
public void testInfiniteMessages() throws IOException {
|
||||
// Read in as many messages as are present in the Python-generated file and write them back
|
||||
FileInputStream inputStream = new FileInputStream(path + "/py_infinite_messages.out");
|
||||
FileOutputStream outputStream = new FileOutputStream(path + "/java_infinite_messages.out");
|
||||
|
||||
Oneof.Test current = Oneof.Test.parseDelimitedFrom(inputStream);
|
||||
while (current != null) {
|
||||
current.writeDelimitedTo(outputStream);
|
||||
current = Oneof.Test.parseDelimitedFrom(inputStream);
|
||||
}
|
||||
|
||||
inputStream.close();
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
27
tests/streams/java/src/main/proto/betterproto/nested.proto
Normal file
27
tests/streams/java/src/main/proto/betterproto/nested.proto
Normal file
@@ -0,0 +1,27 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package nested;
|
||||
option java_package = "betterproto.nested";
|
||||
|
||||
// A test message with a nested message inside of it.
|
||||
message Test {
|
||||
// This is the nested type.
|
||||
message Nested {
|
||||
// Stores a simple counter.
|
||||
int32 count = 1;
|
||||
}
|
||||
// This is the nested enum.
|
||||
enum Msg {
|
||||
NONE = 0;
|
||||
THIS = 1;
|
||||
}
|
||||
|
||||
Nested nested = 1;
|
||||
Sibling sibling = 2;
|
||||
Sibling sibling2 = 3;
|
||||
Msg msg = 4;
|
||||
}
|
||||
|
||||
message Sibling {
|
||||
int32 foo = 1;
|
||||
}
|
||||
19
tests/streams/java/src/main/proto/betterproto/oneof.proto
Normal file
19
tests/streams/java/src/main/proto/betterproto/oneof.proto
Normal file
@@ -0,0 +1,19 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package oneof;
|
||||
option java_package = "betterproto.oneof";
|
||||
|
||||
message Test {
|
||||
oneof foo {
|
||||
int32 pitied = 1;
|
||||
string pitier = 2;
|
||||
}
|
||||
|
||||
int32 just_a_regular_field = 3;
|
||||
|
||||
oneof bar {
|
||||
int32 drinks = 11;
|
||||
string bar_name = 12;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user