From 2996e1ddd22b4570d35f48a8f03a4f877d04ef19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 27 Jun 2023 10:53:33 +0200 Subject: [PATCH] Implement message set wire format (#836) Message set wire format is old and deprecated format used instead of extensions in ancient protos. This adds a new `GeneratedMessage` subclass `$_MessageSet` to override binary serialization and deserialization methods for message sets. Message set messages now extend `$_MessageSet` instead of `GeneratedMessage`. The new class is prefixed as `$_` to make the addition backwards compatible. --- protobuf/CHANGELOG.md | 2 + protobuf/lib/protobuf.dart | 1 + .../lib/src/protobuf/extension_registry.dart | 44 +++++- protobuf/lib/src/protobuf/message_set.dart | 143 ++++++++++++++++++ protobuf/pubspec.yaml | 2 +- protoc_plugin/Makefile | 1 + protoc_plugin/lib/src/message_generator.dart | 9 +- protoc_plugin/test/message_set_test.dart | 48 ++++++ protoc_plugin/test/protos/message_set.proto | 25 +++ 9 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 protobuf/lib/src/protobuf/message_set.dart create mode 100644 protoc_plugin/test/message_set_test.dart create mode 100644 protoc_plugin/test/protos/message_set.proto diff --git a/protobuf/CHANGELOG.md b/protobuf/CHANGELOG.md index e5ef26782..8ae0f018f 100644 --- a/protobuf/CHANGELOG.md +++ b/protobuf/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.0.1-dev + ## 3.0.0 * Require Dart `2.19`. diff --git a/protobuf/lib/protobuf.dart b/protobuf/lib/protobuf.dart index 5f73a14ed..63e6e3f77 100644 --- a/protobuf/lib/protobuf.dart +++ b/protobuf/lib/protobuf.dart @@ -40,6 +40,7 @@ part 'src/protobuf/field_type.dart'; part 'src/protobuf/generated_message.dart'; part 'src/protobuf/generated_service.dart'; part 'src/protobuf/json.dart'; +part 'src/protobuf/message_set.dart'; part 'src/protobuf/pb_list.dart'; part 'src/protobuf/pb_map.dart'; part 'src/protobuf/proto3_json.dart'; diff --git a/protobuf/lib/src/protobuf/extension_registry.dart b/protobuf/lib/src/protobuf/extension_registry.dart index 81b658ecb..997992732 100644 --- a/protobuf/lib/src/protobuf/extension_registry.dart +++ b/protobuf/lib/src/protobuf/extension_registry.dart @@ -104,14 +104,44 @@ T _reparseMessage( final messageUnknownFields = message._fieldSet._unknownFields; if (messageUnknownFields != null) { final codedBufferWriter = CodedBufferWriter(); - extensionRegistry._extensions[message.info_.qualifiedMessageName] - ?.forEach((tagNumber, extension) { - final unknownField = messageUnknownFields._fields[tagNumber]; - if (unknownField != null) { - unknownField.writeTo(tagNumber, codedBufferWriter); - ensureUnknownFields()._fields.remove(tagNumber); + + if (message is $_MessageSet) { + final itemList = messageUnknownFields._fields[_messageSetItemsTag]; + + final parsedItemList = UnknownFieldSetField(); + final unparsedItemList = UnknownFieldSetField(); + + if (itemList != null) { + for (final group in itemList.groups) { + final typeId = + group._fields[_messageSetItemTypeIdTag]!.varints[0].toInt(); + if (extensionRegistry.getExtension( + message.info_.qualifiedMessageName, typeId) == + null) { + unparsedItemList.addGroup(group); + } else { + parsedItemList.addGroup(group); + } + } + + parsedItemList.writeTo(_messageSetItemsTag, codedBufferWriter); + + if (unparsedItemList.groups.isEmpty) { + messageUnknownFields._fields.remove(_messageSetItemsTag); + } else { + messageUnknownFields._fields[_messageSetItemsTag] = unparsedItemList; + } } - }); + } else { + extensionRegistry._extensions[message.info_.qualifiedMessageName] + ?.forEach((tagNumber, extension) { + final unknownField = messageUnknownFields._fields[tagNumber]; + if (unknownField != null) { + unknownField.writeTo(tagNumber, codedBufferWriter); + ensureUnknownFields()._fields.remove(tagNumber); + } + }); + } final buffer = codedBufferWriter.toBuffer(); if (buffer.isNotEmpty) { diff --git a/protobuf/lib/src/protobuf/message_set.dart b/protobuf/lib/src/protobuf/message_set.dart new file mode 100644 index 000000000..2c397b5eb --- /dev/null +++ b/protobuf/lib/src/protobuf/message_set.dart @@ -0,0 +1,143 @@ +// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +part of '../../protobuf.dart'; + +const _messageSetItemsTag = 1; +const _messageSetItemTypeIdTag = 2; +const _messageSetItemMessageTag = 3; + +/// Overrides binary serialization and deserialization methods to implement the +/// message set binary format. +/// +/// Message set format is very old and only used in Google. When a message has +/// this option: +/// +/// ``` +/// option message_set_wire_format = true; +/// ``` +/// +/// The plugin extends the generated message class with this class instead of +/// [GeneratedMessage]. +/// +/// @nodoc +abstract class $_MessageSet extends GeneratedMessage { + @override + void writeToCodedBufferWriter(CodedBufferWriter output) { + final extensions = _fieldSet._ensureExtensions(); + + for (final ext in extensions._values.entries) { + final typeId = ext.key; + final message = ext.value as GeneratedMessage; + + output._writeTag(_messageSetItemsTag, WIRETYPE_START_GROUP); + output._writeTag(_messageSetItemTypeIdTag, WIRETYPE_VARINT); + output._writeVarint32(typeId); + output._writeTag(_messageSetItemMessageTag, WIRETYPE_LENGTH_DELIMITED); + final mark = output._startLengthDelimited(); + message.writeToCodedBufferWriter(output); + output._endLengthDelimited(mark); + output._writeTag(_messageSetItemsTag, WIRETYPE_END_GROUP); + } + + final unknownFields = _fieldSet._unknownFields; + if (unknownFields != null) { + unknownFields.writeToCodedBufferWriter(output); + } + } + + @override + void mergeFromCodedBufferReader(CodedBufferReader input, + [ExtensionRegistry extensionRegistry = ExtensionRegistry.EMPTY]) { + // Parse items. The field for the items looks like: + // + // repeated Item items = 1; + // + // Since message sets are compatible with proto1 items can't be packed. + outer: + while (true) { + final tag = input.readTag(); + final wireType = getTagWireType(tag); + final tagNumber = getTagFieldNumber(tag); + + if (tag == 0) { + break; + } + + if (tagNumber != _messageSetItemsTag) { + throw UnsupportedError( + 'Invalid message set (type = $wireType, tag = $tagNumber)'); + } + + // Parse an item. An item is a message with two fields: + // + // message Item { + // int32 type_id = 2; + // Message message = 3; + // } + // + // We can see the fields in any order, so loop until parsing both fields. + int? typeId; + List? message; + while (true) { + final tag = input.readTag(); + final tagNumber = getTagFieldNumber(tag); + + if (tag == 0) { + break; + } + + if (tagNumber == _messageSetItemTypeIdTag) { + typeId = input.readInt32(); + if (message != null) { + _parseExtension(typeId, message, extensionRegistry); + typeId = null; + message = null; + continue outer; + } + } else if (tagNumber == _messageSetItemMessageTag) { + message = input.readBytes(); + if (typeId != null) { + _parseExtension(typeId, message, extensionRegistry); + typeId = null; + message = null; + continue outer; + } + } else { + throw UnsupportedError('Invalid message set item (tag = $tagNumber)'); + } + } + } + } + + @override + void mergeFromBuffer(List input, + [ExtensionRegistry extensionRegistry = ExtensionRegistry.EMPTY]) { + mergeFromCodedBufferReader(CodedBufferReader(input), extensionRegistry); + } + + void _parseExtension( + int typeId, List message, ExtensionRegistry extensionRegistry) { + final ext = + extensionRegistry.getExtension(info_.qualifiedMessageName, typeId); + if (ext == null) { + final messageItem = UnknownFieldSet(); + messageItem.addField(_messageSetItemTypeIdTag, + UnknownFieldSetField()..varints.add(Int64(typeId))); + messageItem.addField(_messageSetItemMessageTag, + UnknownFieldSetField()..lengthDelimited.add(message)); + + final itemListField = + _fieldSet._ensureUnknownFields().getField(_messageSetItemsTag) ?? + UnknownFieldSetField(); + itemListField.addGroup(messageItem); + + _fieldSet + ._ensureUnknownFields() + .addField(_messageSetItemsTag, itemListField); + } else { + setExtension(ext, ext.subBuilder!()..mergeFromBuffer(message)); + } + } +} diff --git a/protobuf/pubspec.yaml b/protobuf/pubspec.yaml index 696e7947c..5ea7dedfd 100644 --- a/protobuf/pubspec.yaml +++ b/protobuf/pubspec.yaml @@ -1,5 +1,5 @@ name: protobuf -version: 3.0.0 +version: 3.0.1-dev description: >- Runtime library for protocol buffers support. Use with package:protoc_plugin to generate dart code for your '.proto' files. diff --git a/protoc_plugin/Makefile b/protoc_plugin/Makefile index ec693bdca..ef2ad25fe 100644 --- a/protoc_plugin/Makefile +++ b/protoc_plugin/Makefile @@ -43,6 +43,7 @@ TEST_PROTO_LIST = \ map_api2 \ map_enum_value \ map_field \ + message_set \ mixins \ multiple_files_test \ nested_any \ diff --git a/protoc_plugin/lib/src/message_generator.dart b/protoc_plugin/lib/src/message_generator.dart index 886100c97..8727a31a6 100644 --- a/protoc_plugin/lib/src/message_generator.dart +++ b/protoc_plugin/lib/src/message_generator.dart @@ -311,8 +311,15 @@ class MessageGenerator extends ProtobufContainer { ? ', toProto3Json: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, ' 'fromProto3Json: $mixinImportPrefix.${mixin!.name}.fromProto3JsonHelper' : ''; + + final String extendedClass; + if (_descriptor.options.messageSetWireFormat) { + extendedClass = '\$_MessageSet'; + } else { + extendedClass = 'GeneratedMessage'; + } out.addAnnotatedBlock( - 'class $classname extends $protobufImportPrefix.GeneratedMessage$mixinClause {', + 'class $classname extends $protobufImportPrefix.$extendedClass$mixinClause {', '}', [ NamedLocation( name: classname, fieldPathSegment: fieldPath, start: 'class '.length) diff --git a/protoc_plugin/test/message_set_test.dart b/protoc_plugin/test/message_set_test.dart new file mode 100644 index 000000000..1745bc57b --- /dev/null +++ b/protoc_plugin/test/message_set_test.dart @@ -0,0 +1,48 @@ +import 'package:protobuf/protobuf.dart'; +import 'package:test/test.dart'; + +import '../out/protos/message_set.pb.dart'; + +void main() { + final encoded = [ + 0xda, 0x07, 0x0e, 0x0b, 0x10, 0xc8, 0xa6, 0x6b, 0x1a, // + 0x06, 0x08, 0x7b, 0x12, 0x02, 0x68, 0x69, 0x0c + ]; + + test('Simple message set field', () { + final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension); + final msg = TestMessage.fromBuffer(encoded, registry); + final extensionValue = msg.info + .getExtension(TestMessage.messageSetExtension) as ExtensionMessage; + expect(extensionValue.a, 123); + expect(extensionValue.b, 'hi'); + expect(msg.writeToBuffer(), encoded); + }); + + test('Parse as unknown fields and serialize', () { + final msg = TestMessage.fromBuffer(encoded); + expect(msg.writeToBuffer(), encoded); + }); + + test('Reparse with extensions (nested message)', () { + final msg = TestMessage.fromBuffer(encoded); + final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension); + final reparsedInfo = registry.reparseMessage(msg.info); + final extensionValue = reparsedInfo + .getExtension(TestMessage.messageSetExtension) as ExtensionMessage; + expect(extensionValue.a, 123); + expect(extensionValue.b, 'hi'); + expect(reparsedInfo.unknownFields.isEmpty, true); + }); + + test('Reparse with extensions (top-level message)', () { + final msg = TestMessage.fromBuffer(encoded); + final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension); + final reparsedMsg = registry.reparseMessage(msg); + final extensionValue = reparsedMsg.info + .getExtension(TestMessage.messageSetExtension) as ExtensionMessage; + expect(extensionValue.a, 123); + expect(extensionValue.b, 'hi'); + expect(msg.unknownFields.isEmpty, true); + }); +} diff --git a/protoc_plugin/test/protos/message_set.proto b/protoc_plugin/test/protos/message_set.proto new file mode 100644 index 000000000..10f4d1e7e --- /dev/null +++ b/protoc_plugin/test/protos/message_set.proto @@ -0,0 +1,25 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +syntax = "proto2"; + +message MessageSet { + option message_set_wire_format = true; + + extensions 4 to 524999999; + extensions 525000000 to max; +} + +message TestMessage { + extend MessageSet { + optional ExtensionMessage message_set_extension = 1758024; + } + + optional MessageSet info = 123; +} + +message ExtensionMessage { + optional int32 a = 1; + optional string b = 2; +}