Implement message set wire format (#836)

Message set wire format is old and deprecated format used instead of extensions
in ancient protos.

This adds a new `GeneratedMessage` subclass `$_MessageSet` to override binary
serialization and deserialization methods for message sets. Message set messages
now extend `$_MessageSet` instead of `GeneratedMessage`.

The new class is prefixed as `$_` to make the addition backwards compatible.

protobuf/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 3.0.1-dev
+
 ## 3.0.0
 
 * Require Dart `2.19`.

protobuf/lib/protobuf.dart

@@ -40,6 +40,7 @@ part 'src/protobuf/field_type.dart';
 part 'src/protobuf/generated_message.dart';
 part 'src/protobuf/generated_service.dart';
 part 'src/protobuf/json.dart';
+part 'src/protobuf/message_set.dart';
 part 'src/protobuf/pb_list.dart';
 part 'src/protobuf/pb_map.dart';
 part 'src/protobuf/proto3_json.dart';

protobuf/lib/src/protobuf/extension_registry.dart

@@ -104,14 +104,44 @@ T _reparseMessage<T extends GeneratedMessage>(
   final messageUnknownFields = message._fieldSet._unknownFields;
   if (messageUnknownFields != null) {
     final codedBufferWriter = CodedBufferWriter();
-    extensionRegistry._extensions[message.info_.qualifiedMessageName]
-        ?.forEach((tagNumber, extension) {
-      final unknownField = messageUnknownFields._fields[tagNumber];
-      if (unknownField != null) {
-        unknownField.writeTo(tagNumber, codedBufferWriter);
-        ensureUnknownFields()._fields.remove(tagNumber);
+
+    if (message is $_MessageSet) {
+      final itemList = messageUnknownFields._fields[_messageSetItemsTag];
+
+      final parsedItemList = UnknownFieldSetField();
+      final unparsedItemList = UnknownFieldSetField();
+
+      if (itemList != null) {
+        for (final group in itemList.groups) {
+          final typeId =
+              group._fields[_messageSetItemTypeIdTag]!.varints[0].toInt();
+          if (extensionRegistry.getExtension(
+                  message.info_.qualifiedMessageName, typeId) ==
+              null) {
+            unparsedItemList.addGroup(group);
+          } else {
+            parsedItemList.addGroup(group);
+          }
+        }
+
+        parsedItemList.writeTo(_messageSetItemsTag, codedBufferWriter);
+
+        if (unparsedItemList.groups.isEmpty) {
+          messageUnknownFields._fields.remove(_messageSetItemsTag);
+        } else {
+          messageUnknownFields._fields[_messageSetItemsTag] = unparsedItemList;
+        }
       }
-    });
+    } else {
+      extensionRegistry._extensions[message.info_.qualifiedMessageName]
+          ?.forEach((tagNumber, extension) {
+        final unknownField = messageUnknownFields._fields[tagNumber];
+        if (unknownField != null) {
+          unknownField.writeTo(tagNumber, codedBufferWriter);
+          ensureUnknownFields()._fields.remove(tagNumber);
+        }
+      });
+    }
 
     final buffer = codedBufferWriter.toBuffer();
     if (buffer.isNotEmpty) {

protobuf/lib/src/protobuf/message_set.dart

@@ -0,0 +1,143 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+part of '../../protobuf.dart';
+
+const _messageSetItemsTag = 1;
+const _messageSetItemTypeIdTag = 2;
+const _messageSetItemMessageTag = 3;
+
+/// Overrides binary serialization and deserialization methods to implement the
+/// message set binary format.
+///
+/// Message set format is very old and only used in Google. When a message has
+/// this option:
+///
+/// ```
+/// option message_set_wire_format = true;
+/// ```
+///
+/// The plugin extends the generated message class with this class instead of
+/// [GeneratedMessage].
+///
+/// @nodoc
+abstract class $_MessageSet extends GeneratedMessage {
+  @override
+  void writeToCodedBufferWriter(CodedBufferWriter output) {
+    final extensions = _fieldSet._ensureExtensions();
+
+    for (final ext in extensions._values.entries) {
+      final typeId = ext.key;
+      final message = ext.value as GeneratedMessage;
+
+      output._writeTag(_messageSetItemsTag, WIRETYPE_START_GROUP);
+      output._writeTag(_messageSetItemTypeIdTag, WIRETYPE_VARINT);
+      output._writeVarint32(typeId);
+      output._writeTag(_messageSetItemMessageTag, WIRETYPE_LENGTH_DELIMITED);
+      final mark = output._startLengthDelimited();
+      message.writeToCodedBufferWriter(output);
+      output._endLengthDelimited(mark);
+      output._writeTag(_messageSetItemsTag, WIRETYPE_END_GROUP);
+    }
+
+    final unknownFields = _fieldSet._unknownFields;
+    if (unknownFields != null) {
+      unknownFields.writeToCodedBufferWriter(output);
+    }
+  }
+
+  @override
+  void mergeFromCodedBufferReader(CodedBufferReader input,
+      [ExtensionRegistry extensionRegistry = ExtensionRegistry.EMPTY]) {
+    // Parse items. The field for the items looks like:
+    //
+    //   repeated Item items = 1;
+    //
+    // Since message sets are compatible with proto1 items can't be packed.
+    outer:
+    while (true) {
+      final tag = input.readTag();
+      final wireType = getTagWireType(tag);
+      final tagNumber = getTagFieldNumber(tag);
+
+      if (tag == 0) {
+        break;
+      }
+
+      if (tagNumber != _messageSetItemsTag) {
+        throw UnsupportedError(
+            'Invalid message set (type = $wireType, tag = $tagNumber)');
+      }
+
+      // Parse an item. An item is a message with two fields:
+      //
+      //   message Item {
+      //     int32 type_id = 2;
+      //     Message message = 3;
+      //   }
+      //
+      // We can see the fields in any order, so loop until parsing both fields.
+      int? typeId;
+      List<int>? message;
+      while (true) {
+        final tag = input.readTag();
+        final tagNumber = getTagFieldNumber(tag);
+
+        if (tag == 0) {
+          break;
+        }
+
+        if (tagNumber == _messageSetItemTypeIdTag) {
+          typeId = input.readInt32();
+          if (message != null) {
+            _parseExtension(typeId, message, extensionRegistry);
+            typeId = null;
+            message = null;
+            continue outer;
+          }
+        } else if (tagNumber == _messageSetItemMessageTag) {
+          message = input.readBytes();
+          if (typeId != null) {
+            _parseExtension(typeId, message, extensionRegistry);
+            typeId = null;
+            message = null;
+            continue outer;
+          }
+        } else {
+          throw UnsupportedError('Invalid message set item (tag = $tagNumber)');
+        }
+      }
+    }
+  }
+
+  @override
+  void mergeFromBuffer(List<int> input,
+      [ExtensionRegistry extensionRegistry = ExtensionRegistry.EMPTY]) {
+    mergeFromCodedBufferReader(CodedBufferReader(input), extensionRegistry);
+  }
+
+  void _parseExtension(
+      int typeId, List<int> message, ExtensionRegistry extensionRegistry) {
+    final ext =
+        extensionRegistry.getExtension(info_.qualifiedMessageName, typeId);
+    if (ext == null) {
+      final messageItem = UnknownFieldSet();
+      messageItem.addField(_messageSetItemTypeIdTag,
+          UnknownFieldSetField()..varints.add(Int64(typeId)));
+      messageItem.addField(_messageSetItemMessageTag,
+          UnknownFieldSetField()..lengthDelimited.add(message));
+
+      final itemListField =
+          _fieldSet._ensureUnknownFields().getField(_messageSetItemsTag) ??
+              UnknownFieldSetField();
+      itemListField.addGroup(messageItem);
+
+      _fieldSet
+          ._ensureUnknownFields()
+          .addField(_messageSetItemsTag, itemListField);
+    } else {
+      setExtension(ext, ext.subBuilder!()..mergeFromBuffer(message));
+    }
+  }
+}

protobuf/pubspec.yaml

@@ -1,5 +1,5 @@
 name: protobuf
-version: 3.0.0
+version: 3.0.1-dev
 description: >-
   Runtime library for protocol buffers support. Use with package:protoc_plugin
   to generate dart code for your '.proto' files.

protoc_plugin/Makefile

@@ -43,6 +43,7 @@ TEST_PROTO_LIST = \
 	map_api2 \
 	map_enum_value \
 	map_field \
+	message_set \
 	mixins \
 	multiple_files_test \
 	nested_any \

protoc_plugin/lib/src/message_generator.dart

@@ -311,8 +311,15 @@ class MessageGenerator extends ProtobufContainer {
         ? ', toProto3Json: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, '
             'fromProto3Json: $mixinImportPrefix.${mixin!.name}.fromProto3JsonHelper'
         : '';
+
+    final String extendedClass;
+    if (_descriptor.options.messageSetWireFormat) {
+      extendedClass = '\$_MessageSet';
+    } else {
+      extendedClass = 'GeneratedMessage';
+    }
     out.addAnnotatedBlock(
-        'class $classname extends $protobufImportPrefix.GeneratedMessage$mixinClause {',
+        'class $classname extends $protobufImportPrefix.$extendedClass$mixinClause {',
         '}', [
       NamedLocation(
           name: classname, fieldPathSegment: fieldPath, start: 'class '.length)

protoc_plugin/test/message_set_test.dart

@@ -0,0 +1,48 @@
+import 'package:protobuf/protobuf.dart';
+import 'package:test/test.dart';
+
+import '../out/protos/message_set.pb.dart';
+
+void main() {
+  final encoded = [
+    0xda, 0x07, 0x0e, 0x0b, 0x10, 0xc8, 0xa6, 0x6b, 0x1a, //
+    0x06, 0x08, 0x7b, 0x12, 0x02, 0x68, 0x69, 0x0c
+  ];
+
+  test('Simple message set field', () {
+    final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension);
+    final msg = TestMessage.fromBuffer(encoded, registry);
+    final extensionValue = msg.info
+        .getExtension(TestMessage.messageSetExtension) as ExtensionMessage;
+    expect(extensionValue.a, 123);
+    expect(extensionValue.b, 'hi');
+    expect(msg.writeToBuffer(), encoded);
+  });
+
+  test('Parse as unknown fields and serialize', () {
+    final msg = TestMessage.fromBuffer(encoded);
+    expect(msg.writeToBuffer(), encoded);
+  });
+
+  test('Reparse with extensions (nested message)', () {
+    final msg = TestMessage.fromBuffer(encoded);
+    final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension);
+    final reparsedInfo = registry.reparseMessage(msg.info);
+    final extensionValue = reparsedInfo
+        .getExtension(TestMessage.messageSetExtension) as ExtensionMessage;
+    expect(extensionValue.a, 123);
+    expect(extensionValue.b, 'hi');
+    expect(reparsedInfo.unknownFields.isEmpty, true);
+  });
+
+  test('Reparse with extensions (top-level message)', () {
+    final msg = TestMessage.fromBuffer(encoded);
+    final registry = ExtensionRegistry()..add(TestMessage.messageSetExtension);
+    final reparsedMsg = registry.reparseMessage(msg);
+    final extensionValue = reparsedMsg.info
+        .getExtension(TestMessage.messageSetExtension) as ExtensionMessage;
+    expect(extensionValue.a, 123);
+    expect(extensionValue.b, 'hi');
+    expect(msg.unknownFields.isEmpty, true);
+  });
+}

protoc_plugin/test/protos/message_set.proto

@@ -0,0 +1,25 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+syntax = "proto2";
+
+message MessageSet {
+  option message_set_wire_format = true;
+
+  extensions 4 to 524999999;
+  extensions 525000000 to max;
+}
+
+message TestMessage {
+  extend MessageSet {
+    optional ExtensionMessage message_set_extension = 1758024;
+  }
+
+  optional MessageSet info = 123;
+}
+
+message ExtensionMessage {
+  optional int32 a = 1;
+  optional string b = 2;
+}