diff --git a/panel/models/comm_manager.ts b/panel/models/comm_manager.ts
index 644f7502cc..fd1f046d4c 100644
--- a/panel/models/comm_manager.ts
+++ b/panel/models/comm_manager.ts
@@ -4,9 +4,10 @@ import {View} from "@bokehjs/core/view"
 import {Model} from "@bokehjs/model"
 import {Message} from "@bokehjs/protocol/message"
 import {Receiver} from "@bokehjs/protocol/receiver"
+import {Buffer} from "@bokehjs/core/serialization/buffer"
 import type {Patch, DocumentChangedEvent} from "@bokehjs/document"
 import {isArray, isPlainObject} from "@bokehjs/core/util/types"
-import {values, size} from "@bokehjs/core/util/object"
+import {keys} from "@bokehjs/core/util/object"
 
 export const comm_settings: any = {
   debounce: true,
@@ -103,21 +104,26 @@ export class CommManager extends Model {
     }
   }
 
-  protected _extract_buffers(value: unknown, buffers: ArrayBuffer[]): void {
+  protected _extract_buffers(value: unknown, buffers: ArrayBuffer[]): any {
     if (isArray(value)) {
       for (const val of value) {
         this._extract_buffers(val, buffers)
       }
+    } else if (value instanceof Map) {
+      for (const key of value.keys()) {
+        const v = value.get(key)
+        this._extract_buffers(v, buffers)
+      }
+    } else if (value instanceof Buffer) {
+      const {buffer} = value
+      const id = buffers.length
+      buffers.push(buffer)
+      return {id}
     } else if (isPlainObject(value)) {
-      if (size(value) == 1 && value.buffer instanceof ArrayBuffer) {
-        const {buffer} = value
-        delete value.buffer
-        const id = buffers.length
-        value.id = id
-        buffers.push(buffer)
-      } else {
-        for (const val of values(value)) {
-          this._extract_buffers(val, buffers)
+      for (const key of keys(value)) {
+        const replaced = this._extract_buffers(value[key], buffers)
+        if (replaced != null) {
+          value[key] = replaced
         }
       }
     }