From 51973d37c9ecf3133e10e1c9f7c41880800bd34c Mon Sep 17 00:00:00 2001
From: neuecc <ils@neue.cc>
Date: Fri, 24 Jan 2025 10:46:10 +0900
Subject: [PATCH 1/3] remove ByteListFormatter to keep binary compatibility for
 List<byte>

---
 .../Formatters/PrimitiveFormatter.cs          | 80 +++----------------
 .../Formatters/PrimitiveFormatter.tt          | 77 +-----------------
 src/MessagePack/Resolvers/BuiltinResolver.cs  |  1 -
 3 files changed, 13 insertions(+), 145 deletions(-)

diff --git a/src/MessagePack/Formatters/PrimitiveFormatter.cs b/src/MessagePack/Formatters/PrimitiveFormatter.cs
index 771471966..9e5fbb6dd 100644
--- a/src/MessagePack/Formatters/PrimitiveFormatter.cs
+++ b/src/MessagePack/Formatters/PrimitiveFormatter.cs
@@ -173,6 +173,7 @@ public void Serialize(ref MessagePackWriter writer, List<Int16>? value, MessageP
             return list;
         }
     }
+
 #endif
 
     public sealed class Int32Formatter : IMessagePackFormatter<Int32>
@@ -329,6 +330,7 @@ public void Serialize(ref MessagePackWriter writer, List<Int32>? value, MessageP
             return list;
         }
     }
+
 #endif
 
     public sealed class Int64Formatter : IMessagePackFormatter<Int64>
@@ -485,6 +487,7 @@ public void Serialize(ref MessagePackWriter writer, List<Int64>? value, MessageP
             return list;
         }
     }
+
 #endif
 
     public sealed class UInt16Formatter : IMessagePackFormatter<UInt16>
@@ -641,6 +644,7 @@ public void Serialize(ref MessagePackWriter writer, List<UInt16>? value, Message
             return list;
         }
     }
+
 #endif
 
     public sealed class UInt32Formatter : IMessagePackFormatter<UInt32>
@@ -797,6 +801,7 @@ public void Serialize(ref MessagePackWriter writer, List<UInt32>? value, Message
             return list;
         }
     }
+
 #endif
 
     public sealed class UInt64Formatter : IMessagePackFormatter<UInt64>
@@ -953,6 +958,7 @@ public void Serialize(ref MessagePackWriter writer, List<UInt64>? value, Message
             return list;
         }
     }
+
 #endif
 
     public sealed class SingleFormatter : IMessagePackFormatter<Single>
@@ -1109,6 +1115,7 @@ public void Serialize(ref MessagePackWriter writer, List<Single>? value, Message
             return list;
         }
     }
+
 #endif
 
     public sealed class DoubleFormatter : IMessagePackFormatter<Double>
@@ -1265,6 +1272,7 @@ public void Serialize(ref MessagePackWriter writer, List<Double>? value, Message
             return list;
         }
     }
+
 #endif
 
     public sealed class BooleanFormatter : IMessagePackFormatter<Boolean>
@@ -1513,76 +1521,6 @@ public void Serialize(ref MessagePackWriter writer, Byte? value, MessagePackSeri
         }
     }
 
-#if NET8_0_OR_GREATER
-    public sealed class ByteListFormatter : IMessagePackFormatter<List<Byte>?>
-    {
-        public static readonly ByteListFormatter Instance = new ByteListFormatter();
-
-        private ByteListFormatter()
-        {
-        }
-
-        public void Serialize(ref MessagePackWriter writer, List<Byte>? value, MessagePackSerializerOptions options)
-        {
-            if (value == null)
-            {
-                writer.WriteNil();
-            }
-            else
-            {
-                writer.Write(CollectionsMarshal.AsSpan(value));
-            }
-        }
-
-        public List<Byte>? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
-        {
-            if (reader.NextMessagePackType == MessagePackType.Array)
-            {
-                int len = reader.ReadArrayHeader();
-                if (len == 0)
-                {
-                    return [];
-                }
-
-                var list = new List<byte>(len);
-                options.Security.DepthStep(ref reader);
-                try
-                {
-                    CollectionsMarshal.SetCount(list, len);
-                    var span = CollectionsMarshal.AsSpan(list);
-                    for (int i = 0; i < len; i++)
-                    {
-                        reader.CancellationToken.ThrowIfCancellationRequested();
-                        span[i] = reader.ReadByte();
-                    }
-                }
-                finally
-                {
-                    reader.Depth--;
-                }
-
-                return list;
-            }
-            else
-            {
-                var sequence = reader.ReadBytes();
-                if (sequence == null)
-                {
-                    return null;
-                }
-
-                int len = checked((int)sequence.Value.Length);
-                var list = new List<byte>(len);
-                CollectionsMarshal.SetCount(list, len);
-                var span = CollectionsMarshal.AsSpan(list);
-                sequence.Value.CopyTo(span);
-                return list;
-            }
-        }
-    }
-
-#endif
-
     public sealed class SByteFormatter : IMessagePackFormatter<SByte>
     {
         public static readonly SByteFormatter Instance = new SByteFormatter();
@@ -1737,6 +1675,7 @@ public void Serialize(ref MessagePackWriter writer, List<SByte>? value, MessageP
             return list;
         }
     }
+
 #endif
 
     public sealed class CharFormatter : IMessagePackFormatter<Char>
@@ -1893,6 +1832,7 @@ public void Serialize(ref MessagePackWriter writer, List<Char>? value, MessagePa
             return list;
         }
     }
+
 #endif
 
     public sealed class DateTimeFormatter : IMessagePackFormatter<DateTime>
diff --git a/src/MessagePack/Formatters/PrimitiveFormatter.tt b/src/MessagePack/Formatters/PrimitiveFormatter.tt
index 365951ce0..4d3b61a6f 100644
--- a/src/MessagePack/Formatters/PrimitiveFormatter.tt
+++ b/src/MessagePack/Formatters/PrimitiveFormatter.tt
@@ -95,79 +95,7 @@ namespace MessagePack.Formatters
             }
         }
     }
-<# if(t.Name == "Byte") { #>
-
-#if NET8_0_OR_GREATER
-    public sealed class <#= t.Name #>ListFormatter : IMessagePackFormatter<List<<#= t.Name #>>?>
-    {
-        public static readonly <#= t.Name #>ListFormatter Instance = new <#= t.Name #>ListFormatter();
-
-        private <#= t.Name #>ListFormatter()
-        {
-        }
-
-        public void Serialize(ref MessagePackWriter writer, List<<#= t.Name #>>? value, MessagePackSerializerOptions options)
-        {
-            if (value == null)
-            {
-                writer.WriteNil();
-            }
-            else
-            {
-                writer.Write(CollectionsMarshal.AsSpan(value));
-            }
-        }
-
-        public List<<#= t.Name #>>? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
-        {
-            if (reader.NextMessagePackType == MessagePackType.Array)
-            {
-                int len = reader.ReadArrayHeader();
-                if (len == 0)
-                {
-                    return [];
-                }
-
-                var list = new List<byte>(len);
-                options.Security.DepthStep(ref reader);
-                try
-                {
-                    CollectionsMarshal.SetCount(list, len);
-                    var span = CollectionsMarshal.AsSpan(list);
-                    for (int i = 0; i < len; i++)
-                    {
-                        reader.CancellationToken.ThrowIfCancellationRequested();
-                        span[i] = reader.ReadByte();
-                    }
-                }
-                finally
-                {
-                    reader.Depth--;
-                }
-
-                return list;
-            }
-            else
-            {
-                var sequence = reader.ReadBytes();
-                if (sequence == null)
-                {
-                    return null;
-                }
-
-                int len = checked((int)sequence.Value.Length);
-                var list = new List<byte>(len);
-                CollectionsMarshal.SetCount(list, len);
-                var span = CollectionsMarshal.AsSpan(list);
-                sequence.Value.CopyTo(span);
-                return list;
-            }
-        }
-    }
-
-#endif
-<# } #>
-<# else if(t.Name == "DateTime") { #>
+<# if(t.Name == "DateTime") { #>
 
     public sealed class <#= t.Name #>ArrayFormatter : IMessagePackFormatter<<#= t.Name #>[]?>
     {
@@ -360,7 +288,7 @@ namespace MessagePack.Formatters
     }
 #endif
 <# } #>
-<# else { #>
+<# else if(t.Name != "Byte") { #>
 
     public sealed class <#= t.Name #>ArrayFormatter : IMessagePackFormatter<<#= t.Name #>[]?>
     {
@@ -464,6 +392,7 @@ namespace MessagePack.Formatters
             return list;
         }
     }
+
 #endif
 <# } #>
 <# } #>
diff --git a/src/MessagePack/Resolvers/BuiltinResolver.cs b/src/MessagePack/Resolvers/BuiltinResolver.cs
index 00ba57521..84e404c06 100644
--- a/src/MessagePack/Resolvers/BuiltinResolver.cs
+++ b/src/MessagePack/Resolvers/BuiltinResolver.cs
@@ -131,7 +131,6 @@ internal static class BuiltinResolverGetFormatterHelper
             { typeof(List<Single>), SingleListFormatter.Instance },
             { typeof(List<Double>), DoubleListFormatter.Instance },
             { typeof(List<Boolean>), BooleanListFormatter.Instance },
-            { typeof(List<byte>), ByteListFormatter.Instance },
             { typeof(List<SByte>), SByteListFormatter.Instance },
             { typeof(List<Char>), CharListFormatter.Instance },
 #else

From 99ad5e9091c89fcb39e339fb87bec1f5d0ddaa74 Mon Sep 17 00:00:00 2001
From: neuecc <ils@neue.cc>
Date: Fri, 24 Jan 2025 12:03:53 +0900
Subject: [PATCH 2/3] add compatible list formatter

---
 .../Formatters/CollectionFormatter.cs         | 40 +++++++++++++++++++
 src/MessagePack/Resolvers/BuiltinResolver.cs  |  1 +
 tests/MessagePack.Tests/CollectionTest.cs     | 32 +++++++++++++++
 3 files changed, 73 insertions(+)

diff --git a/src/MessagePack/Formatters/CollectionFormatter.cs b/src/MessagePack/Formatters/CollectionFormatter.cs
index 623d3745e..d66515042 100644
--- a/src/MessagePack/Formatters/CollectionFormatter.cs
+++ b/src/MessagePack/Formatters/CollectionFormatter.cs
@@ -158,6 +158,46 @@ public ArraySegment<byte> Deserialize(ref MessagePackReader reader, MessagePackS
         }
     }
 
+    // special formatter to maintain compatibility due to bugs, https://github.com/MessagePack-CSharp/MessagePack-CSharp/issues/2134
+    public sealed class ByteListFormatter : IMessagePackFormatter<List<byte>?>
+    {
+        public static readonly ByteListFormatter Instance = new ByteListFormatter();
+
+        private static readonly ListFormatter<byte> InnerFormatter = new ListFormatter<byte>();
+
+        public void Serialize(ref MessagePackWriter writer, List<byte>? value, MessagePackSerializerOptions options)
+        {
+            InnerFormatter.Serialize(ref writer, value, options);
+        }
+
+        public List<byte>? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
+        {
+            if (reader.NextMessagePackType == MessagePackType.Array)
+            {
+                return InnerFormatter.Deserialize(ref reader, options);
+            }
+            else
+            {
+                var sequence = reader.ReadBytes();
+                if (sequence == null)
+                {
+                    return null;
+                }
+
+#if NET8_0_OR_GREATER
+                int len = checked((int)sequence.Value.Length);
+                var list = new List<byte>(len);
+                CollectionsMarshal.SetCount(list, len);
+                var span = CollectionsMarshal.AsSpan(list);
+                sequence.Value.CopyTo(span);
+                return list;
+#else
+                return new List<byte>(sequence.Value.ToArray());
+#endif
+            }
+        }
+    }
+
     public sealed class MemoryFormatter<T> : IMessagePackFormatter<Memory<T>>
     {
         public void Serialize(ref MessagePackWriter writer, Memory<T> value, MessagePackSerializerOptions options)
diff --git a/src/MessagePack/Resolvers/BuiltinResolver.cs b/src/MessagePack/Resolvers/BuiltinResolver.cs
index 84e404c06..d649292c1 100644
--- a/src/MessagePack/Resolvers/BuiltinResolver.cs
+++ b/src/MessagePack/Resolvers/BuiltinResolver.cs
@@ -149,6 +149,7 @@ internal static class BuiltinResolverGetFormatterHelper
 #endif
             { typeof(List<DateTime>), new ListFormatter<DateTime>() },
             { typeof(List<string>), new ListFormatter<string>() },
+            { typeof(List<Byte>), ByteListFormatter.Instance }, // special formatter to maintain compatibility due to bugs
 
             { typeof(object[]), new ArrayFormatter<object>() },
             { typeof(List<object>), new ListFormatter<object>() },
diff --git a/tests/MessagePack.Tests/CollectionTest.cs b/tests/MessagePack.Tests/CollectionTest.cs
index ade9561ad..eb590983a 100644
--- a/tests/MessagePack.Tests/CollectionTest.cs
+++ b/tests/MessagePack.Tests/CollectionTest.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Buffers;
+using System.Buffers.Binary;
 using System.Collections;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -10,6 +11,8 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using MessagePack.Formatters;
+using MessagePack.Resolvers;
 using Xunit;
 
 namespace MessagePack.Tests
@@ -313,6 +316,35 @@ public void CustomGenericReadOnlyDictionaryTest()
             d2["foo"].Is(10);
             d2["bar"].Is(20);
         }
+
+        [Fact]
+        public void ByteListFormatterSerializeTest()
+        {
+            // ByteListFormatter and ListFormatter<byte> result must be same
+            var option1 = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create([ByteListFormatter.Instance, ByteFormatter.Instance, new ListFormatter<byte>()]));
+            var option2 = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create([ByteFormatter.Instance, new ListFormatter<byte>()]));
+
+            var bin1 = MessagePackSerializer.Serialize(new List<byte> { 1, 2, 3 }, option1);
+            var bin2 = MessagePackSerializer.Serialize(new List<byte> { 1, 2, 3 }, option2);
+
+            bin1.ShouldBe(bin2);
+        }
+
+        [Fact]
+        public void ByteListFormatterDeserializeTest()
+        {
+            var binary = MessagePackSerializer.Serialize(new byte[] { 1, 2, 3 });
+            var array = MessagePackSerializer.Serialize(new List<byte> { 1, 2, 3 });
+
+            MessagePackPrimitives.TryReadBinHeader(binary, out _, out _).ShouldBe(MessagePackPrimitives.DecodeResult.Success);
+            MessagePackPrimitives.TryReadArrayHeader(array, out _, out _).ShouldBe(MessagePackPrimitives.DecodeResult.Success);
+
+            var v1 = MessagePackSerializer.Deserialize<List<byte>>(binary);
+            var v2 = MessagePackSerializer.Deserialize<List<byte>>(array);
+
+            v1.ShouldBe([1, 2, 3]);
+            v2.ShouldBe([1, 2, 3]);
+        }
     }
 
     public class ImplNonGenericList : IList

From bea23374453a88360b7a7eb01d1c213066a516a0 Mon Sep 17 00:00:00 2001
From: neuecc <ils@neue.cc>
Date: Fri, 24 Jan 2025 12:18:02 +0900
Subject: [PATCH 3/3] fix duplicate registration

---
 src/MessagePack/Resolvers/BuiltinResolver.cs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/MessagePack/Resolvers/BuiltinResolver.cs b/src/MessagePack/Resolvers/BuiltinResolver.cs
index d649292c1..5a3cfbfbe 100644
--- a/src/MessagePack/Resolvers/BuiltinResolver.cs
+++ b/src/MessagePack/Resolvers/BuiltinResolver.cs
@@ -143,7 +143,6 @@ internal static class BuiltinResolverGetFormatterHelper
             { typeof(List<Single>), new ListFormatter<Single>() },
             { typeof(List<Double>), new ListFormatter<Double>() },
             { typeof(List<Boolean>), new ListFormatter<Boolean>() },
-            { typeof(List<byte>), new ListFormatter<byte>() },
             { typeof(List<SByte>), new ListFormatter<SByte>() },
             { typeof(List<Char>), new ListFormatter<Char>() },
 #endif