Add StrictUTF8 (neo-project#1710)

ShawnYun · Jun 22, 2020 · 236a46c · 236a46c
1 parent 22d8e1e
commit 236a46c
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 21 deletions.
diff --git a/src/neo/IO/Helper.cs b/src/neo/IO/Helper.cs
@@ -6,7 +6,6 @@
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
-using System.Text;
 
 namespace Neo.IO
 {
@@ -15,7 +14,7 @@ public static class Helper
         public static T AsSerializable<T>(this byte[] value, int start = 0) where T : ISerializable, new()
         {
             using (MemoryStream ms = new MemoryStream(value, start, value.Length - start, false))
-            using (BinaryReader reader = new BinaryReader(ms, Encoding.UTF8))
+            using (BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8))
             {
                 return reader.ReadSerializable<T>();
             }
@@ -27,7 +26,7 @@ public static class Helper
             fixed (byte* pointer = value)
             {
                 using UnmanagedMemoryStream ms = new UnmanagedMemoryStream(pointer, value.Length);
-                using BinaryReader reader = new BinaryReader(ms, Encoding.UTF8);
+                using BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8);
                 return reader.ReadSerializable<T>();
             }
         }
@@ -38,7 +37,7 @@ public static ISerializable AsSerializable(this byte[] value, Type type)
                 throw new InvalidCastException();
             ISerializable serializable = (ISerializable)Activator.CreateInstance(type);
             using (MemoryStream ms = new MemoryStream(value, false))
-            using (BinaryReader reader = new BinaryReader(ms, Encoding.UTF8))
+            using (BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8))
             {
                 serializable.Deserialize(reader);
             }
@@ -48,7 +47,7 @@ public static ISerializable AsSerializable(this byte[] value, Type type)
         public static T[] AsSerializableArray<T>(this byte[] value, int max = 0x1000000) where T : ISerializable, new()
         {
             using (MemoryStream ms = new MemoryStream(value, false))
-            using (BinaryReader reader = new BinaryReader(ms, Encoding.UTF8))
+            using (BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8))
             {
                 return reader.ReadSerializableArray<T>(max);
             }
@@ -60,7 +59,7 @@ public static ISerializable AsSerializable(this byte[] value, Type type)
             fixed (byte* pointer = value)
             {
                 using UnmanagedMemoryStream ms = new UnmanagedMemoryStream(pointer, value.Length);
-                using BinaryReader reader = new BinaryReader(ms, Encoding.UTF8);
+                using BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8);
                 return reader.ReadSerializableArray<T>(max);
             }
         }
@@ -138,7 +137,7 @@ public static int GetVarSize<T>(this IReadOnlyCollection<T> value)
 
         public static int GetVarSize(this string value)
         {
-            int size = Encoding.UTF8.GetByteCount(value);
+            int size = Utility.StrictUTF8.GetByteCount(value);
             return GetVarSize(size) + size;
         }
 
@@ -166,7 +165,7 @@ public static byte[] ReadFixedBytes(this BinaryReader reader, int size)
         public static string ReadFixedString(this BinaryReader reader, int length)
         {
             byte[] data = reader.ReadFixedBytes(length);
-            return Encoding.UTF8.GetString(data.TakeWhile(p => p != 0).ToArray());
+            return Utility.StrictUTF8.GetString(data.TakeWhile(p => p != 0).ToArray());
         }
 
         public static T[] ReadNullableArray<T>(this BinaryReader reader, int max = 0x1000000) where T : class, ISerializable, new()
@@ -218,13 +217,13 @@ public static ulong ReadVarInt(this BinaryReader reader, ulong max = ulong.MaxVa
 
         public static string ReadVarString(this BinaryReader reader, int max = 0x1000000)
         {
-            return Encoding.UTF8.GetString(reader.ReadVarBytes(max));
+            return Utility.StrictUTF8.GetString(reader.ReadVarBytes(max));
         }
 
         public static byte[] ToArray(this ISerializable value)
         {
             using (MemoryStream ms = new MemoryStream())
-            using (BinaryWriter writer = new BinaryWriter(ms, Encoding.UTF8))
+            using (BinaryWriter writer = new BinaryWriter(ms, Utility.StrictUTF8))
             {
                 value.Serialize(writer);
                 writer.Flush();
@@ -235,7 +234,7 @@ public static byte[] ToArray(this ISerializable value)
         public static byte[] ToByteArray<T>(this IReadOnlyCollection<T> value) where T : ISerializable
         {
             using (MemoryStream ms = new MemoryStream())
-            using (BinaryWriter writer = new BinaryWriter(ms, Encoding.UTF8))
+            using (BinaryWriter writer = new BinaryWriter(ms, Utility.StrictUTF8))
             {
                 writer.Write(value);
                 writer.Flush();
@@ -263,7 +262,7 @@ public static void WriteFixedString(this BinaryWriter writer, string value, int
                 throw new ArgumentNullException(nameof(value));
             if (value.Length > length)
                 throw new ArgumentException();
-            byte[] bytes = Encoding.UTF8.GetBytes(value);
+            byte[] bytes = Utility.StrictUTF8.GetBytes(value);
             if (bytes.Length > length)
                 throw new ArgumentException();
             writer.Write(bytes);
@@ -316,7 +315,7 @@ public static void WriteVarInt(this BinaryWriter writer, long value)
 
         public static void WriteVarString(this BinaryWriter writer, string value)
         {
-            writer.WriteVarBytes(Encoding.UTF8.GetBytes(value));
+            writer.WriteVarBytes(Utility.StrictUTF8.GetBytes(value));
         }
     }
 }
diff --git a/src/neo/IO/Json/JObject.cs b/src/neo/IO/Json/JObject.cs
@@ -80,7 +80,7 @@ public static JObject Parse(ReadOnlySpan<byte> value, int max_nest = 100)
 
         public static JObject Parse(string value, int max_nest = 100)
         {
-            return Parse(Encoding.UTF8.GetBytes(value), max_nest);
+            return Parse(Utility.StrictUTF8.GetBytes(value), max_nest);
         }
 
         private static JObject Read(ref Utf8JsonReader reader, bool skipReading = false)
@@ -158,7 +158,7 @@ public override string ToString()
 
         public string ToString(bool indented)
         {
-            return Encoding.UTF8.GetString(ToByteArray(indented));
+            return Utility.StrictUTF8.GetString(ToByteArray(indented));
         }
 
         public virtual T TryGetEnum<T>(T defaultValue = default, bool ignoreCase = false) where T : Enum

diff --git a/src/neo/SmartContract/ApplicationEngine.Runtime.cs b/src/neo/SmartContract/ApplicationEngine.Runtime.cs
@@ -143,15 +143,15 @@ internal int GetInvocationCounter()
         internal void RuntimeLog(byte[] state)
         {
             if (state.Length > MaxNotificationSize) throw new ArgumentException();
-            string message = Encoding.UTF8.GetString(state);
+            string message = Utility.StrictUTF8.GetString(state);
             Log?.Invoke(this, new LogEventArgs(ScriptContainer, CurrentScriptHash, message));
         }
 
         internal void RuntimeNotify(byte[] eventName, Array state)
         {
             if (eventName.Length > MaxEventName) throw new ArgumentException();
             if (!CheckItemForNotification(state)) throw new ArgumentException();
-            SendNotification(CurrentScriptHash, Encoding.UTF8.GetString(eventName), state);
+            SendNotification(CurrentScriptHash, Utility.StrictUTF8.GetString(eventName), state);
         }
 
         internal void SendNotification(UInt160 hash, string eventName, Array state)

diff --git a/src/neo/SmartContract/ContractParametersContext.cs b/src/neo/SmartContract/ContractParametersContext.cs
@@ -198,7 +198,7 @@ public static ContractParametersContext FromJson(JObject json)
 
             var verifiable = (IVerifiable)Activator.CreateInstance(type);
             using (MemoryStream ms = new MemoryStream(Convert.FromBase64String(json["hex"].AsString()), false))
-            using (BinaryReader reader = new BinaryReader(ms, Encoding.UTF8))
+            using (BinaryReader reader = new BinaryReader(ms, Utility.StrictUTF8))
             {
                 verifiable.DeserializeUnsigned(reader);
             }
@@ -262,7 +262,7 @@ public JObject ToJson()
             JObject json = new JObject();
             json["type"] = Verifiable.GetType().FullName;
             using (MemoryStream ms = new MemoryStream())
-            using (BinaryWriter writer = new BinaryWriter(ms, Encoding.UTF8))
+            using (BinaryWriter writer = new BinaryWriter(ms, Utility.StrictUTF8))
             {
                 Verifiable.SerializeUnsigned(writer);
                 writer.Flush();

diff --git a/src/neo/SmartContract/JsonSerializer.cs b/src/neo/SmartContract/JsonSerializer.cs
@@ -118,7 +118,7 @@ public static byte[] SerializeToByteArray(StackItem item, uint maxSize)
                         writer.WriteEndObject();
                         break;
                     case JsonTokenType.PropertyName:
-                        writer.WritePropertyName(((ByteString)stack.Pop()).GetSpan());
+                        writer.WritePropertyName(((ByteString)stack.Pop()).GetString());
                         break;
                     case Null _:
                         writer.WriteNullValue();

diff --git a/src/neo/Utility.cs b/src/neo/Utility.cs
@@ -5,6 +5,7 @@
 using System;
 using System.IO;
 using System.Reflection;
+using System.Text;
 
 namespace Neo
 {
@@ -19,6 +20,15 @@ public Logger()
             }
         }
 
+        public static Encoding StrictUTF8 { get; }
+
+        static Utility()
+        {
+            StrictUTF8 = (Encoding)Encoding.UTF8.Clone();
+            StrictUTF8.DecoderFallback = DecoderFallback.ExceptionFallback;
+            StrictUTF8.EncoderFallback = EncoderFallback.ExceptionFallback;
+        }
+
         /// <summary>
         /// Load configuration with different Environment Variable
         /// </summary>

diff --git a/src/neo/VM/Helper.cs b/src/neo/VM/Helper.cs
@@ -198,7 +198,7 @@ public static ReadOnlySpan<byte> GetSpan(this StackItem item)
 
         public static string GetString(this StackItem item)
         {
-            return Encoding.UTF8.GetString(item.GetSpan());
+            return Utility.StrictUTF8.GetString(item.GetSpan());
         }
 
         /// <summary>

diff --git a/tests/neo.UnitTests/SmartContract/UT_JsonSerializer.cs b/tests/neo.UnitTests/SmartContract/UT_JsonSerializer.cs
@@ -6,6 +6,7 @@
 using System;
 using System.Linq;
 using System.Numerics;
+using System.Text;
 
 namespace Neo.UnitTests.SmartContract
 {
@@ -48,6 +49,17 @@ public void JsonTest_Array()
             Assert.AreEqual("[1,\"a==\",-1.3,null]", parsed.ToString());
         }
 
+        [TestMethod]
+        public void JsonTest_Serialize_Map_Test()
+        {
+            var entry = new Map
+            {
+                [new byte[] { 0xC1 }] = 1,
+                [new byte[] { 0xC2 }] = 2,
+            };
+            Assert.ThrowsException<DecoderFallbackException>(() => JsonSerializer.Serialize(entry));
+        }
+
         [TestMethod]
         public void JsonTest_Bool()
         {