From 6667669e71bf638b0dbda8ef1ffcc71befe060b0 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Fri, 24 Feb 2017 18:18:04 -0700
Subject: [PATCH 1/8] Add ICustomMarshaler StrMarshaler

Useful resources
https://msdn.microsoft.com/en-us/library/system.runtime.interopservices.icustommarshaler(v=vs.110).aspx
https://limbioliong.wordpress.com/2013/11/03/understanding-custom-marshaling-part-1/
https://github.com/mono/mono/blob/master/mcs/class/Mono.Posix/Mono.Unix/UnixMarshal.cs
http://stackoverflow.com/a/33514037/5208670
---
 src/runtime/CustomMarshaler.cs    | 80 +++++++++++++++++++++++++++++++
 src/runtime/Python.Runtime.csproj |  1 +
 src/runtime/debughelper.cs        | 22 +++++++++
 src/runtime/runtime.cs            | 52 ++------------------
 4 files changed, 108 insertions(+), 47 deletions(-)
 create mode 100644 src/runtime/CustomMarshaler.cs

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
new file mode 100644
index 000000000..bb3a4946b
--- /dev/null
+++ b/src/runtime/CustomMarshaler.cs
@@ -0,0 +1,80 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace Python.Runtime
+{
+    /// <summary>
+    /// Abstract class defining boiler plate methods that
+    /// Custom Marshalers will use.
+    /// </summary>
+    public abstract class MarshalerBase : ICustomMarshaler
+    {
+        public object MarshalNativeToManaged(IntPtr pNativeData)
+        {
+            throw new NotImplementedException();
+        }
+
+        public abstract IntPtr MarshalManagedToNative(object managedObj);
+
+        public void CleanUpNativeData(IntPtr pNativeData)
+        {
+            Marshal.FreeHGlobal(pNativeData);
+        }
+
+        public void CleanUpManagedData(object managedObj)
+        {
+            // Let GC deal with it
+        }
+
+        public int GetNativeDataSize()
+        {
+            return IntPtr.Size;
+        }
+    }
+
+
+    /// <summary>
+    /// Custom Marshaler to deal with Managed String to Native
+    /// conversion differences on UCS2/UCS4.
+    /// </summary>
+    public class StrMarshaler : MarshalerBase
+    {
+        private static readonly MarshalerBase Instance = new StrMarshaler();
+
+        public override IntPtr MarshalManagedToNative(object managedObj)
+        {
+            Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+            var s = managedObj as string;
+
+            if (s == null)
+            {
+                return IntPtr.Zero;
+            }
+
+            int minByteCount = encoding.GetMaxByteCount(1);
+            char[] cStr = s.ToCharArray(0, s.Length);
+            byte[] bStr = new byte[encoding.GetByteCount(cStr) + minByteCount];
+            encoding.GetBytes(cStr, 0, cStr.Length, bStr, 0);
+            DebugUtil.PrintHexBytes(bStr);
+
+            IntPtr mem = Marshal.AllocHGlobal(bStr.Length);
+            try
+            {
+                Marshal.Copy(bStr, 0, mem, bStr.Length);
+            }
+            catch (Exception)
+            {
+                Marshal.FreeHGlobal(mem);
+                throw;
+            }
+
+            return mem;
+        }
+
+        public static ICustomMarshaler GetInstance(string cookie)
+        {
+            return Instance;
+        }
+    }
+}
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
index 3dbad5f75..8580b7f61 100644
--- a/src/runtime/Python.Runtime.csproj
+++ b/src/runtime/Python.Runtime.csproj
@@ -89,6 +89,7 @@
     <Compile Include="constructorbinder.cs" />
     <Compile Include="constructorbinding.cs" />
     <Compile Include="converter.cs" />
+    <Compile Include="CustomMarshaler.cs" />
     <Compile Include="debughelper.cs" />
     <Compile Include="delegatemanager.cs" />
     <Compile Include="delegateobject.cs" />
diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs
index 777a61e35..2a91a74b4 100644
--- a/src/runtime/debughelper.cs
+++ b/src/runtime/debughelper.cs
@@ -115,5 +115,27 @@ internal static void debug(string msg)
             Console.WriteLine("thread {0} : {1}", tid, caller);
             Console.WriteLine("  {0}", msg);
         }
+
+        /// <summary>
+        /// Helper function to inspect/compare managed to native conversions.
+        /// Especially useful when debugging CustomMarshaler.
+        /// </summary>
+        /// <param name="bytes"></param>
+        [Conditional("DEBUG")]
+        public static void PrintHexBytes(byte[] bytes)
+        {
+            if ((bytes == null) || (bytes.Length == 0))
+            {
+                Console.WriteLine("<none>");
+            }
+            else
+            {
+                foreach (byte t in bytes)
+                {
+                    Console.Write("{0:X2} ", t);
+                }
+                Console.WriteLine();
+            }
+        }
     }
 }
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 6398d3345..50e09dd95 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -1688,34 +1688,10 @@ internal unsafe static extern IntPtr
         internal unsafe static extern IntPtr
             PyUnicode_FromKindAndString(
                 int kind,
-                IntPtr s,
+                [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrMarshaler))] string s,
                 int size
             );
 
-        internal static unsafe IntPtr PyUnicode_FromKindAndString(
-            int kind,
-            string s,
-            int size)
-        {
-            var bufLength = Math.Max(s.Length, size) * 4;
-
-            IntPtr mem = Marshal.AllocHGlobal(bufLength);
-            try
-            {
-                fixed (char* ps = s)
-                {
-                    Encoding.UTF32.GetBytes(ps, s.Length, (byte*)mem, bufLength);
-                }
-
-                var result = PyUnicode_FromKindAndString(kind, mem, size);
-                return result;
-            }
-            finally
-            {
-                Marshal.FreeHGlobal(mem);
-            }
-        }
-
         internal static IntPtr PyUnicode_FromUnicode(string s, int size)
         {
             return PyUnicode_FromKindAndString(4, s, size);
@@ -1758,28 +1734,10 @@ internal unsafe static extern IntPtr
             EntryPoint = "PyUnicodeUCS4_FromUnicode",
             ExactSpelling = true)]
         internal unsafe static extern IntPtr
-            PyUnicode_FromUnicode(IntPtr s, int size);
-
-        internal static unsafe IntPtr PyUnicode_FromUnicode(string s, int size)
-        {
-            var bufLength = Math.Max(s.Length, size) * 4;
-
-            IntPtr mem = Marshal.AllocHGlobal(bufLength);
-            try
-            {
-                fixed (char* ps = s)
-                {
-                    Encoding.UTF32.GetBytes(ps, s.Length, (byte*)mem, bufLength);
-                }
-
-                var result = PyUnicode_FromUnicode(mem, size);
-                return result;
-            }
-            finally
-            {
-                Marshal.FreeHGlobal(mem);
-            }
-        }
+            PyUnicode_FromUnicode(
+                [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrMarshaler))] string s,
+                int size
+            );
 
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             EntryPoint = "PyUnicodeUCS4_GetSize",

From d01f25fffd94458aed7e1f6ef3a2d79dde12525d Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 25 Feb 2017 02:15:13 -0700
Subject: [PATCH 2/8] Add ICustomMarshaler StrArrayMarshaler

---
 src/runtime/CustomMarshaler.cs | 54 ++++++++++++++++++++++++++
 src/runtime/runtime.cs         | 69 +---------------------------------
 2 files changed, 56 insertions(+), 67 deletions(-)

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
index bb3a4946b..f776cb1e3 100644
--- a/src/runtime/CustomMarshaler.cs
+++ b/src/runtime/CustomMarshaler.cs
@@ -77,4 +77,58 @@ public static ICustomMarshaler GetInstance(string cookie)
             return Instance;
         }
     }
+
+
+    /// <summary>
+    /// Custom Marshaler to deal with Managed String Arrays to Native
+    /// conversion differences on UCS2/UCS4.
+    /// </summary>
+    public class StrArrayMarshaler : MarshalerBase
+    {
+        private static readonly MarshalerBase Instance = new StrArrayMarshaler();
+
+        public override IntPtr MarshalManagedToNative(object managedObj)
+        {
+            var argv = managedObj as string[];
+
+            if (argv == null)
+            {
+                return IntPtr.Zero;
+            }
+
+            var totalStrLength = 0;
+            foreach (string arg in argv)
+            {
+                totalStrLength += arg.Length + 1;
+            }
+            int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime.UCS;
+
+            IntPtr mem = Marshal.AllocHGlobal(memSize);
+            try
+            {
+                // Preparing array of pointers to strings
+                IntPtr curStrPtr = mem + argv.Length * IntPtr.Size;
+                for (var i = 0; i < argv.Length; i++)
+                {
+                    Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+                    byte[] bStr = encoding.GetBytes(argv[i] + "\0");
+                    Marshal.Copy(bStr, 0, curStrPtr, bStr.Length);
+                    Marshal.WriteIntPtr(mem + i * IntPtr.Size, curStrPtr);
+                    curStrPtr += bStr.Length;
+                }
+            }
+            catch (Exception)
+            {
+                Marshal.FreeHGlobal(mem);
+                throw;
+            }
+
+            return mem;
+        }
+
+        public static ICustomMarshaler GetInstance(string cookie)
+        {
+            return Instance;
+        }
+    }
 }
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 50e09dd95..3d5eda0fe 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -667,41 +667,8 @@ internal unsafe static extern IntPtr
         public unsafe static extern int
             Py_Main(
                 int argc,
-                IntPtr lplpargv
+                [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv
             );
-
-        public static int Py_Main(int argc, string[] argv)
-        {
-            // Totally ignoring argc.
-            argc = argv.Length;
-
-            var allStringsLength = 0;
-            foreach (string x in argv)
-            {
-                allStringsLength += x.Length + 1;
-            }
-            int requiredSize = IntPtr.Size * argc + allStringsLength * UCS;
-            IntPtr mem = Marshal.AllocHGlobal(requiredSize);
-            try
-            {
-                // Preparing array of pointers to UTF32 strings.
-                IntPtr curStrPtr = mem + argc * IntPtr.Size;
-                for (var i = 0; i < argv.Length; i++)
-                {
-                    // Unicode or UTF8 work
-                    Encoding enc = UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
-                    byte[] zstr = enc.GetBytes(argv[i] + "\0");
-                    Marshal.Copy(zstr, 0, curStrPtr, zstr.Length);
-                    Marshal.WriteIntPtr(mem + IntPtr.Size * i, curStrPtr);
-                    curStrPtr += zstr.Length;
-                }
-                return Py_Main(argc, mem);
-            }
-            finally
-            {
-                Marshal.FreeHGlobal(mem);
-            }
-        }
 #elif PYTHON2
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Ansi)]
@@ -2078,41 +2045,9 @@ internal unsafe static extern IntPtr
         internal unsafe static extern void
             PySys_SetArgvEx(
                 int argc,
-                IntPtr lplpargv,
+                [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv,
                 int updatepath
             );
-
-        internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath)
-        {
-            // Totally ignoring argc.
-            argc = argv.Length;
-
-            var allStringsLength = 0;
-            foreach (string x in argv)
-            {
-                allStringsLength += x.Length + 1;
-            }
-            int requiredSize = IntPtr.Size * argc + allStringsLength * UCS;
-            IntPtr mem = Marshal.AllocHGlobal(requiredSize);
-            try
-            {
-                // Preparing array of pointers to UTF32 strings.
-                IntPtr curStrPtr = mem + argc * IntPtr.Size;
-                for (var i = 0; i < argv.Length; i++)
-                {
-                    Encoding enc = UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
-                    byte[] zstr = enc.GetBytes(argv[i] + "\0");
-                    Marshal.Copy(zstr, 0, curStrPtr, zstr.Length);
-                    Marshal.WriteIntPtr(mem + IntPtr.Size * i, curStrPtr);
-                    curStrPtr += zstr.Length;
-                }
-                PySys_SetArgvEx(argc, mem, updatepath);
-            }
-            finally
-            {
-                Marshal.FreeHGlobal(mem);
-            }
-        }
 #elif PYTHON2
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Ansi)]

From 74440a6b90beaaa32a2df8bc6b3660ce5edafbcb Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 25 Feb 2017 10:05:16 -0700
Subject: [PATCH 3/8] Refactor Marshals

---
 src/runtime/CustomMarshaler.cs | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
index f776cb1e3..1004122cd 100644
--- a/src/runtime/CustomMarshaler.cs
+++ b/src/runtime/CustomMarshaler.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
 
@@ -41,10 +42,10 @@ public int GetNativeDataSize()
     public class StrMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrMarshaler();
+        private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
-            Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
             var s = managedObj as string;
 
             if (s == null)
@@ -52,12 +53,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
                 return IntPtr.Zero;
             }
 
-            int minByteCount = encoding.GetMaxByteCount(1);
-            char[] cStr = s.ToCharArray(0, s.Length);
-            byte[] bStr = new byte[encoding.GetByteCount(cStr) + minByteCount];
-            encoding.GetBytes(cStr, 0, cStr.Length, bStr, 0);
-            DebugUtil.PrintHexBytes(bStr);
-
+            byte[] bStr = PyEncoding.GetBytes(s + "\0");
             IntPtr mem = Marshal.AllocHGlobal(bStr.Length);
             try
             {
@@ -86,6 +82,7 @@ public static ICustomMarshaler GetInstance(string cookie)
     public class StrArrayMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrArrayMarshaler();
+        private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -96,11 +93,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
                 return IntPtr.Zero;
             }
 
-            var totalStrLength = 0;
-            foreach (string arg in argv)
-            {
-                totalStrLength += arg.Length + 1;
-            }
+            int totalStrLength = argv.Sum(arg => arg.Length + 1);
             int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime.UCS;
 
             IntPtr mem = Marshal.AllocHGlobal(memSize);
@@ -110,8 +103,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
                 IntPtr curStrPtr = mem + argv.Length * IntPtr.Size;
                 for (var i = 0; i < argv.Length; i++)
                 {
-                    Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
-                    byte[] bStr = encoding.GetBytes(argv[i] + "\0");
+                    byte[] bStr = PyEncoding.GetBytes(argv[i] + "\0");
                     Marshal.Copy(bStr, 0, curStrPtr, bStr.Length);
                     Marshal.WriteIntPtr(mem + i * IntPtr.Size, curStrPtr);
                     curStrPtr += bStr.Length;

From e487076d90ac86f15150c1b41113565364fb7850 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 25 Feb 2017 12:58:27 -0700
Subject: [PATCH 4/8] Add ICustomMarshaler Utf8Marshaler

Refactor PyString_FromStringAndSize
Link explains why `MarshalAs(UnmanagedType.LPWStr)` or `CharSet.Unicode` don't work

http://stackoverflow.com/a/25128147/5208670
---
 src/runtime/CustomMarshaler.cs | 45 ++++++++++++++++++++++++++++++++++
 src/runtime/runtime.cs         | 23 +++++------------
 2 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
index 1004122cd..a4947a909 100644
--- a/src/runtime/CustomMarshaler.cs
+++ b/src/runtime/CustomMarshaler.cs
@@ -123,4 +123,49 @@ public static ICustomMarshaler GetInstance(string cookie)
             return Instance;
         }
     }
+
+
+    /// <summary>
+    /// Custom Marshaler to deal with Managed String to Native
+    /// conversion on UTF-8. Use on functions that expect UTF-8 encoded
+    /// strings like `PyUnicode_FromStringAndSize`
+    /// </summary>
+    /// <remarks>
+    /// If instead we used `MarshalAs(UnmanagedType.LPWStr)` the output to
+    /// `foo` would be `f\x00o\x00o\x00`.
+    /// </remarks>
+    public class Utf8Marshaler : MarshalerBase
+    {
+        private static readonly MarshalerBase Instance = new Utf8Marshaler();
+        private static readonly Encoding PyEncoding = Encoding.UTF8;
+
+        public override IntPtr MarshalManagedToNative(object managedObj)
+        {
+            var s = managedObj as string;
+
+            if (s == null)
+            {
+                return IntPtr.Zero;
+            }
+
+            byte[] bStr = PyEncoding.GetBytes(s + "\0");
+            IntPtr mem = Marshal.AllocHGlobal(bStr.Length);
+            try
+            {
+                Marshal.Copy(bStr, 0, mem, bStr.Length);
+            }
+            catch (Exception)
+            {
+                Marshal.FreeHGlobal(mem);
+                throw;
+            }
+
+            return mem;
+        }
+
+        public static ICustomMarshaler GetInstance(string cookie)
+        {
+            return Instance;
+        }
+    }
 }
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 3d5eda0fe..50c31ddb4 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -1508,23 +1508,12 @@ internal static IntPtr PyBytes_AS_STRING(IntPtr ob)
             return ob + BytesOffset.ob_sval;
         }
 
-        internal static IntPtr PyString_FromStringAndSize(string value, int length)
-        {
-            // copy the string into an unmanaged UTF-8 buffer
-            int len = Encoding.UTF8.GetByteCount(value);
-            byte[] buffer = new byte[len + 1];
-            Encoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0);
-            IntPtr nativeUtf8 = Marshal.AllocHGlobal(buffer.Length);
-            try
-            {
-                Marshal.Copy(buffer, 0, nativeUtf8, buffer.Length);
-                return PyUnicode_FromStringAndSize(nativeUtf8, length);
-            }
-            finally
-            {
-                Marshal.FreeHGlobal(nativeUtf8);
-            }
-        }
+        [DllImport(Runtime.dll, EntryPoint = "PyUnicode_FromStringAndSize")]
+        internal static extern IntPtr
+            PyString_FromStringAndSize(
+                [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value,
+                int size
+            );
 
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Unicode)]

From ccd4521298e87c53d1217670acbc3963371020a3 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 25 Feb 2017 19:57:52 -0700
Subject: [PATCH 5/8] Match PyUnicode_AsUnicode signature in UCS2/UCS4

---
 src/runtime/runtime.cs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 50c31ddb4..13549b581 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -1572,7 +1572,7 @@ internal unsafe static extern int
 
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Unicode)]
-        internal unsafe static extern char*
+        internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
@@ -1613,7 +1613,7 @@ internal unsafe static extern int
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             EntryPoint = "PyUnicodeUCS2_AsUnicode",
             ExactSpelling = true)]
-        internal unsafe static extern char*
+        internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
@@ -1749,9 +1749,9 @@ internal unsafe static string GetManagedString(IntPtr op)
                 Marshal.Copy(p, buffer, 0, size);
                 return Encoding.UTF32.GetString(buffer, 0, size);
 #elif UCS2
-                char* p = Runtime.PyUnicode_AsUnicode(op);
-                int size = Runtime.PyUnicode_GetSize(op);
-                return new String(p, 0, size);
+                IntPtr p = Runtime.PyUnicode_AsUnicode(op);
+                int length = Runtime.PyUnicode_GetSize(op);
+                return Marshal.PtrToStringUni(p, length);
 #endif
             }
 

From 07f87de2ae0456d7e5c09333a09ab6bbbc11cf1f Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 25 Feb 2017 23:09:48 -0700
Subject: [PATCH 6/8] Refactor GetManagedString

---
 src/runtime/runtime.cs | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 13549b581..a9dcdd81e 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -1725,7 +1725,20 @@ internal static IntPtr PyUnicode_FromString(string s)
             return PyUnicode_FromUnicode(s, (s.Length));
         }
 
-        internal unsafe static string GetManagedString(IntPtr op)
+        /// <summary>
+        /// Function to access the internal PyUnicode/PyString object and
+        /// convert it to a managed string with the correct encoding.
+        /// </summary>
+        /// <remarks>
+        /// We can't easily do this through through the CustomMarshaler's on
+        /// the returns because will have access to the IntPtr but not size.
+        /// <para />
+        /// For PyUnicodeType, we can't convert with Marshal.PtrToStringUni
+        /// since it only works for UCS2.
+        /// </remarks>
+        /// <param name="op">PyStringType or PyUnicodeType object to convert</param>
+        /// <returns>Managed String</returns>
+        internal static string GetManagedString(IntPtr op)
         {
             IntPtr type = PyObject_TYPE(op);
 
@@ -1741,18 +1754,15 @@ internal unsafe static string GetManagedString(IntPtr op)
 
             if (type == Runtime.PyUnicodeType)
             {
-#if UCS4
-                IntPtr p = Runtime.PyUnicode_AsUnicode(op);
-                int length = Runtime.PyUnicode_GetSize(op);
-                int size = length * 4;
-                byte[] buffer = new byte[size];
+                Encoding encoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+
+                IntPtr p = PyUnicode_AsUnicode(op);
+                int length = PyUnicode_GetSize(op);
+
+                int size = length * UCS;
+                var buffer = new byte[size];
                 Marshal.Copy(p, buffer, 0, size);
-                return Encoding.UTF32.GetString(buffer, 0, size);
-#elif UCS2
-                IntPtr p = Runtime.PyUnicode_AsUnicode(op);
-                int length = Runtime.PyUnicode_GetSize(op);
-                return Marshal.PtrToStringUni(p, length);
-#endif
+                return encoding.GetString(buffer, 0, size);
             }
 
             return null;

From edafdf2db546908e4cc60f55656a0125e25bac4f Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sun, 26 Feb 2017 00:18:37 -0700
Subject: [PATCH 7/8] Remove internal PyUnicode_AS_UNICODE

Its redundant with PyUnicode_AsUnicode now that the signature
is fixed between UCS2/UCS4.

Apply char conversion that work on both UCS2/UCS4
---
 src/runtime/converter.cs | 19 ++++---------------
 src/runtime/runtime.cs   | 24 ------------------------
 2 files changed, 4 insertions(+), 39 deletions(-)

diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs
index aeaf2d871..8b58e5e9a 100644
--- a/src/runtime/converter.cs
+++ b/src/runtime/converter.cs
@@ -599,21 +599,10 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo
                     {
                         if (Runtime.PyUnicode_GetSize(value) == 1)
                         {
-                            op = Runtime.PyUnicode_AS_UNICODE(value);
-                            if (Runtime.UCS == 2) // Don't trust linter, statement not always true.
-                            {
-                                // 2011-01-02: Marshal as character array because the cast
-                                // result = (char)Marshal.ReadInt16(op); throws an OverflowException
-                                // on negative numbers with Check Overflow option set on the project
-                                Char[] buff = new Char[1];
-                                Marshal.Copy(op, buff, 0, 1);
-                                result = buff[0];
-                            }
-                            else // UCS4
-                            {
-                                // XXX this is probably NOT correct?
-                                result = (char)Marshal.ReadInt32(op);
-                            }
+                            op = Runtime.PyUnicode_AsUnicode(value);
+                            Char[] buff = new Char[1];
+                            Marshal.Copy(op, buff, 0, 1);
+                            result = buff[0];
                             return true;
                         }
                         goto type_error;
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index a9dcdd81e..9558a5ac3 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -1575,12 +1575,6 @@ internal unsafe static extern int
         internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
-        [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
-            EntryPoint = "PyUnicode_AsUnicode",
-            ExactSpelling = true, CharSet = CharSet.Unicode)]
-        internal unsafe static extern IntPtr
-            PyUnicode_AS_UNICODE(IntPtr op);
-
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Unicode)]
         internal unsafe static extern IntPtr
@@ -1616,12 +1610,6 @@ internal unsafe static extern int
         internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
-        [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
-            EntryPoint = "PyUnicodeUCS2_AsUnicode",
-            ExactSpelling = true, CharSet = CharSet.Unicode)]
-        internal unsafe static extern IntPtr
-            PyUnicode_AS_UNICODE(IntPtr op);
-
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             EntryPoint = "PyUnicodeUCS2_FromOrdinal",
             ExactSpelling = true, CharSet = CharSet.Unicode)]
@@ -1663,12 +1651,6 @@ internal unsafe static extern int
         internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
-        [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
-            EntryPoint = "PyUnicode_AsUnicode",
-            ExactSpelling = true, CharSet = CharSet.Unicode)]
-        internal unsafe static extern IntPtr
-            PyUnicode_AS_UNICODE(IntPtr op);
-
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             ExactSpelling = true, CharSet = CharSet.Unicode)]
         internal unsafe static extern IntPtr
@@ -1707,12 +1689,6 @@ internal unsafe static extern int
         internal unsafe static extern IntPtr
             PyUnicode_AsUnicode(IntPtr ob);
 
-        [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
-            EntryPoint = "PyUnicodeUCS4_AsUnicode",
-            ExactSpelling = true, CharSet = CharSet.Unicode)]
-        internal unsafe static extern IntPtr
-            PyUnicode_AS_UNICODE(IntPtr op);
-
         [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl,
             EntryPoint = "PyUnicodeUCS4_FromOrdinal",
             ExactSpelling = true, CharSet = CharSet.Unicode)]

From 78939c599eecdeccad0384feb4f584fd36171024 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sun, 26 Feb 2017 10:30:06 -0700
Subject: [PATCH 8/8] Refactor Encoding check

This won't change during runtime.
---
 src/runtime/CustomMarshaler.cs | 4 ++--
 src/runtime/runtime.cs         | 9 ++++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
index a4947a909..475fa05eb 100644
--- a/src/runtime/CustomMarshaler.cs
+++ b/src/runtime/CustomMarshaler.cs
@@ -42,7 +42,7 @@ public int GetNativeDataSize()
     public class StrMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -82,7 +82,7 @@ public static ICustomMarshaler GetInstance(string cookie)
     public class StrArrayMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrArrayMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 9558a5ac3..337b82fbf 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -170,6 +170,11 @@ public class Runtime
         internal static bool IsPython2;
         internal static bool IsPython3;
 
+        /// <summary>
+        /// Encoding to use to convert Unicode to/from Managed to Native
+        /// </summary>
+        internal static readonly Encoding PyEncoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
+
         /// <summary>
         /// Initialize the runtime...
         /// </summary>
@@ -1730,15 +1735,13 @@ internal static string GetManagedString(IntPtr op)
 
             if (type == Runtime.PyUnicodeType)
             {
-                Encoding encoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
-
                 IntPtr p = PyUnicode_AsUnicode(op);
                 int length = PyUnicode_GetSize(op);
 
                 int size = length * UCS;
                 var buffer = new byte[size];
                 Marshal.Copy(p, buffer, 0, size);
-                return encoding.GetString(buffer, 0, size);
+                return PyEncoding.GetString(buffer, 0, size);
             }
 
             return null;