dotnet · stephentoub · Apr 22, 2023 · Mar 12, 2023 · Mar 13, 2023 · Mar 13, 2023
diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT
@@ -1196,3 +1196,46 @@ https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
 
 MurmurHash3 was written by Austin Appleby, and is placed in the public
 domain. The author hereby disclaims copyright to this source
+
+License for Fast CRC Computation
+--------------------------------------
+
+https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc32_ieee_by4.asm
+
+Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+  * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License for C# Implementation of Fast CRC Computation
+-----------------------------------------------------
+
+https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs
+
+Copyright (c) Six Labors.
+Licensed under the Apache License, Version 2.0.
+
+Available at
+https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/LICENSE
diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj
@@ -11,8 +11,14 @@ System.IO.Hashing.XxHash32</PackageDescription>
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="System\ThrowHelper.cs"
+             Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net7.0'))" />
     <Compile Include="System\IO\Hashing\Crc32.cs" />
+    <Compile Include="System\IO\Hashing\Crc32.Arm.cs"
+             Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net6.0'))" />
     <Compile Include="System\IO\Hashing\Crc32.Table.cs" />
+    <Compile Include="System\IO\Hashing\Crc32.Vectorized.cs"
+             Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net7.0'))" />
     <Compile Include="System\IO\Hashing\Crc64.cs" />
     <Compile Include="System\IO\Hashing\Crc64.Table.cs" />
     <Compile Include="System\IO\Hashing\XxHash128.cs" />

diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Arm.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Arm.cs
@@ -0,0 +1,69 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using ArmCrc = System.Runtime.Intrinsics.Arm.Crc32;
+
+namespace System.IO.Hashing
+{
+    public partial class Crc32
+    {
+        private static uint UpdateScalarArm64(uint crc, ReadOnlySpan<byte> source)
+        {
+            Debug.Assert(ArmCrc.Arm64.IsSupported, "ARM CRC support is required.");
+
+            // Compute in 8 byte chunks
+            if (source.Length >= sizeof(ulong))
+            {
+                ref byte ptr = ref MemoryMarshal.GetReference(source);
+                int longLength = source.Length & ~0x7; // Exclude trailing bytes not a multiple of 8
+
+                for (int i = 0; i < longLength; i += sizeof(ulong))
+                {
+                    crc = ArmCrc.Arm64.ComputeCrc32(crc,
+                        Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref ptr, i)));
+                }
+
+                source = source.Slice(longLength);
+            }
+
+            // Compute remaining bytes
+            for (int i = 0; i < source.Length; i++)
+            {
+                crc = ArmCrc.ComputeCrc32(crc, source[i]);
+            }
+
+            return crc;
+        }
+
+        private static uint UpdateScalarArm32(uint crc, ReadOnlySpan<byte> source)
+        {
+            Debug.Assert(ArmCrc.IsSupported, "ARM CRC support is required.");
+
+            // Compute in 4 byte chunks
+            if (source.Length >= sizeof(uint))
+            {
+                ref byte ptr = ref MemoryMarshal.GetReference(source);
+                int intLength = source.Length & ~0x3; // Exclude trailing bytes not a multiple of 4
+
+                for (int i = 0; i < intLength; i += sizeof(uint))
+                {
+                    crc = ArmCrc.ComputeCrc32(crc,
+                        Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref ptr, i)));
+                }
+
+                source = source.Slice(intLength);
+            }
+
+            // Compute remaining bytes
+            for (int i = 0; i < source.Length; i++)
+            {
+                crc = ArmCrc.ComputeCrc32(crc, source[i]);
+            }
+
+            return crc;
+        }
+    }
+}
diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Vectorized.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Vectorized.cs
@@ -0,0 +1,216 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.Arm;
+using Aes = System.Runtime.Intrinsics.Arm.Aes;
+
+namespace System.IO.Hashing
+{
+    public partial class Crc32
+    {
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector128<ulong> CarrylessMultiplyLower(Vector128<ulong> left, Vector128<ulong> right)
+        {
+            if (Pclmulqdq.IsSupported)
+            {
+                return Pclmulqdq.CarrylessMultiply(left, right, 0x00);
+            }
+
+            if (Aes.IsSupported)
+            {
+                return Aes.PolynomialMultiplyWideningLower(left.GetLower(), right.GetLower());
+            }
+
+            ThrowHelper.ThrowUnreachableException();
+            return default;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector128<ulong> CarrylessMultiplyUpper(Vector128<ulong> left, Vector128<ulong> right)
+        {
+            if (Pclmulqdq.IsSupported)
+            {
+                return Pclmulqdq.CarrylessMultiply(left, right, 0x11);
+            }
+
+            if (Aes.IsSupported)
+            {
+                return Aes.PolynomialMultiplyWideningUpper(left, right);
+            }
+
+            ThrowHelper.ThrowUnreachableException();
+            return default;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector128<ulong> CarrylessMultiplyLeftLowerRightUpper(Vector128<ulong> left, Vector128<ulong> right)
+        {
+            if (Pclmulqdq.IsSupported)
+            {
+                return Pclmulqdq.CarrylessMultiply(left, right, 0x10);
+            }
+
+            if (Aes.IsSupported)
+            {
+                return Aes.PolynomialMultiplyWideningLower(left.GetLower(), right.GetUpper());
+            }
+
+            ThrowHelper.ThrowUnreachableException();
+            return default;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector128<ulong> ShiftRightBytesInVector(Vector128<ulong> operand,
+            [ConstantExpected(Max = (byte)15)] byte numBytesToShift)
+        {
+            if (Sse2.IsSupported)
+            {
+                return Sse2.ShiftRightLogical128BitLane(operand, numBytesToShift);
+            }
+
+            if (AdvSimd.IsSupported)
+            {
+                return AdvSimd.ExtractVector128(operand.AsByte(), Vector128<byte>.Zero, numBytesToShift).AsUInt64();
+            }
+
+            ThrowHelper.ThrowUnreachableException();
+            return default;
+        }
+
+        // We check for little endian byte order here in case we're ever on ARM in big endian mode.
+        // All of these checks except the length check are elided by JIT, so the JITted implementation
+        // will be either a return false or a length check against a constant. This means this method
+        // should be inlined into the caller.
+        private static bool CanBeVectorized(ReadOnlySpan<byte> source) =>
+            BitConverter.IsLittleEndian
+            && (Pclmulqdq.IsSupported || (Aes.IsSupported && AdvSimd.IsSupported))
+            && source.Length >= Vector128<byte>.Count * 4;
+
+        // Processes the bytes in source in 64 byte chunks using carryless/polynomial multiplication intrinsics,
+        // followed by processing 16 byte chunks, and then processing remaining bytes individually. Requires
+        // little endian byte order and support for PCLMULQDQ intrinsics on Intel architecture or AES and
+        // AdvSimd intrinsics on ARM architecture. Based on the algorithm put forth in the Intel paper "Fast CRC
+        // Computation for Generic Polynomials Using PCLMULQDQ Instruction" in December, 2009.
+        // https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc32_ieee_by4.asm
+        // https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs#L80
+        private static uint UpdateVectorized(uint crc, ReadOnlySpan<byte> source)
+        {
+            Debug.Assert(CanBeVectorized(source), "source cannot be vectorized.");
+
+            // Work with a reference to where we're at in the ReadOnlySpan and a local length
+            // to avoid extraneous range checks.
+            ref byte srcRef = ref MemoryMarshal.GetReference(source);
+            int length = source.Length;
+
+            Vector128<ulong> x1 = Vector128.LoadUnsafe(ref srcRef).AsUInt64();
+            Vector128<ulong> x2 = Vector128.LoadUnsafe(ref srcRef, 16).AsUInt64();
+            Vector128<ulong> x3 = Vector128.LoadUnsafe(ref srcRef, 32).AsUInt64();
+            Vector128<ulong> x4 = Vector128.LoadUnsafe(ref srcRef, 48).AsUInt64();
+            Vector128<ulong> x5;
+
+            x1 ^= Vector128.CreateScalar(crc).AsUInt64();
+            Vector128<ulong> x0 = Vector128.Create(0x0154442bd4UL, 0x01c6e41596UL); // k1, k2
+
+            srcRef = ref Unsafe.Add(ref srcRef, Vector128<byte>.Count * 4);
+            length -= Vector128<byte>.Count * 4;
+
+            // Parallel fold blocks of 64, if any.
+            while (length >= Vector128<byte>.Count * 4)
+            {
+                x5 = CarrylessMultiplyLower(x1, x0);
+                Vector128<ulong> x6 = CarrylessMultiplyLower(x2, x0);
+                Vector128<ulong> x7 = CarrylessMultiplyLower(x3, x0);
+                Vector128<ulong> x8 = CarrylessMultiplyLower(x4, x0);
+
+                x1 = CarrylessMultiplyUpper(x1, x0);
+                x2 = CarrylessMultiplyUpper(x2, x0);
+                x3 = CarrylessMultiplyUpper(x3, x0);
+                x4 = CarrylessMultiplyUpper(x4, x0);
+
+                Vector128<ulong> y5 = Vector128.LoadUnsafe(ref srcRef).AsUInt64();
+                Vector128<ulong> y6 = Vector128.LoadUnsafe(ref srcRef, 16).AsUInt64();
+                Vector128<ulong> y7 = Vector128.LoadUnsafe(ref srcRef, 32).AsUInt64();
+                Vector128<ulong> y8 = Vector128.LoadUnsafe(ref srcRef, 48).AsUInt64();
+
+                x1 ^= x5;
+                x2 ^= x6;
+                x3 ^= x7;
+                x4 ^= x8;
+
+                x1 ^= y5;
+                x2 ^= y6;
+                x3 ^= y7;
+                x4 ^= y8;
+
+                srcRef = ref Unsafe.Add(ref srcRef, Vector128<byte>.Count * 4);
+                length -= Vector128<byte>.Count * 4;
+            }
+
+            // Fold into 128-bits.
+            x0 = Vector128.Create(0x01751997d0UL, 0x00ccaa009eUL); // k3, k4
+
+            x5 = CarrylessMultiplyLower(x1, x0);
+            x1 = CarrylessMultiplyUpper(x1, x0);
+            x1 ^= x2;
+            x1 ^= x5;
+
+            x5 = CarrylessMultiplyLower(x1, x0);
+            x1 = CarrylessMultiplyUpper(x1, x0);
+            x1 ^= x3;
+            x1 ^= x5;
+
+            x5 = CarrylessMultiplyLower(x1, x0);
+            x1 = CarrylessMultiplyUpper(x1, x0);
+            x1 ^= x4;
+            x1 ^= x5;
+
+            // Single fold blocks of 16, if any.
+            while (length >= Vector128<byte>.Count)
+            {
+                x2 = Vector128.LoadUnsafe(ref srcRef).AsUInt64();
+
+                x5 = CarrylessMultiplyLower(x1, x0);
+                x1 = CarrylessMultiplyUpper(x1, x0);
+                x1 ^= x2;
+                x1 ^= x5;
+
+                srcRef = ref Unsafe.Add(ref srcRef, Vector128<byte>.Count);
+                length -= Vector128<byte>.Count;
+            }
+
+            // Fold 128 bits to 64 bits.
+            x2 = CarrylessMultiplyLeftLowerRightUpper(x1, x0);
+            x3 = Vector128.Create(~0, 0, ~0, 0).AsUInt64();
+            x1 = ShiftRightBytesInVector(x1, 8);
+            x1 ^= x2;
+
+            x0 = Vector128.CreateScalar(0x0163cd6124UL); // k5, k0
+
+            x2 = ShiftRightBytesInVector(x1, 4);
+            x1 &= x3;
+            x1 = CarrylessMultiplyLower(x1, x0);
+            x1 ^= x2;
+
+            // Reduce to 32 bits.
+            x0 = Vector128.Create(0x01db710641UL, 0x01f7011641UL); // polynomial
+
+            x2 = x1 & x3;
+            x2 = CarrylessMultiplyLeftLowerRightUpper(x2, x0);
+            x2 &= x3;
+            x2 = CarrylessMultiplyLower(x2, x0);
+            x1 ^= x2;
+
+            // Process the remaining bytes, if any
+            uint result = x1.AsUInt32().GetElement(1);
+            return length > 0
+                ? UpdateScalar(result, MemoryMarshal.CreateReadOnlySpan(ref srcRef, length))
+                : result;
+        }
+    }
+}
diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs
@@ -169,6 +169,32 @@ public static uint HashToUInt32(ReadOnlySpan<byte> source) =>
 
         private static uint Update(uint crc, ReadOnlySpan<byte> source)
         {
+#if NET7_0_OR_GREATER
+            if (CanBeVectorized(source))
+            {
+                return UpdateVectorized(crc, source);
+            }
+#endif
+
+            return UpdateScalar(crc, source);
+        }
+
+        private static uint UpdateScalar(uint crc, ReadOnlySpan<byte> source)
+        {
+#if NET6_0_OR_GREATER
+            // Use ARM intrinsics for CRC if available. This is used for the trailing bytes on the vectorized path
+            // and is the primary method if the vectorized path is unavailable.
+            if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported)
+            {
+                return UpdateScalarArm64(crc, source);
+            }
+
+            if (System.Runtime.Intrinsics.Arm.Crc32.IsSupported)
+            {
+                return UpdateScalarArm32(crc, source);
+            }
+#endif
+
             ReadOnlySpan<uint> crcLookup = CrcLookup;
             for (int i = 0; i < source.Length; i++)
             {

diff --git a/src/libraries/System.IO.Hashing/src/System/ThrowHelper.cs b/src/libraries/System.IO.Hashing/src/System/ThrowHelper.cs
@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+
+namespace System
+{
+    internal static partial class ThrowHelper
+    {
+        [DoesNotReturn]
+        internal static void ThrowUnreachableException() => throw new UnreachableException();
+    }
+}