diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index feb4d4fe851c8c..8ea754bc237f05 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -1196,3 +1196,46 @@ https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author hereby disclaims copyright to this source + +License for Fast CRC Computation +-------------------------------------- + +https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc32_ieee_by4.asm + +Copyright(c) 2011-2015 Intel Corporation All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License for C# Implementation of Fast CRC Computation +----------------------------------------------------- + +https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs + +Copyright (c) Six Labors. +Licensed under the Apache License, Version 2.0. + +Available at +https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/LICENSE diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 43f0bcf90d9c68..85072ee61e9d98 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -11,8 +11,14 @@ System.IO.Hashing.XxHash32 + + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Arm.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Arm.cs new file mode 100644 index 00000000000000..dae53164389da0 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Arm.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using ArmCrc = System.Runtime.Intrinsics.Arm.Crc32; + +namespace System.IO.Hashing +{ + public partial class Crc32 + { + private static uint UpdateScalarArm64(uint crc, ReadOnlySpan source) + { + Debug.Assert(ArmCrc.Arm64.IsSupported, "ARM CRC support is required."); + + // Compute in 8 byte chunks + if (source.Length >= sizeof(ulong)) + { + ref byte ptr = ref MemoryMarshal.GetReference(source); + int longLength = source.Length & ~0x7; // Exclude trailing bytes not a multiple of 8 + + for (int i = 0; i < longLength; i += sizeof(ulong)) + { + crc = ArmCrc.Arm64.ComputeCrc32(crc, + Unsafe.ReadUnaligned(ref Unsafe.Add(ref ptr, i))); + } + + source = source.Slice(longLength); + } + + // Compute remaining bytes + for (int i = 0; i < source.Length; i++) + { + crc = ArmCrc.ComputeCrc32(crc, source[i]); + } + + return crc; + } + + private static uint UpdateScalarArm32(uint crc, ReadOnlySpan source) + { + Debug.Assert(ArmCrc.IsSupported, "ARM CRC support is required."); + + // Compute in 4 byte chunks + if (source.Length >= sizeof(uint)) + { + ref byte ptr = ref MemoryMarshal.GetReference(source); + int intLength = source.Length & ~0x3; // Exclude trailing bytes not a multiple of 4 + + for (int i = 0; i < intLength; i += sizeof(uint)) + { + crc = ArmCrc.ComputeCrc32(crc, + Unsafe.ReadUnaligned(ref Unsafe.Add(ref ptr, i))); + } + + source = source.Slice(intLength); + } + + // Compute remaining bytes + for (int i = 0; i < source.Length; i++) + { + crc = ArmCrc.ComputeCrc32(crc, source[i]); + } + + return crc; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Vectorized.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Vectorized.cs new file mode 100644 index 00000000000000..f584efa6e7d7af --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Vectorized.cs @@ -0,0 +1,216 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Arm; +using Aes = System.Runtime.Intrinsics.Arm.Aes; + +namespace System.IO.Hashing +{ + public partial class Crc32 + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CarrylessMultiplyLower(Vector128 left, Vector128 right) + { + if (Pclmulqdq.IsSupported) + { + return Pclmulqdq.CarrylessMultiply(left, right, 0x00); + } + + if (Aes.IsSupported) + { + return Aes.PolynomialMultiplyWideningLower(left.GetLower(), right.GetLower()); + } + + ThrowHelper.ThrowUnreachableException(); + return default; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CarrylessMultiplyUpper(Vector128 left, Vector128 right) + { + if (Pclmulqdq.IsSupported) + { + return Pclmulqdq.CarrylessMultiply(left, right, 0x11); + } + + if (Aes.IsSupported) + { + return Aes.PolynomialMultiplyWideningUpper(left, right); + } + + ThrowHelper.ThrowUnreachableException(); + return default; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CarrylessMultiplyLeftLowerRightUpper(Vector128 left, Vector128 right) + { + if (Pclmulqdq.IsSupported) + { + return Pclmulqdq.CarrylessMultiply(left, right, 0x10); + } + + if (Aes.IsSupported) + { + return Aes.PolynomialMultiplyWideningLower(left.GetLower(), right.GetUpper()); + } + + ThrowHelper.ThrowUnreachableException(); + return default; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ShiftRightBytesInVector(Vector128 operand, + [ConstantExpected(Max = (byte)15)] byte numBytesToShift) + { + if (Sse2.IsSupported) + { + return Sse2.ShiftRightLogical128BitLane(operand, numBytesToShift); + } + + if (AdvSimd.IsSupported) + { + return AdvSimd.ExtractVector128(operand.AsByte(), Vector128.Zero, numBytesToShift).AsUInt64(); + } + + ThrowHelper.ThrowUnreachableException(); + return default; + } + + // We check for little endian byte order here in case we're ever on ARM in big endian mode. + // All of these checks except the length check are elided by JIT, so the JITted implementation + // will be either a return false or a length check against a constant. This means this method + // should be inlined into the caller. + private static bool CanBeVectorized(ReadOnlySpan source) => + BitConverter.IsLittleEndian + && (Pclmulqdq.IsSupported || (Aes.IsSupported && AdvSimd.IsSupported)) + && source.Length >= Vector128.Count * 4; + + // Processes the bytes in source in 64 byte chunks using carryless/polynomial multiplication intrinsics, + // followed by processing 16 byte chunks, and then processing remaining bytes individually. Requires + // little endian byte order and support for PCLMULQDQ intrinsics on Intel architecture or AES and + // AdvSimd intrinsics on ARM architecture. Based on the algorithm put forth in the Intel paper "Fast CRC + // Computation for Generic Polynomials Using PCLMULQDQ Instruction" in December, 2009. + // https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc32_ieee_by4.asm + // https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs#L80 + private static uint UpdateVectorized(uint crc, ReadOnlySpan source) + { + Debug.Assert(CanBeVectorized(source), "source cannot be vectorized."); + + // Work with a reference to where we're at in the ReadOnlySpan and a local length + // to avoid extraneous range checks. + ref byte srcRef = ref MemoryMarshal.GetReference(source); + int length = source.Length; + + Vector128 x1 = Vector128.LoadUnsafe(ref srcRef).AsUInt64(); + Vector128 x2 = Vector128.LoadUnsafe(ref srcRef, 16).AsUInt64(); + Vector128 x3 = Vector128.LoadUnsafe(ref srcRef, 32).AsUInt64(); + Vector128 x4 = Vector128.LoadUnsafe(ref srcRef, 48).AsUInt64(); + Vector128 x5; + + x1 ^= Vector128.CreateScalar(crc).AsUInt64(); + Vector128 x0 = Vector128.Create(0x0154442bd4UL, 0x01c6e41596UL); // k1, k2 + + srcRef = ref Unsafe.Add(ref srcRef, Vector128.Count * 4); + length -= Vector128.Count * 4; + + // Parallel fold blocks of 64, if any. + while (length >= Vector128.Count * 4) + { + x5 = CarrylessMultiplyLower(x1, x0); + Vector128 x6 = CarrylessMultiplyLower(x2, x0); + Vector128 x7 = CarrylessMultiplyLower(x3, x0); + Vector128 x8 = CarrylessMultiplyLower(x4, x0); + + x1 = CarrylessMultiplyUpper(x1, x0); + x2 = CarrylessMultiplyUpper(x2, x0); + x3 = CarrylessMultiplyUpper(x3, x0); + x4 = CarrylessMultiplyUpper(x4, x0); + + Vector128 y5 = Vector128.LoadUnsafe(ref srcRef).AsUInt64(); + Vector128 y6 = Vector128.LoadUnsafe(ref srcRef, 16).AsUInt64(); + Vector128 y7 = Vector128.LoadUnsafe(ref srcRef, 32).AsUInt64(); + Vector128 y8 = Vector128.LoadUnsafe(ref srcRef, 48).AsUInt64(); + + x1 ^= x5; + x2 ^= x6; + x3 ^= x7; + x4 ^= x8; + + x1 ^= y5; + x2 ^= y6; + x3 ^= y7; + x4 ^= y8; + + srcRef = ref Unsafe.Add(ref srcRef, Vector128.Count * 4); + length -= Vector128.Count * 4; + } + + // Fold into 128-bits. + x0 = Vector128.Create(0x01751997d0UL, 0x00ccaa009eUL); // k3, k4 + + x5 = CarrylessMultiplyLower(x1, x0); + x1 = CarrylessMultiplyUpper(x1, x0); + x1 ^= x2; + x1 ^= x5; + + x5 = CarrylessMultiplyLower(x1, x0); + x1 = CarrylessMultiplyUpper(x1, x0); + x1 ^= x3; + x1 ^= x5; + + x5 = CarrylessMultiplyLower(x1, x0); + x1 = CarrylessMultiplyUpper(x1, x0); + x1 ^= x4; + x1 ^= x5; + + // Single fold blocks of 16, if any. + while (length >= Vector128.Count) + { + x2 = Vector128.LoadUnsafe(ref srcRef).AsUInt64(); + + x5 = CarrylessMultiplyLower(x1, x0); + x1 = CarrylessMultiplyUpper(x1, x0); + x1 ^= x2; + x1 ^= x5; + + srcRef = ref Unsafe.Add(ref srcRef, Vector128.Count); + length -= Vector128.Count; + } + + // Fold 128 bits to 64 bits. + x2 = CarrylessMultiplyLeftLowerRightUpper(x1, x0); + x3 = Vector128.Create(~0, 0, ~0, 0).AsUInt64(); + x1 = ShiftRightBytesInVector(x1, 8); + x1 ^= x2; + + x0 = Vector128.CreateScalar(0x0163cd6124UL); // k5, k0 + + x2 = ShiftRightBytesInVector(x1, 4); + x1 &= x3; + x1 = CarrylessMultiplyLower(x1, x0); + x1 ^= x2; + + // Reduce to 32 bits. + x0 = Vector128.Create(0x01db710641UL, 0x01f7011641UL); // polynomial + + x2 = x1 & x3; + x2 = CarrylessMultiplyLeftLowerRightUpper(x2, x0); + x2 &= x3; + x2 = CarrylessMultiplyLower(x2, x0); + x1 ^= x2; + + // Process the remaining bytes, if any + uint result = x1.AsUInt32().GetElement(1); + return length > 0 + ? UpdateScalar(result, MemoryMarshal.CreateReadOnlySpan(ref srcRef, length)) + : result; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs index f48dc32d8a68ac..3a844a60637b07 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs @@ -169,6 +169,32 @@ public static uint HashToUInt32(ReadOnlySpan source) => private static uint Update(uint crc, ReadOnlySpan source) { +#if NET7_0_OR_GREATER + if (CanBeVectorized(source)) + { + return UpdateVectorized(crc, source); + } +#endif + + return UpdateScalar(crc, source); + } + + private static uint UpdateScalar(uint crc, ReadOnlySpan source) + { +#if NET6_0_OR_GREATER + // Use ARM intrinsics for CRC if available. This is used for the trailing bytes on the vectorized path + // and is the primary method if the vectorized path is unavailable. + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + return UpdateScalarArm64(crc, source); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.IsSupported) + { + return UpdateScalarArm32(crc, source); + } +#endif + ReadOnlySpan crcLookup = CrcLookup; for (int i = 0; i < source.Length; i++) { diff --git a/src/libraries/System.IO.Hashing/src/System/ThrowHelper.cs b/src/libraries/System.IO.Hashing/src/System/ThrowHelper.cs new file mode 100644 index 00000000000000..a7034983d33f9b --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/ThrowHelper.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace System +{ + internal static partial class ThrowHelper + { + [DoesNotReturn] + internal static void ThrowUnreachableException() => throw new UnreachableException(); + } +} diff --git a/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs index 217d7bfe90c8b4..f056f823f805c2 100644 --- a/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs +++ b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs @@ -65,6 +65,21 @@ public static IEnumerable TestCases "The quick brown fox jumps over the lazy dog", "The quick brown fox jumps over the lazy dog"u8.ToArray(), "39A34F41"), + // Test a multiple of 64 bytes for vector optimizations + new TestCase( + "Lorem ipsum 128", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi quis iaculis nisl. Sed ornare sapien non nulla hendrerit viverra."u8.ToArray(), + "931A6737"), + // Test a multiple of 64 bytes + 16 bytes for vector optimizations + new TestCase( + "Lorem ipsum 144", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla libero est, semper in pharetra at, cursus id nulla. Class aptent taciti volutpat."u8.ToArray(), + "2B719549"), + // Test data that is > 64 bytes but not a multiple of 16 for vector optimizations + new TestCase( + "Lorem ipsum 1001", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer ac urna vitae nibh sagittis porttitor et vel ante. Ut molestie sit amet velit ac mattis. Sed ullamcorper nunc non neque imperdiet, vehicula bibendum sapien efficitur. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Suspendisse potenti. Duis sem dui, malesuada non pharetra at, feugiat id mi. Nulla facilisi. Fusce a scelerisque magna. Ut leo justo, auctor quis nisi et, sollicitudin pretium odio. Sed eu nibh mollis, pretium lectus nec, posuere nulla. Morbi ac euismod purus. Morbi rhoncus leo est, at volutpat nunc pretium in. Aliquam erat volutpat. Curabitur eu lacus mollis, varius lectus ut, tincidunt eros. Nullam a velit hendrerit, euismod magna id, fringilla sem. Phasellus scelerisque hendrerit est, vel imperdiet enim auctor a. Aenean vel ultricies nunc. Suspendisse ac tincidunt urna. Nulla tempor dolor ut ligula accumsan, tempus auctor massa gravida. Aenean non odio et augue pellena."u8.ToArray(), + "0464ED5F"), }; protected override NonCryptographicHashAlgorithm CreateInstance() => new Crc32();