wip reversal optimizations

dotnet · stephentoub · Jul 11, 2024 · May 24, 2024 · May 24, 2024 · May 26, 2024
commit 5ac29f36906e6afff5b3b1835d7155ff524e4bf6
diff --git a/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj b/src/libraries/System.Text.RegularExpressions/src/System.Text.RegularExpressions.csproj
@@ -5,6 +5,8 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);SYSTEM_TEXT_REGULAREXPRESSIONS</DefineConstants>
     <UseCompilerGeneratedDocXmlFile>false</UseCompilerGeneratedDocXmlFile>
+<!--    <NoWarn>IL2026;IL2075;IDE0059;CA1823</NoWarn>-->
+
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/...rExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.Automata.cs b/...rExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.Automata.cs
@@ -50,6 +50,15 @@ internal sealed partial class SymbolicRegexMatcher<TSet>
         /// </summary>
         private bool[] _canBeAcceleratedArray;
 
+#if DEBUG
+        // private readonly Action<string> _wout = st =>
+        // {
+        //     var a_cons = System.Reflection.Assembly.Load("System.Console");
+        //     var t_cons = a_cons.GetType("System.Console")!;
+        //     var wl = t_cons.GetMethod("WriteLine", [typeof(string)]);
+        //     wl!.Invoke(null, [st]);
+        // };
+#endif
         /// <summary>
         /// The transition function for DFA mode.
         /// Each state has a range of consecutive entries for each minterm ID. A range of size 2^L, where L is
@@ -162,6 +171,69 @@ private MatchingState<TSet> GetOrCreateState(SymbolicRegexNode<TSet> node, uint
             return GetOrCreateState_NoLock(node, prevCharKind);
         }
 
+        /// <summary>
+        /// Optimized reversal state computation which takes skips the fixed length parts
+        /// </summary>
+        /// <param name="node"></param>
+        /// <returns></returns>
+        private (int, MatchingState<TSet>) CreateOptimizedReversal(SymbolicRegexNode<TSet> node)
+        {
+            var pos = 0;
+            var current = node;
+            var canLoop = true;
+            var incrPos = new Func<(int, SymbolicRegexNode<TSet>), (bool, SymbolicRegexNode<TSet>)>(value =>
+            {
+                pos += value.Item1;
+                return (true, value.Item2);
+            });
+            var decrLoop = new Func<SymbolicRegexNode<TSet>, (bool, SymbolicRegexNode<TSet>)>(value =>
+            {
+                var concat = value;
+                var loop = concat._left;
+                switch (loop!._left!.Kind)
+                {
+                    case SymbolicRegexNodeKind.Singleton:
+                        if (loop._lower == loop._upper)
+                        {
+                            pos += loop._lower;
+                            return (true, concat._right!);
+                        }
+                        if (loop._lower > 0)
+                        {
+                            var delta = loop._upper - loop._lower;
+                            var newLeft = _builder.CreateLoop(loop._left, loop.IsLazy, 0, delta);
+                            var newNode = _builder.CreateConcat(newLeft, concat._right!);
+                            pos += loop._lower;
+                            return (true, newNode);
+                        }
+                        return (false, concat);
+                    default:
+                        return (false, concat);
+                }
+            });
+            while (canLoop)
+            {
+#if DEBUG
+                // _wout($"{pos} {current._kind} l:{current._left!._kind} {current}");
+#endif
+                (bool loop, SymbolicRegexNode<TSet> next) = current switch
+                {
+                    {_kind:SymbolicRegexNodeKind.Concat, _left._kind: SymbolicRegexNodeKind.CaptureEnd} =>
+                        (true, current._right!),
+                    {_kind: SymbolicRegexNodeKind.Concat, _left._kind: SymbolicRegexNodeKind.BoundaryAnchor } =>
+                        (true, current._right!),
+                    {_kind:SymbolicRegexNodeKind.Concat, _left._kind: SymbolicRegexNodeKind.Singleton} =>
+                        incrPos((1, current._right!)),
+                    {_kind: SymbolicRegexNodeKind.Concat, _left._kind: SymbolicRegexNodeKind.Loop } =>
+                        decrLoop(current),
+                    _ => (false, current)
+                };
+                canLoop = loop;
+                current = next;
+            }
+            return (pos, GetOrCreateState_NoLock(_builder.CreateDisableBacktrackingSimulation(current), 0, false));
+        }
+
         /// <summary>
         /// Create a state with given node and previous character context.
         /// </summary>

diff --git a/...xt.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.cs b/...xt.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexMatcher.cs
@@ -96,6 +96,8 @@ internal sealed partial class SymbolicRegexMatcher<TSet> : SymbolicRegexMatcher
         /// <remarks>If the pattern doesn't contain any anchors, there will only be a single initial state.</remarks>
         private readonly MatchingState<TSet>[] _reverseInitialStates;
 
+        private readonly (int, MatchingState<TSet>) _optimizedReversalState;
+
         /// <summary>Partition of the input space of sets.</summary>
         private readonly TSet[] _minterms;
 
@@ -172,6 +174,7 @@ private SymbolicRegexMatcher(SymbolicRegexBuilder<TSet> builder, SymbolicRegexNo
                 ((BitVectorSolver)(object)builder._solver)._classifier;
             _capsize = captureCount;
 
+
             // Initialization for fields in SymbolicRegexMatcher.Automata.cs
             _stateArray = new MatchingState<TSet>[InitialDfaStateCapacity];
             _stateFlagsArray = new StateFlags[InitialDfaStateCapacity];
@@ -262,6 +265,9 @@ private SymbolicRegexMatcher(SymbolicRegexBuilder<TSet> builder, SymbolicRegexNo
             }
             _reverseInitialStates = reverseInitialStates;
 
+            // Create optimized reversal
+            _optimizedReversalState = CreateOptimizedReversal(_pattern.Reverse(builder));
+
             // Maps a minterm ID to a character kind
             uint CalculateMintermIdKind(int mintermId)
             {
@@ -776,9 +782,29 @@ private int FindStartPosition<TInputReader, TNullabilityHandler>(ReadOnlySpan<ch
 
             // Get the starting state for the reverse pattern. This depends on previous character (which, because we're
             // going backwards, is character number i).
-            var currentState = new CurrentState(_reverseInitialStates[GetCharKind<TInputReader>(input, i)]);
-
+            CurrentState currentState;
             int lastStart = -1; // invalid sentinel value
+            // if possible use optimized reversal instead
+            if (_optimizedReversalState.Item1 > 0)
+            {
+                i -= _optimizedReversalState.Item1;
+                currentState = new CurrentState(_optimizedReversalState.Item2);
+                // anchor variant may need context to be computed if nullable
+                if (_pattern._info.ContainsSomeAnchor && _canBeNullableArray[currentState.DfaStateId])
+                {
+                    int positionId = TInputReader.GetPositionId(this, input, i);
+                    if (TNullabilityHandler.IsNullableAt<DfaStateHandler>(this,
+                            in currentState, positionId,
+                            DfaStateHandler.GetStateFlags(this, in currentState)))
+                    {
+                        lastStart = i;
+                    }
+                }
+            }
+            else
+            {
+                currentState = new CurrentState(_reverseInitialStates[GetCharKind<TInputReader>(input, i)]);
+            }
 
             // Walk backwards to the furthest accepting state of the reverse pattern but no earlier than matchStartBoundary.
             while (true)

diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/NonBacktrackingTests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/NonBacktrackingTests.cs
@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections;
+using System.Collections.Generic;
+using Xunit;
+
+namespace System.Text.RegularExpressions.Tests
+{
+    /// <summary>
+    /// TODO: Create tests here later
+    /// </summary>
+    public static partial class NonBacktrackingTests
+    {
+
+        // [Fact]
+        // public static void Test()
+        // {
+        // }
+
+    }
+}
diff --git a/...Text.RegularExpressions/tests/FunctionalTests/System.Text.RegularExpressions.Tests.csproj b/...Text.RegularExpressions/tests/FunctionalTests/System.Text.RegularExpressions.Tests.csproj
@@ -19,6 +19,7 @@
     <Compile Include="GroupCollectionTests.cs" />
     <Compile Include="MatchCollectionTests.cs" />
     <Compile Include="MonoRegexTests.cs" />
+    <Compile Include="NonBacktrackingTests.cs" />
     <Compile Include="Regex.CompileToAssembly.Tests.cs" />
     <Compile Include="Regex.Ctor.Tests.cs" />
     <Compile Include="Regex.Cache.Tests.cs" />