From 4020e9ce0620ca2e482a61cccd9910aa675f5100 Mon Sep 17 00:00:00 2001 From: wfurt Date: Wed, 14 Jun 2023 18:22:51 +0200 Subject: [PATCH 1/5] add zero byte read to SslStream --- .../System/IO/StreamConformanceTests.cs | 12 ++-- .../src/System/Net/Security/SslStream.IO.cs | 71 +++++++++---------- .../SslStreamConformanceTests.cs | 1 + 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/libraries/Common/tests/StreamConformanceTests/System/IO/StreamConformanceTests.cs b/src/libraries/Common/tests/StreamConformanceTests/System/IO/StreamConformanceTests.cs index 5a2c897571ccf9..bdc402758c4127 100644 --- a/src/libraries/Common/tests/StreamConformanceTests/System/IO/StreamConformanceTests.cs +++ b/src/libraries/Common/tests/StreamConformanceTests/System/IO/StreamConformanceTests.cs @@ -2739,6 +2739,8 @@ public abstract class WrappingConnectedStreamConformanceTests : ConnectedStreamC /// protected virtual bool ZeroByteReadPerformsZeroByteReadOnUnderlyingStream => false; + protected virtual bool ExtraZeroByteReadsAllowed => false; + [Theory] [InlineData(false)] [InlineData(true)] @@ -2938,7 +2940,7 @@ public virtual async Task ZeroByteRead_PerformsZeroByteReadOnUnderlyingStreamWhe using StreamPair innerStreams = ConnectedStreams.CreateBidirectional(); (Stream innerWriteable, Stream innerReadable) = GetReadWritePair(innerStreams); - var tracker = new ZeroByteReadTrackingStream(innerReadable); + var tracker = new ZeroByteReadTrackingStream(innerReadable, ExtraZeroByteReadsAllowed); using StreamPair streams = await CreateWrappedConnectedStreamsAsync((innerWriteable, tracker)); (Stream writeable, Stream readable) = GetReadWritePair(streams); @@ -2993,9 +2995,11 @@ public virtual async Task ZeroByteRead_PerformsZeroByteReadOnUnderlyingStreamWhe private sealed class ZeroByteReadTrackingStream : DelegatingStream { private TaskCompletionSource? _signal; + private bool _extraZeroByteReadsAllowed; - public ZeroByteReadTrackingStream(Stream innerStream) : base(innerStream) + public ZeroByteReadTrackingStream(Stream innerStream, bool extraZeroByteReadsAllowed = false) : base(innerStream) { + _extraZeroByteReadsAllowed = extraZeroByteReadsAllowed; } public Task WaitForZeroByteReadAsync() @@ -3014,13 +3018,13 @@ private void CheckForZeroByteRead(int bufferLength) if (bufferLength == 0) { var signal = _signal; - if (signal is null) + if (signal is null && !_extraZeroByteReadsAllowed) { throw new Exception("Unexpected zero byte read"); } _signal = null; - signal.SetResult(); + signal?.SetResult(); } } diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs index 7ea62aaa705714..8eebf2d99dcc32 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs @@ -27,6 +27,8 @@ public partial class SslStream private const int HandshakeTypeOffsetSsl2 = 2; // Offset of HelloType in Sslv2 and Unified frames private const int HandshakeTypeOffsetTls = 5; // Offset of HelloType in Sslv3 and TLS frames + private const int UnknownTlsFrameLength = int.MaxValue; // frame too short to determine length + private bool _receivedEOF; // Used by Telemetry to ensure we log connection close exactly once @@ -211,12 +213,10 @@ private async Task RenegotiateAsync(CancellationToken cancellationTo throw SslStreamPal.GetException(status); } - _buffer.EnsureAvailableSpace(InitialHandshakeBufferSize); - ProtocolToken message; do { - int frameSize = await ReceiveTlsFrameAsync(cancellationToken).ConfigureAwait(false); + int frameSize = await ReceiveHandshakeFrameAsync(cancellationToken).ConfigureAwait(false); ProcessTlsFrame(frameSize, out message); if (message.Size > 0) @@ -291,7 +291,7 @@ private async Task ForceAuthenticationAsync(bool receiveFirst, byte[ while (!handshakeCompleted) { - int frameSize = await ReceiveTlsFrameAsync(cancellationToken).ConfigureAwait(false); + int frameSize = await ReceiveHandshakeFrameAsync(cancellationToken).ConfigureAwait(false); ProcessTlsFrame(frameSize, out message); ReadOnlyMemory payload = default; @@ -359,10 +359,10 @@ private async Task ForceAuthenticationAsync(bool receiveFirst, byte[ } // This method will make sure we have at least one full TLS frame buffered. - private async ValueTask ReceiveTlsFrameAsync(CancellationToken cancellationToken) + private async ValueTask ReceiveHandshakeFrameAsync(CancellationToken cancellationToken) where TIOAdapter : IReadWriteAdapter { - int frameSize = await EnsureFullTlsFrameAsync(cancellationToken).ConfigureAwait(false); + int frameSize = await EnsureFullTlsFrameAsync(cancellationToken, InitialHandshakeBufferSize).ConfigureAwait(false); if (frameSize == 0) { @@ -699,37 +699,37 @@ private void ReturnReadBufferIfEmpty() private bool HaveFullTlsFrame(out int frameSize) { - if (_buffer.EncryptedLength < TlsFrameHelper.HeaderSize) - { - frameSize = int.MaxValue; - return false; - } - frameSize = GetFrameSize(_buffer.EncryptedReadOnlySpan); return _buffer.EncryptedLength >= frameSize; } [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - private async ValueTask EnsureFullTlsFrameAsync(CancellationToken cancellationToken) + private async ValueTask EnsureFullTlsFrameAsync(CancellationToken cancellationToken, int estimatedSize) where TIOAdapter : IReadWriteAdapter { - int frameSize; - if (HaveFullTlsFrame(out frameSize)) + if (HaveFullTlsFrame(out int frameSize)) { return frameSize; } - if (frameSize != int.MaxValue) + // Do zero byte read to conserve resources: + // 1. We postpone allocation of read buffers. For example Kestrel may have idle connections without buffer until new request comes in. + // 2. The underlying stream may need to allocate GCHandle for async IO. If we postpone that until we have some data, we may be able to avoid it. + // That is not only important for CPU but also for memory fragmentation as pinned buffers are difficult to deal with. + // Note that if the underlying stream does not support blocking on zero byte reads, then this will + // complete immediately and won't save any memory, but will still function correctly. + await TIOAdapter.ReadAsync(InnerStream, Memory.Empty, cancellationToken).ConfigureAwait(false); + + if (frameSize == UnknownTlsFrameLength) { - // make sure we have space for the whole frame - _buffer.EnsureAvailableSpace(frameSize - _buffer.EncryptedLength); + // We do not have enough data to determine frame size. Use provided estimate e.g. + // full TLS frame for read, and somewhat shorter frame for handshake or renegotiation + _buffer.EnsureAvailableSpace(estimatedSize); } else { - // move existing data to the beginning of the buffer (they will - // be couple of bytes only, otherwise we would have entire - // header and know exact size) - _buffer.EnsureAvailableSpace(_buffer.Capacity - _buffer.EncryptedLength); + // make sure we have space for the whole frame + _buffer.EnsureAvailableSpace(frameSize - _buffer.EncryptedLength); } while (_buffer.EncryptedLength < frameSize) @@ -819,11 +819,12 @@ private async ValueTask ReadAsyncInternal(Memory buffer, try { int processedLength = 0; + int nextTlsFrameLength = UnknownTlsFrameLength; if (_buffer.DecryptedLength != 0) { processedLength = CopyDecryptedData(buffer); - if (processedLength == buffer.Length || !HaveFullTlsFrame(out _)) + if (processedLength == buffer.Length || !HaveFullTlsFrame(out nextTlsFrameLength)) { // We either filled whole buffer or used all buffered frames. return processedLength; @@ -832,32 +833,19 @@ private async ValueTask ReadAsyncInternal(Memory buffer, buffer = buffer.Slice(processedLength); } - if (_receivedEOF) + if (_receivedEOF && nextTlsFrameLength == UnknownTlsFrameLength) { + // there should be no frames waiting for processing Debug.Assert(_buffer.EncryptedLength == 0); // We received EOF during previous read but had buffered data to return. return 0; } - if (buffer.Length == 0 && _buffer.ActiveLength == 0) - { - // User requested a zero-byte read, and we have no data available in the buffer for processing. - // This zero-byte read indicates their desire to trade off the extra cost of a zero-byte read - // for reduced memory consumption when data is not immediately available. - // So, we will issue our own zero-byte read against the underlying stream and defer buffer allocation - // until data is actually available from the underlying stream. - // Note that if the underlying stream does not supporting blocking on zero byte reads, then this will - // complete immediately and won't save any memory, but will still function correctly. - await TIOAdapter.ReadAsync(InnerStream, Memory.Empty, cancellationToken).ConfigureAwait(false); - } - Debug.Assert(_buffer.DecryptedLength == 0); - _buffer.EnsureAvailableSpace(ReadBufferSize - _buffer.ActiveLength); - while (true) { - int payloadBytes = await EnsureFullTlsFrameAsync(cancellationToken).ConfigureAwait(false); + int payloadBytes = await EnsureFullTlsFrameAsync(cancellationToken, ReadBufferSize).ConfigureAwait(false); if (payloadBytes == 0) { _receivedEOF = true; @@ -1009,6 +997,11 @@ private int CopyDecryptedData(Memory buffer) // Returns TLS Frame size including header size. private int GetFrameSize(ReadOnlySpan buffer) { + if (buffer.Length < TlsFrameHelper.HeaderSize) + { + return UnknownTlsFrameLength; + } + if (!TlsFrameHelper.TryGetFrameHeader(buffer, ref _lastFrame.Header)) { throw new IOException(SR.net_ssl_io_frame); diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamConformanceTests.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamConformanceTests.cs index 6b47328446b7a8..c89c4802c8d467 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamConformanceTests.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamConformanceTests.cs @@ -16,6 +16,7 @@ public abstract class SslStreamConformanceTests : WrappingConnectedStreamConform protected override bool BlocksOnZeroByteReads => true; protected override bool ZeroByteReadPerformsZeroByteReadOnUnderlyingStream => true; protected override Type UnsupportedConcurrentExceptionType => typeof(NotSupportedException); + protected override bool ExtraZeroByteReadsAllowed => true; protected virtual SslProtocols GetSslProtocols() => SslProtocols.None; From be72b47f2c93378f170534ee480657152beec0b2 Mon Sep 17 00:00:00 2001 From: wfurt Date: Thu, 15 Jun 2023 19:52:42 +0200 Subject: [PATCH 2/5] fix test --- .../FunctionalTests/ResponseStreamZeroByteReadTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs index a5312b12aec846..df1d771d328207 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs @@ -123,7 +123,12 @@ public async Task ZeroByteRead_IssuesZeroByteReadOnUnderlyingStream(StreamConfor using HttpResponseMessage response = await clientTask.WaitAsync(TestHelper.PassingTestTimeout); using Stream clientStream = response.Content.ReadAsStream(); - Assert.False(sawZeroByteRead.Task.IsCompleted); + + if (!useSsl) + { + // SslStream does zero byte reads under the cover + Assert.False(sawZeroByteRead.Task.IsCompleted); + } Task zeroByteReadTask = Task.Run(() => StreamConformanceTests.ReadAsync(readMode, clientStream, Array.Empty(), 0, 0, CancellationToken.None)); Assert.False(zeroByteReadTask.IsCompleted); From 0e7bdbb748b09215edd827c875773f62990bc258 Mon Sep 17 00:00:00 2001 From: Tomas Weinfurt Date: Fri, 16 Jun 2023 10:02:40 +0200 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Stephen Toub --- .../tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs | 2 +- .../src/System/Net/Security/SslStream.IO.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs index df1d771d328207..74ab2087a7533c 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/ResponseStreamZeroByteReadTests.cs @@ -126,7 +126,7 @@ public async Task ZeroByteRead_IssuesZeroByteReadOnUnderlyingStream(StreamConfor if (!useSsl) { - // SslStream does zero byte reads under the cover + // SslStream does zero byte reads under the covers Assert.False(sawZeroByteRead.Task.IsCompleted); } diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs index 8eebf2d99dcc32..8cd024bfa68f15 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs @@ -713,8 +713,8 @@ private async ValueTask EnsureFullTlsFrameAsync(CancellationTok } // Do zero byte read to conserve resources: - // 1. We postpone allocation of read buffers. For example Kestrel may have idle connections without buffer until new request comes in. - // 2. The underlying stream may need to allocate GCHandle for async IO. If we postpone that until we have some data, we may be able to avoid it. + // 1. We postpone allocation of read buffers (on all platforms). For example Kestrel may have idle connections without buffer until new request comes in. + // 2. The underlying stream may need to allocate GCHandle for async IO (on Windows). If we postpone that until we have some data, we may be able to avoid it. // That is not only important for CPU but also for memory fragmentation as pinned buffers are difficult to deal with. // Note that if the underlying stream does not support blocking on zero byte reads, then this will // complete immediately and won't save any memory, but will still function correctly. From 89f925006401db608d99f766560a2fc45e9390ff Mon Sep 17 00:00:00 2001 From: wfurt Date: Mon, 19 Jun 2023 01:44:45 -0700 Subject: [PATCH 4/5] feedback --- .../src/System/Net/Security/SslStream.IO.cs | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs index 8cd024bfa68f15..eca3c7dc670c69 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs @@ -712,25 +712,13 @@ private async ValueTask EnsureFullTlsFrameAsync(CancellationTok return frameSize; } - // Do zero byte read to conserve resources: - // 1. We postpone allocation of read buffers (on all platforms). For example Kestrel may have idle connections without buffer until new request comes in. - // 2. The underlying stream may need to allocate GCHandle for async IO (on Windows). If we postpone that until we have some data, we may be able to avoid it. - // That is not only important for CPU but also for memory fragmentation as pinned buffers are difficult to deal with. - // Note that if the underlying stream does not support blocking on zero byte reads, then this will - // complete immediately and won't save any memory, but will still function correctly. - await TIOAdapter.ReadAsync(InnerStream, Memory.Empty, cancellationToken).ConfigureAwait(false); - if (frameSize == UnknownTlsFrameLength) - { - // We do not have enough data to determine frame size. Use provided estimate e.g. - // full TLS frame for read, and somewhat shorter frame for handshake or renegotiation - _buffer.EnsureAvailableSpace(estimatedSize); - } - else - { - // make sure we have space for the whole frame - _buffer.EnsureAvailableSpace(frameSize - _buffer.EncryptedLength); - } + // If we don't have enough data to determine the frame size, use the provided estimate + // (e.g. a full TLS frame for reads, and a somewhat shorter frame for handshake / renegotiation). + // If we do know the frame size, ensure we have space for the whole frame. + _buffer.EnsureAvailableSpace(frameSize == UnknownTlsFrameLength ? + estimatedSize : + frameSize - _buffer.EncryptedLength); while (_buffer.EncryptedLength < frameSize) { From 06e03ff1e51321b130849efea71151722901e5a8 Mon Sep 17 00:00:00 2001 From: wfurt Date: Mon, 19 Jun 2023 09:48:29 -0700 Subject: [PATCH 5/5] add back missing line --- .../System.Net.Security/src/System/Net/Security/SslStream.IO.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs index eca3c7dc670c69..d7438c0a0ff82d 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.IO.cs @@ -712,6 +712,7 @@ private async ValueTask EnsureFullTlsFrameAsync(CancellationTok return frameSize; } + await TIOAdapter.ReadAsync(InnerStream, Memory.Empty, cancellationToken).ConfigureAwait(false); // If we don't have enough data to determine the frame size, use the provided estimate // (e.g. a full TLS frame for reads, and a somewhat shorter frame for handshake / renegotiation). @@ -794,6 +795,7 @@ private SecurityStatusPal DecryptData(int frameSize) private async ValueTask ReadAsyncInternal(Memory buffer, CancellationToken cancellationToken) where TIOAdapter : IReadWriteAdapter { + // Throw first if we already have exception. // Check for disposal is not atomic so we will check again below. ThrowIfExceptionalOrNotAuthenticated();