From a1613c223097a1c9817dd7112dc1cbe7f8245dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leo=20Bl=C3=B6cher?= Date: Sun, 23 Apr 2023 16:55:56 +0000 Subject: [PATCH 01/46] http2: handle trailing colon in authorityAddr This change modifies the authorityAddr result for authorities with empty port information, such as "example.com:". Previously, such authorities passed through the function unchanged. This conflicts with the result from net/http's canonicalAddr, which returns "example.com:443" (for HTTPS). net/http's canonicalAddr result is passed to http2's upgradeFn (defined inside http2.configureTransports) from net/http's (*Transport).dialConn. The connection is then added to http2's cache under the canonicalAddr key. However, cache lookups are performed in (*Transport).RoundTripOpt using the result from authorityAddr applied directly to the input URL. The lookup thus fails if authorityAddr and canonicalAddr don't agree. http2's lookup error propagates upwards to net/http's (*Transport).roundTrip, where the request is retried. The end result is an infinite loop of the request being repeated, each time with a freshly dialed connection, that can only be stopped by a timeout. Aligning the results of http2's authorityAddr and net/http's canonicalAddr fixes the bug. While an authority with a trailing colon is invalid per URL specifications, I have personally come across misconfigured web servers emitting such URLs as redirects. This is how I discovered this issue in http2. Change-Id: If47aa61b8d256d76a3451090076e6eb5ff596c9e GitHub-Last-Rev: cb0470115705139cfc60a3d27ec432363fd54a1c GitHub-Pull-Request: golang/net#170 Reviewed-on: https://go-review.googlesource.com/c/net/+/487915 Run-TryBot: Damien Neil Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- http2/transport.go | 5 ++++- http2/transport_test.go | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/http2/transport.go b/http2/transport.go index b9632380e7..b20c749171 100644 --- a/http2/transport.go +++ b/http2/transport.go @@ -518,11 +518,14 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { func authorityAddr(scheme string, authority string) (addr string) { host, port, err := net.SplitHostPort(authority) if err != nil { // authority didn't have a port + host = authority + port = "" + } + if port == "" { // authority's port was empty port = "443" if scheme == "http" { port = "80" } - host = authority } if a, err := idna.ToASCII(host); err == nil { host = a diff --git a/http2/transport_test.go b/http2/transport_test.go index d3156208cf..99848485b9 100644 --- a/http2/transport_test.go +++ b/http2/transport_test.go @@ -4456,11 +4456,14 @@ func TestAuthorityAddr(t *testing.T) { }{ {"http", "foo.com", "foo.com:80"}, {"https", "foo.com", "foo.com:443"}, + {"https", "foo.com:", "foo.com:443"}, {"https", "foo.com:1234", "foo.com:1234"}, {"https", "1.2.3.4:1234", "1.2.3.4:1234"}, {"https", "1.2.3.4", "1.2.3.4:443"}, + {"https", "1.2.3.4:", "1.2.3.4:443"}, {"https", "[::1]:1234", "[::1]:1234"}, {"https", "[::1]", "[::1]:443"}, + {"https", "[::1]:", "[::1]:443"}, } for _, tt := range tests { got := authorityAddr(tt.scheme, tt.authority) From 81261084d015c2928e689ffda4e7c246f562fbf9 Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Wed, 5 Jul 2023 08:45:45 +0000 Subject: [PATCH 02/46] dns/dnsmessage: update Parser docs The current API returns ErrSectionDone, not (nil,nil). Change-Id: I95c721c6c198e7302b9154bc39617b502e3d62f9 GitHub-Last-Rev: c66bcff3b11bac48439712a2a6867857d26fb865 GitHub-Pull-Request: golang/net#181 Reviewed-on: https://go-review.googlesource.com/c/net/+/507955 Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- dns/dnsmessage/message.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index 1577d4a19d..37da3de4d3 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -527,12 +527,14 @@ func (r *Resource) pack(msg []byte, compression map[string]int, compressionOff i // When parsing is started, the Header is parsed. Next, each Question can be // either parsed or skipped. Alternatively, all Questions can be skipped at // once. When all Questions have been parsed, attempting to parse Questions -// will return (nil, nil) and attempting to skip Questions will return -// (true, nil). After all Questions have been either parsed or skipped, all +// will return the [ErrSectionDone] error. +// After all Questions have been either parsed or skipped, all // Answers, Authorities and Additionals can be either parsed or skipped in the // same way, and each type of Resource must be fully parsed or skipped before // proceeding to the next type of Resource. // +// Parser is safe to copy to preserve the parsing state. +// // Note that there is no requirement to fully skip or parse the message. type Parser struct { msg []byte From d8f9c0143e94e55c0e871e302e81cf982732df30 Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Fri, 2 Jun 2023 11:59:55 +0000 Subject: [PATCH 03/46] dns/dnsmessage: add fuzz test After CL 443215 pack(unpack(msg)) should never fail, so we can add a fuzz test to prove that. Change-Id: Ia2abfc30e2b2a492b4dd5de6ca6f29d2324bd737 GitHub-Last-Rev: 1d9812a34c3295730951535bd79917f5bb2c187e GitHub-Pull-Request: golang/net#177 Reviewed-on: https://go-review.googlesource.com/c/net/+/500296 Auto-Submit: Ian Lance Taylor Reviewed-by: Joedian Reid TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Run-TryBot: Mateusz Poliwczak --- dns/dnsmessage/message_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index ce2716e42d..64c6db86d1 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1643,3 +1643,31 @@ func TestNoFmt(t *testing.T) { } } } + +func FuzzUnpackPack(f *testing.F) { + for _, msg := range []Message{smallTestMsg(), largeTestMsg()} { + bytes, _ := msg.Pack() + f.Add(bytes) + } + + f.Fuzz(func(t *testing.T, msg []byte) { + var m Message + if err := m.Unpack(msg); err != nil { + return + } + + msgPacked, err := m.Pack() + if err != nil { + t.Fatalf("failed to pack message that was succesfully unpacked: %v", err) + } + + var m2 Message + if err := m2.Unpack(msgPacked); err != nil { + t.Fatalf("failed to unpack message that was succesfully packed: %v", err) + } + + if !reflect.DeepEqual(m, m2) { + t.Fatal("unpack(msg) is not deep equal to unpack(pack(unpack(msg)))") + } + }) +} From 9475ce144dec10136752baa1aa72dae6b96b4ece Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 23 Jun 2023 10:42:34 -0700 Subject: [PATCH 04/46] quic: fix typos in comments For golang/go#58547 Change-Id: I79f06d22fc010bf2e339df47abed3df170d18339 Reviewed-on: https://go-review.googlesource.com/c/net/+/506075 Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- internal/quic/errors.go | 2 +- internal/quic/sent_val.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/quic/errors.go b/internal/quic/errors.go index a9ebbe4b74..55d32f3106 100644 --- a/internal/quic/errors.go +++ b/internal/quic/errors.go @@ -10,7 +10,7 @@ import ( "fmt" ) -// A transportError is an transport error code from RFC 9000 Section 20.1. +// A transportError is a transport error code from RFC 9000 Section 20.1. // // The transportError type doesn't implement the error interface to ensure we always // distinguish between errors sent to and received from the peer. diff --git a/internal/quic/sent_val.go b/internal/quic/sent_val.go index b33d8b00f2..31f69e47d0 100644 --- a/internal/quic/sent_val.go +++ b/internal/quic/sent_val.go @@ -37,7 +37,7 @@ func (s sentVal) isSet() bool { return s != 0 } // shouldSend reports whether the value is set and has not been sent to the peer. func (s sentVal) shouldSend() bool { return s.state() == sentValUnsent } -// shouldSend reports whether the the value needs to be sent to the peer. +// shouldSend reports whether the value needs to be sent to the peer. // The value needs to be sent if it is set and has not been sent. // If pto is true, indicating that we are sending a PTO probe, the value // should also be sent if it is set and has not been acknowledged. From 304cc91b19ae873219f3d0807c8533267629cf2e Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 14 Oct 2022 10:19:03 -0700 Subject: [PATCH 05/46] quic: tracking of received packets and acks to send RFC 9000, Section 13.2. For golang/go#58547 Change-Id: I0aad4c03fabb9087964dc9030bb8777d5159360c Reviewed-on: https://go-review.googlesource.com/c/net/+/506595 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/acks.go | 184 ++++++++++++++++++++++++ internal/quic/acks_test.go | 248 +++++++++++++++++++++++++++++++++ internal/quic/rangeset.go | 5 + internal/quic/rangeset_test.go | 20 +++ 4 files changed, 457 insertions(+) create mode 100644 internal/quic/acks.go create mode 100644 internal/quic/acks_test.go diff --git a/internal/quic/acks.go b/internal/quic/acks.go new file mode 100644 index 0000000000..ba860efb2b --- /dev/null +++ b/internal/quic/acks.go @@ -0,0 +1,184 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "time" +) + +// ackState tracks packets received from a peer within a number space. +// It handles packet deduplication (don't process the same packet twice) and +// determines the timing and content of ACK frames. +type ackState struct { + seen rangeset[packetNumber] + + // The time at which we must send an ACK frame, even if we have no other data to send. + nextAck time.Time + + // The time we received the largest-numbered packet in seen. + maxRecvTime time.Time + + // The largest-numbered ack-eliciting packet in seen. + maxAckEliciting packetNumber + + // The number of ack-eliciting packets in seen that we have not yet acknowledged. + unackedAckEliciting int +} + +// shouldProcess reports whether a packet should be handled or discarded. +func (acks *ackState) shouldProcess(num packetNumber) bool { + if packetNumber(acks.seen.min()) > num { + // We've discarded the state for this range of packet numbers. + // Discard the packet rather than potentially processing a duplicate. + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.3-5 + return false + } + if acks.seen.contains(num) { + // Discard duplicate packets. + return false + } + return true +} + +// receive records receipt of a packet. +func (acks *ackState) receive(now time.Time, space numberSpace, num packetNumber, ackEliciting bool) { + if ackEliciting { + acks.unackedAckEliciting++ + if acks.mustAckImmediately(space, num) { + acks.nextAck = now + } else if acks.nextAck.IsZero() { + // This packet does not need to be acknowledged immediately, + // but the ack must not be intentionally delayed by more than + // the max_ack_delay transport parameter we sent to the peer. + // + // We always delay acks by the maximum allowed, less the timer + // granularity. ("[max_ack_delay] SHOULD include the receiver's + // expected delays in alarms firing.") + // + // https://www.rfc-editor.org/rfc/rfc9000#section-18.2-4.28.1 + acks.nextAck = now.Add(maxAckDelay - timerGranularity) + } + if num > acks.maxAckEliciting { + acks.maxAckEliciting = num + } + } + + acks.seen.add(num, num+1) + if num == acks.seen.max() { + acks.maxRecvTime = now + } + + // Limit the total number of ACK ranges by dropping older ranges. + // + // Remembering more ranges results in larger ACK frames. + // + // Remembering a large number of ranges could result in ACK frames becoming + // too large to fit in a packet, in which case we will silently drop older + // ranges during packet construction. + // + // Remembering fewer ranges can result in unnecessary retransmissions, + // since we cannot accept packets older than the oldest remembered range. + // + // The limit here is completely arbitrary. If it seems wrong, it probably is. + // + // https://www.rfc-editor.org/rfc/rfc9000#section-13.2.3 + const maxAckRanges = 8 + if overflow := acks.seen.numRanges() - maxAckRanges; overflow > 0 { + acks.seen.removeranges(0, overflow) + } +} + +// mustAckImmediately reports whether an ack-eliciting packet must be acknowledged immediately, +// or whether the ack may be deferred. +func (acks *ackState) mustAckImmediately(space numberSpace, num packetNumber) bool { + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1 + if space != appDataSpace { + // "[...] all ack-eliciting Initial and Handshake packets [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-2 + return true + } + if num < acks.maxAckEliciting { + // "[...] when the received packet has a packet number less than another + // ack-eliciting packet that has been received [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-8.1 + return true + } + if acks.seen.rangeContaining(acks.maxAckEliciting).end != num { + // "[...] when the packet has a packet number larger than the highest-numbered + // ack-eliciting packet that has been received and there are missing packets + // between that packet and this packet." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-8.2 + // + // This case is a bit tricky. Let's say we've received: + // 0, ack-eliciting + // 1, ack-eliciting + // 3, NOT ack eliciting + // + // We have sent ACKs for 0 and 1. If we receive ack-eliciting packet 2, + // we do not need to send an immediate ACK, because there are no missing + // packets between it and the highest-numbered ack-eliciting packet (1). + // If we receive ack-eliciting packet 4, we do need to send an immediate ACK, + // because there's a gap (the missing packet 2). + // + // We check for this by looking up the ACK range which contains the + // highest-numbered ack-eliciting packet: [0, 1) in the above example. + // If the range ends just before the packet we are now processing, + // there are no gaps. If it does not, there must be a gap. + return true + } + if acks.unackedAckEliciting >= 2 { + // "[...] after receiving at least two ack-eliciting packets." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.2 + return true + } + return false +} + +// shouldSendAck reports whether the connection should send an ACK frame at this time, +// in an ACK-only packet if necessary. +func (acks *ackState) shouldSendAck(now time.Time) bool { + return !acks.nextAck.IsZero() && !acks.nextAck.After(now) +} + +// acksToSend returns the set of packet numbers to ACK at this time, and the current ack delay. +// It may return acks even if shouldSendAck returns false, when there are unacked +// ack-eliciting packets whose ack is being delayed. +func (acks *ackState) acksToSend(now time.Time) (nums rangeset[packetNumber], ackDelay time.Duration) { + if acks.nextAck.IsZero() && acks.unackedAckEliciting == 0 { + return nil, 0 + } + // "[...] the delays intentionally introduced between the time the packet with the + // largest packet number is received and the time an acknowledgement is sent." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.2.5-1 + delay := now.Sub(acks.maxRecvTime) + if delay < 0 { + delay = 0 + } + return acks.seen, delay +} + +// sentAck records that an ACK frame has been sent. +func (acks *ackState) sentAck() { + acks.nextAck = time.Time{} + acks.unackedAckEliciting = 0 +} + +// handleAck records that an ack has been received for a ACK frame we sent +// containing the given Largest Acknowledged field. +func (acks *ackState) handleAck(largestAcked packetNumber) { + // We can stop acking packets less or equal to largestAcked. + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.4-1 + // + // We rely on acks.seen containing the largest packet number that has been successfully + // processed, so we retain the range containing largestAcked and discard previous ones. + acks.seen.sub(0, acks.seen.rangeContaining(largestAcked).start) +} + +// largestSeen reports the largest seen packet. +func (acks *ackState) largestSeen() packetNumber { + return acks.seen.max() +} diff --git a/internal/quic/acks_test.go b/internal/quic/acks_test.go new file mode 100644 index 0000000000..4f1032910f --- /dev/null +++ b/internal/quic/acks_test.go @@ -0,0 +1,248 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "testing" + "time" +) + +func TestAcksDisallowDuplicate(t *testing.T) { + // Don't process a packet that we've seen before. + acks := ackState{} + now := time.Now() + receive := []packetNumber{0, 1, 2, 4, 7, 6, 9} + seen := map[packetNumber]bool{} + for i, pnum := range receive { + acks.receive(now, appDataSpace, pnum, true) + seen[pnum] = true + for ppnum := packetNumber(0); ppnum < 11; ppnum++ { + if got, want := acks.shouldProcess(ppnum), !seen[ppnum]; got != want { + t.Fatalf("after receiving %v: acks.shouldProcess(%v) = %v, want %v", receive[:i+1], ppnum, got, want) + } + } + } +} + +func TestAcksDisallowDiscardedAckRanges(t *testing.T) { + // Don't process a packet with a number in a discarded range. + acks := ackState{} + now := time.Now() + for pnum := packetNumber(0); ; pnum += 2 { + acks.receive(now, appDataSpace, pnum, true) + send, _ := acks.acksToSend(now) + for ppnum := packetNumber(0); ppnum < packetNumber(send.min()); ppnum++ { + if acks.shouldProcess(ppnum) { + t.Fatalf("after limiting ack ranges to %v: acks.shouldProcess(%v) (in discarded range) = true, want false", send, ppnum) + } + } + if send.min() > 10 { + break + } + } +} + +func TestAcksSent(t *testing.T) { + type packet struct { + pnum packetNumber + ackEliciting bool + } + for _, test := range []struct { + name string + space numberSpace + + // ackedPackets and packets are packets that we receive. + // After receiving all packets in ackedPackets, we send an ack. + // Then we receive the subsequent packets in packets. + ackedPackets []packet + packets []packet + + wantDelay time.Duration + wantAcks rangeset[packetNumber] + }{{ + name: "no packets to ack", + space: initialSpace, + }, { + name: "non-ack-eliciting packets are not acked", + space: initialSpace, + packets: []packet{{ + pnum: 0, + ackEliciting: false, + }}, + }, { + name: "ack-eliciting Initial packets are acked immediately", + space: initialSpace, + packets: []packet{{ + pnum: 0, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{0, 1}}, + wantDelay: 0, + }, { + name: "ack-eliciting Handshake packets are acked immediately", + space: handshakeSpace, + packets: []packet{{ + pnum: 0, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{0, 1}}, + wantDelay: 0, + }, { + name: "ack-eliciting AppData packets are acked after max_ack_delay", + space: appDataSpace, + packets: []packet{{ + pnum: 0, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{0, 1}}, + wantDelay: maxAckDelay - timerGranularity, + }, { + name: "reordered ack-eliciting packets are acked immediately", + space: appDataSpace, + ackedPackets: []packet{{ + pnum: 1, + ackEliciting: true, + }}, + packets: []packet{{ + pnum: 0, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{0, 2}}, + wantDelay: 0, + }, { + name: "gaps in ack-eliciting packets are acked immediately", + space: appDataSpace, + packets: []packet{{ + pnum: 1, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{1, 2}}, + wantDelay: 0, + }, { + name: "reordered non-ack-eliciting packets are not acked immediately", + space: appDataSpace, + ackedPackets: []packet{{ + pnum: 1, + ackEliciting: true, + }}, + packets: []packet{{ + pnum: 2, + ackEliciting: true, + }, { + pnum: 0, + ackEliciting: false, + }, { + pnum: 4, + ackEliciting: false, + }}, + wantAcks: rangeset[packetNumber]{{0, 3}, {4, 5}}, + wantDelay: maxAckDelay - timerGranularity, + }, { + name: "immediate ack after two ack-eliciting packets are received", + space: appDataSpace, + packets: []packet{{ + pnum: 0, + ackEliciting: true, + }, { + pnum: 1, + ackEliciting: true, + }}, + wantAcks: rangeset[packetNumber]{{0, 2}}, + wantDelay: 0, + }} { + t.Run(test.name, func(t *testing.T) { + acks := ackState{} + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + for _, p := range test.ackedPackets { + t.Logf("receive %v.%v, ack-eliciting=%v", test.space, p.pnum, p.ackEliciting) + acks.receive(start, test.space, p.pnum, p.ackEliciting) + } + t.Logf("send an ACK frame") + acks.sentAck() + for _, p := range test.packets { + t.Logf("receive %v.%v, ack-eliciting=%v", test.space, p.pnum, p.ackEliciting) + acks.receive(start, test.space, p.pnum, p.ackEliciting) + } + switch { + case len(test.wantAcks) == 0: + // No ACK should be sent, even well after max_ack_delay. + if acks.shouldSendAck(start.Add(10 * maxAckDelay)) { + t.Errorf("acks.shouldSendAck(T+10*max_ack_delay) = true, want false") + } + case test.wantDelay > 0: + // No ACK should be sent before a delay. + if acks.shouldSendAck(start.Add(test.wantDelay - 1)) { + t.Errorf("acks.shouldSendAck(T+%v-1ns) = true, want false", test.wantDelay) + } + fallthrough + default: + // ACK should be sent after a delay. + if !acks.shouldSendAck(start.Add(test.wantDelay)) { + t.Errorf("acks.shouldSendAck(T+%v) = false, want true", test.wantDelay) + } + } + // acksToSend always reports the available packets that can be acked, + // and the amount of time that has passed since the most recent acked + // packet was received. + for _, delay := range []time.Duration{ + 0, + test.wantDelay, + test.wantDelay + 1, + } { + gotNums, gotDelay := acks.acksToSend(start.Add(delay)) + wantDelay := delay + if len(gotNums) == 0 { + wantDelay = 0 + } + if !slicesEqual(gotNums, test.wantAcks) || gotDelay != wantDelay { + t.Errorf("acks.acksToSend(T+%v) = %v, %v; want %v, %v", delay, gotNums, gotDelay, test.wantAcks, wantDelay) + } + } + }) + } +} + +// slicesEqual reports whether two slices are equal. +// Replace this with slices.Equal once the module go.mod is go1.17 or newer. +func slicesEqual[E comparable](s1, s2 []E) bool { + if len(s1) != len(s2) { + return false + } + for i := range s1 { + if s1[i] != s2[i] { + return false + } + } + return true +} + +func TestAcksDiscardAfterAck(t *testing.T) { + acks := ackState{} + now := time.Now() + acks.receive(now, appDataSpace, 0, true) + acks.receive(now, appDataSpace, 2, true) + acks.receive(now, appDataSpace, 4, true) + acks.receive(now, appDataSpace, 5, true) + acks.receive(now, appDataSpace, 6, true) + acks.handleAck(6) // discards all ranges prior to the one containing packet 6 + acks.receive(now, appDataSpace, 7, true) + got, _ := acks.acksToSend(now) + if len(got) != 1 { + t.Errorf("acks.acksToSend contains ranges prior to last acknowledged ack; got %v, want 1 range", got) + } +} + +func TestAcksLargestSeen(t *testing.T) { + acks := ackState{} + now := time.Now() + acks.receive(now, appDataSpace, 0, true) + acks.receive(now, appDataSpace, 4, true) + acks.receive(now, appDataSpace, 1, true) + if got, want := acks.largestSeen(), packetNumber(4); got != want { + t.Errorf("acks.largestSeen() = %v, want %v", got, want) + } +} diff --git a/internal/quic/rangeset.go b/internal/quic/rangeset.go index 5339c5ac51..4966a99d2c 100644 --- a/internal/quic/rangeset.go +++ b/internal/quic/rangeset.go @@ -154,6 +154,11 @@ func (s rangeset[T]) end() T { return s[len(s)-1].end } +// numRanges returns the number of ranges in the rangeset. +func (s rangeset[T]) numRanges() int { + return len(s) +} + // isrange reports if the rangeset covers exactly the range [start, end). func (s rangeset[T]) isrange(start, end T) bool { switch len(s) { diff --git a/internal/quic/rangeset_test.go b/internal/quic/rangeset_test.go index 308046905a..2027f14b88 100644 --- a/internal/quic/rangeset_test.go +++ b/internal/quic/rangeset_test.go @@ -295,3 +295,23 @@ func TestRangesetIsRange(t *testing.T) { } } } + +func TestRangesetNumRanges(t *testing.T) { + for _, test := range []struct { + s rangeset[int64] + want int + }{{ + s: rangeset[int64]{}, + want: 0, + }, { + s: rangeset[int64]{{0, 100}}, + want: 1, + }, { + s: rangeset[int64]{{0, 100}, {200, 300}}, + want: 2, + }} { + if got, want := test.s.numRanges(), test.want; got != want { + t.Errorf("%+v.numRanges() = %v, want %v", test.s, got, want) + } + } +} From 57553cbff16307d5178b250ad301e7b466f9d969 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 13 Oct 2022 12:09:20 -0700 Subject: [PATCH 06/46] quic: connection ids Each side of a QUIC connection chooses the connection IDs used by its peer. In our case, we use 8-byte random IDs. A connection has a list of connection IDs that it may receive packets on, and a list that it may send packets to. Add a minimal data structure for tracking these lists, and handling of the connection IDs tracked across Initial and Handshake packets. This does not yet handle post-handshake connection ID changes made in NEW_CONNECTION_ID and RETIRE_CONNECTION_ID frames. RFC 9000, Section 5.1. For golang/go#58547 Change-Id: I3e059393cacafbcea04a1b4131c0c7dc28acad5e Reviewed-on: https://go-review.googlesource.com/c/net/+/506675 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/conn_id.go | 147 ++++++++++++++++++++++++++++++++++ internal/quic/conn_id_test.go | 109 +++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 internal/quic/conn_id.go create mode 100644 internal/quic/conn_id_test.go diff --git a/internal/quic/conn_id.go b/internal/quic/conn_id.go new file mode 100644 index 0000000000..deea70d326 --- /dev/null +++ b/internal/quic/conn_id.go @@ -0,0 +1,147 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/rand" +) + +// connIDState is a conn's connection IDs. +type connIDState struct { + // The destination connection IDs of packets we receive are local. + // The destination connection IDs of packets we send are remote. + // + // Local IDs are usually issued by us, and remote IDs by the peer. + // The exception is the transient destination connection ID sent in + // a client's Initial packets, which is chosen by the client. + local []connID + remote []connID +} + +// A connID is a connection ID and associated metadata. +type connID struct { + // cid is the connection ID itself. + cid []byte + + // seq is the connection ID's sequence number: + // https://www.rfc-editor.org/rfc/rfc9000.html#section-5.1.1-1 + // + // For the transient destination ID in a client's Initial packet, this is -1. + seq int64 +} + +func (s *connIDState) initClient(newID newConnIDFunc) error { + // Client chooses its initial connection ID, and sends it + // in the Source Connection ID field of the first Initial packet. + locid, err := newID() + if err != nil { + return err + } + s.local = append(s.local, connID{ + seq: 0, + cid: locid, + }) + + // Client chooses an initial, transient connection ID for the server, + // and sends it in the Destination Connection ID field of the first Initial packet. + remid, err := newID() + if err != nil { + return err + } + s.remote = append(s.remote, connID{ + seq: -1, + cid: remid, + }) + return nil +} + +func (s *connIDState) initServer(newID newConnIDFunc, dstConnID []byte) error { + // Client-chosen, transient connection ID received in the first Initial packet. + // The server will not use this as the Source Connection ID of packets it sends, + // but remembers it because it may receive packets sent to this destination. + s.local = append(s.local, connID{ + seq: -1, + cid: cloneBytes(dstConnID), + }) + + // Server chooses a connection ID, and sends it in the Source Connection ID of + // the response to the clent. + locid, err := newID() + if err != nil { + return err + } + s.local = append(s.local, connID{ + seq: 0, + cid: locid, + }) + return nil +} + +// srcConnID is the Source Connection ID to use in a sent packet. +func (s *connIDState) srcConnID() []byte { + if s.local[0].seq == -1 && len(s.local) > 1 { + // Don't use the transient connection ID if another is available. + return s.local[1].cid + } + return s.local[0].cid +} + +// dstConnID is the Destination Connection ID to use in a sent packet. +func (s *connIDState) dstConnID() []byte { + return s.remote[0].cid +} + +// handlePacket updates the connection ID state during the handshake +// (Initial and Handshake packets). +func (s *connIDState) handlePacket(side connSide, ptype packetType, srcConnID []byte) { + switch { + case ptype == packetTypeInitial && side == clientSide: + if len(s.remote) == 1 && s.remote[0].seq == -1 { + // We're a client connection processing the first Initial packet + // from the server. Replace the transient remote connection ID + // with the Source Connection ID from the packet. + s.remote[0] = connID{ + seq: 0, + cid: cloneBytes(srcConnID), + } + } + case ptype == packetTypeInitial && side == serverSide: + if len(s.remote) == 0 { + // We're a server connection processing the first Initial packet + // from the client. Set the client's connection ID. + s.remote = append(s.remote, connID{ + seq: 0, + cid: cloneBytes(srcConnID), + }) + } + case ptype == packetTypeHandshake && side == serverSide: + if len(s.local) > 0 && s.local[0].seq == -1 { + // We're a server connection processing the first Handshake packet from + // the client. Discard the transient, client-chosen connection ID used + // for Initial packets; the client will never send it again. + s.local = append(s.local[:0], s.local[1:]...) + } + } +} + +func cloneBytes(b []byte) []byte { + n := make([]byte, len(b)) + copy(n, b) + return n +} + +type newConnIDFunc func() ([]byte, error) + +func newRandomConnID() ([]byte, error) { + // It is not necessary for connection IDs to be cryptographically secure, + // but it doesn't hurt. + id := make([]byte, connIDLen) + if _, err := rand.Read(id); err != nil { + return nil, err + } + return id, nil +} diff --git a/internal/quic/conn_id_test.go b/internal/quic/conn_id_test.go new file mode 100644 index 0000000000..7c31e9d560 --- /dev/null +++ b/internal/quic/conn_id_test.go @@ -0,0 +1,109 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "fmt" + "reflect" + "testing" +) + +func TestConnIDClientHandshake(t *testing.T) { + // On initialization, the client chooses local and remote IDs. + // + // The order in which we allocate the two isn't actually important, + // but test is a lot simpler if we assume. + var s connIDState + s.initClient(newConnIDSequence()) + if got, want := string(s.srcConnID()), "local-1"; got != want { + t.Errorf("after initClient: srcConnID = %q, want %q", got, want) + } + if got, want := string(s.dstConnID()), "local-2"; got != want { + t.Errorf("after initClient: dstConnID = %q, want %q", got, want) + } + + // The server's first Initial packet provides the client with a + // non-transient remote connection ID. + s.handlePacket(clientSide, packetTypeInitial, []byte("remote-1")) + if got, want := string(s.dstConnID()), "remote-1"; got != want { + t.Errorf("after receiving Initial: dstConnID = %q, want %q", got, want) + } + + wantLocal := []connID{{ + cid: []byte("local-1"), + seq: 0, + }} + if !reflect.DeepEqual(s.local, wantLocal) { + t.Errorf("local ids: %v, want %v", s.local, wantLocal) + } + wantRemote := []connID{{ + cid: []byte("remote-1"), + seq: 0, + }} + if !reflect.DeepEqual(s.remote, wantRemote) { + t.Errorf("remote ids: %v, want %v", s.remote, wantRemote) + } +} + +func TestConnIDServerHandshake(t *testing.T) { + // On initialization, the server is provided with the client-chosen + // transient connection ID, and allocates an ID of its own. + // The Initial packet sets the remote connection ID. + var s connIDState + s.initServer(newConnIDSequence(), []byte("transient")) + s.handlePacket(serverSide, packetTypeInitial, []byte("remote-1")) + if got, want := string(s.srcConnID()), "local-1"; got != want { + t.Errorf("after initClient: srcConnID = %q, want %q", got, want) + } + if got, want := string(s.dstConnID()), "remote-1"; got != want { + t.Errorf("after initClient: dstConnID = %q, want %q", got, want) + } + + wantLocal := []connID{{ + cid: []byte("transient"), + seq: -1, + }, { + cid: []byte("local-1"), + seq: 0, + }} + if !reflect.DeepEqual(s.local, wantLocal) { + t.Errorf("local ids: %v, want %v", s.local, wantLocal) + } + wantRemote := []connID{{ + cid: []byte("remote-1"), + seq: 0, + }} + if !reflect.DeepEqual(s.remote, wantRemote) { + t.Errorf("remote ids: %v, want %v", s.remote, wantRemote) + } + + // The client's first Handshake packet permits the server to discard the + // transient connection ID. + s.handlePacket(serverSide, packetTypeHandshake, []byte("remote-1")) + wantLocal = []connID{{ + cid: []byte("local-1"), + seq: 0, + }} + if !reflect.DeepEqual(s.local, wantLocal) { + t.Errorf("after handshake local ids: %v, want %v", s.local, wantLocal) + } +} + +func newConnIDSequence() newConnIDFunc { + var n uint64 + return func() ([]byte, error) { + n++ + return []byte(fmt.Sprintf("local-%v", n)), nil + } +} + +func TestNewRandomConnID(t *testing.T) { + cid, err := newRandomConnID() + if len(cid) != connIDLen || err != nil { + t.Fatalf("newConnID() = %x, %v; want %v bytes", cid, connIDLen, err) + } +} From 4a3f925950ab4f8466e4582f84f3a4a8444f0271 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 27 Jun 2023 15:17:02 -0700 Subject: [PATCH 07/46] quic: basic connection event loop Add the Conn type, representing a QUIC connection. A Conn's behavior is driven by an event loop goroutine. This goroutine owns most Conn state. External events (datagrams received, user operations such as writing to streams) send events to the loop goroutine on a message channel. The testConn type, used in tests, wraps a Conn and takes control of its event loop. The testConn permits tests to interact with a Conn synchronously, sending it events, observing the result, and controlling the Conn's view of time passing. Add a very minimal implementation of connection idle timeouts (RFC 9000, Section 10.1) to test the implementation of synthetic time. For golang/go#58547 Change-Id: Ic517e5e7bb019f4a677f892a807ca0417d6e19b1 Reviewed-on: https://go-review.googlesource.com/c/net/+/506678 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam --- internal/quic/conn.go | 155 ++++++++++++++++++++++++++++++ internal/quic/conn_test.go | 188 +++++++++++++++++++++++++++++++++++++ internal/quic/quic.go | 2 + 3 files changed, 345 insertions(+) create mode 100644 internal/quic/conn.go create mode 100644 internal/quic/conn_test.go diff --git a/internal/quic/conn.go b/internal/quic/conn.go new file mode 100644 index 0000000000..d6dbac1a93 --- /dev/null +++ b/internal/quic/conn.go @@ -0,0 +1,155 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "errors" + "fmt" + "time" +) + +// A Conn is a QUIC connection. +// +// Multiple goroutines may invoke methods on a Conn simultaneously. +type Conn struct { + msgc chan any + donec chan struct{} // closed when conn loop exits + exited bool // set to make the conn loop exit immediately + + testHooks connTestHooks + + // idleTimeout is the time at which the connection will be closed due to inactivity. + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1 + maxIdleTimeout time.Duration + idleTimeout time.Time +} + +// connTestHooks override conn behavior in tests. +type connTestHooks interface { + nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any) +} + +func newConn(now time.Time, hooks connTestHooks) (*Conn, error) { + c := &Conn{ + donec: make(chan struct{}), + testHooks: hooks, + maxIdleTimeout: defaultMaxIdleTimeout, + idleTimeout: now.Add(defaultMaxIdleTimeout), + } + + // A one-element buffer allows us to wake a Conn's event loop as a + // non-blocking operation. + c.msgc = make(chan any, 1) + + go c.loop(now) + return c, nil +} + +type timerEvent struct{} + +// loop is the connection main loop. +// +// Except where otherwise noted, all connection state is owned by the loop goroutine. +// +// The loop processes messages from c.msgc and timer events. +// Other goroutines may examine or modify conn state by sending the loop funcs to execute. +func (c *Conn) loop(now time.Time) { + defer close(c.donec) + + // The connection timer sends a message to the connection loop on expiry. + // We need to give it an expiry when creating it, so set the initial timeout to + // an arbitrary large value. The timer will be reset before this expires (and it + // isn't a problem if it does anyway). Skip creating the timer in tests which + // take control of the connection message loop. + var timer *time.Timer + var lastTimeout time.Time + hooks := c.testHooks + if hooks == nil { + timer = time.AfterFunc(1*time.Hour, func() { + c.sendMsg(timerEvent{}) + }) + defer timer.Stop() + } + + for !c.exited { + nextTimeout := c.idleTimeout + + var m any + if hooks != nil { + // Tests only: Wait for the test to tell us to continue. + now, m = hooks.nextMessage(c.msgc, nextTimeout) + } else if !nextTimeout.IsZero() && nextTimeout.Before(now) { + // A connection timer has expired. + now = time.Now() + m = timerEvent{} + } else { + // Reschedule the connection timer if necessary + // and wait for the next event. + if !nextTimeout.Equal(lastTimeout) && !nextTimeout.IsZero() { + // Resetting a timer created with time.AfterFunc guarantees + // that the timer will run again. We might generate a spurious + // timer event under some circumstances, but that's okay. + timer.Reset(nextTimeout.Sub(now)) + lastTimeout = nextTimeout + } + m = <-c.msgc + now = time.Now() + } + switch m := m.(type) { + case timerEvent: + // A connection timer has expired. + if !now.Before(c.idleTimeout) { + // "[...] the connection is silently closed and + // its state is discarded [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-1 + c.exited = true + return + } + case func(time.Time, *Conn): + // Send a func to msgc to run it on the main Conn goroutine + m(now, c) + default: + panic(fmt.Sprintf("quic: unrecognized conn message %T", m)) + } + } +} + +// sendMsg sends a message to the conn's loop. +// It does not wait for the message to be processed. +func (c *Conn) sendMsg(m any) error { + select { + case c.msgc <- m: + case <-c.donec: + return errors.New("quic: connection closed") + } + return nil +} + +// runOnLoop executes a function within the conn's loop goroutine. +func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { + donec := make(chan struct{}) + if err := c.sendMsg(func(now time.Time, c *Conn) { + defer close(donec) + f(now, c) + }); err != nil { + return err + } + select { + case <-donec: + case <-c.donec: + return errors.New("quic: connection closed") + } + return nil +} + +// exit fully terminates a connection immediately. +func (c *Conn) exit() { + c.runOnLoop(func(now time.Time, c *Conn) { + c.exited = true + }) + <-c.donec +} diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go new file mode 100644 index 0000000000..a1709958e1 --- /dev/null +++ b/internal/quic/conn_test.go @@ -0,0 +1,188 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "math" + "testing" + "time" +) + +func TestConnTestConn(t *testing.T) { + tc := newTestConn(t, serverSide) + if got, want := tc.timeUntilEvent(), defaultMaxIdleTimeout; got != want { + t.Errorf("new conn timeout=%v, want %v (max_idle_timeout)", got, want) + } + + var ranAt time.Time + tc.conn.runOnLoop(func(now time.Time, c *Conn) { + ranAt = now + }) + if !ranAt.Equal(tc.now) { + t.Errorf("func ran on loop at %v, want %v", ranAt, tc.now) + } + tc.wait() + + nextTime := tc.now.Add(defaultMaxIdleTimeout / 2) + tc.advanceTo(nextTime) + tc.conn.runOnLoop(func(now time.Time, c *Conn) { + ranAt = now + }) + if !ranAt.Equal(nextTime) { + t.Errorf("func ran on loop at %v, want %v", ranAt, nextTime) + } + tc.wait() + + tc.advanceToTimer() + if err := tc.conn.sendMsg(nil); err == nil { + t.Errorf("after advancing to idle timeout, sendMsg = nil, want error") + } + if !tc.conn.exited { + t.Errorf("after advancing to idle timeout, exited = false, want true") + } +} + +// A testConn is a Conn whose external interactions (sending and receiving packets, +// setting timers) can be manipulated in tests. +type testConn struct { + t *testing.T + conn *Conn + now time.Time + timer time.Time + timerLastFired time.Time + idlec chan struct{} // only accessed on the conn's loop +} + +// newTestConn creates a Conn for testing. +// +// The Conn's event loop is controlled by the test, +// allowing test code to access Conn state directly +// by first ensuring the loop goroutine is idle. +func newTestConn(t *testing.T, side connSide) *testConn { + t.Helper() + tc := &testConn{ + t: t, + now: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + } + t.Cleanup(tc.cleanup) + + conn, err := newConn(tc.now, (*testConnHooks)(tc)) + if err != nil { + tc.t.Fatal(err) + } + tc.conn = conn + + tc.wait() + return tc +} + +// advance causes time to pass. +func (tc *testConn) advance(d time.Duration) { + tc.t.Helper() + tc.advanceTo(tc.now.Add(d)) +} + +// advanceTo sets the current time. +func (tc *testConn) advanceTo(now time.Time) { + tc.t.Helper() + if tc.now.After(now) { + tc.t.Fatalf("time moved backwards: %v -> %v", tc.now, now) + } + tc.now = now + if tc.timer.After(tc.now) { + return + } + tc.conn.sendMsg(timerEvent{}) + tc.wait() +} + +// advanceToTimer sets the current time to the time of the Conn's next timer event. +func (tc *testConn) advanceToTimer() { + if tc.timer.IsZero() { + tc.t.Fatalf("advancing to timer, but timer is not set") + } + tc.advanceTo(tc.timer) +} + +const infiniteDuration = time.Duration(math.MaxInt64) + +// timeUntilEvent returns the amount of time until the next connection event. +func (tc *testConn) timeUntilEvent() time.Duration { + if tc.timer.IsZero() { + return infiniteDuration + } + if tc.timer.Before(tc.now) { + return 0 + } + return tc.timer.Sub(tc.now) +} + +// wait blocks until the conn becomes idle. +// The conn is idle when it is blocked waiting for a packet to arrive or a timer to expire. +// Tests shouldn't need to call wait directly. +// testConn methods that wake the Conn event loop will call wait for them. +func (tc *testConn) wait() { + tc.t.Helper() + idlec := make(chan struct{}) + fail := false + tc.conn.sendMsg(func(now time.Time, c *Conn) { + if tc.idlec != nil { + tc.t.Errorf("testConn.wait called concurrently") + fail = true + close(idlec) + } else { + // nextMessage will close idlec. + tc.idlec = idlec + } + }) + select { + case <-idlec: + case <-tc.conn.donec: + } + if fail { + panic(fail) + } +} + +func (tc *testConn) cleanup() { + if tc.conn == nil { + return + } + tc.conn.exit() +} + +// testConnHooks implements connTestHooks. +type testConnHooks testConn + +// nextMessage is called by the Conn's event loop to request its next event. +func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.Time, m any) { + tc.timer = timer + if !timer.IsZero() && !timer.After(tc.now) { + if timer.Equal(tc.timerLastFired) { + // If the connection timer fires at time T, the Conn should take some + // action to advance the timer into the future. If the Conn reschedules + // the timer for the same time, it isn't making progress and we have a bug. + tc.t.Errorf("connection timer spinning; now=%v timer=%v", tc.now, timer) + } else { + tc.timerLastFired = timer + return tc.now, timerEvent{} + } + } + select { + case m := <-msgc: + return tc.now, m + default: + } + // If the message queue is empty, then the conn is idle. + if tc.idlec != nil { + idlec := tc.idlec + tc.idlec = nil + close(idlec) + } + m = <-msgc + return tc.now, m +} diff --git a/internal/quic/quic.go b/internal/quic/quic.go index 982c6751b7..c69c0b9840 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -19,6 +19,8 @@ const connIDLen = 8 // Local values of various transport parameters. // https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2 const ( + defaultMaxIdleTimeout = 30 * time.Second // max_idle_timeout + // The max_udp_payload_size transport parameter is the size of our // network receive buffer. // From 16cc77a3d1797230d5e8bfd5a27fb0979d24faaf Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 31 Jan 2023 09:00:25 -0800 Subject: [PATCH 08/46] quic: print better stacks on SIGQUIT When handling an uncaught SIGQUIT (C-\), the runtime prints stacks with GOTRACEBACK=all. This is more detail than we need or want when debugging a hung test by killing it with C-\. Add a signal handler in tests to print stacks with GOTRACEBACK=all instead. For golang/go#58547 Change-Id: I8b381cec41a645568aa2eb675ca7f936f35e145a Reviewed-on: https://go-review.googlesource.com/c/net/+/509016 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam --- internal/quic/gotraceback_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 internal/quic/gotraceback_test.go diff --git a/internal/quic/gotraceback_test.go b/internal/quic/gotraceback_test.go new file mode 100644 index 0000000000..c22702faa4 --- /dev/null +++ b/internal/quic/gotraceback_test.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 && unix + +package quic + +import ( + "os" + "os/signal" + "runtime/debug" + "syscall" +) + +// When killed with SIGQUIT (C-\), print stacks with GOTRACEBACK=all rather than system, +// to reduce irrelevant noise when debugging hung tests. +func init() { + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGQUIT) + go func() { + <-ch + debug.SetTraceback("all") + panic("SIGQUIT") + }() +} From 0adcadfb6b87451705307d07906d4cfdc7677584 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 13 Jun 2023 14:02:18 -0700 Subject: [PATCH 09/46] quic: send and receive datagrams Add the ability for Conns to send and receive datagrams. No socket handling yet; this only functions in tests for now. Extend testConn to permit tests to send packets to Conns and observe the packets Conns send. There's a circular dependency here: We can't test Handshake and 1-RTT packets until we have the handshake implemented, but we can't implement the handshake without the ability to send and receive Handshake and 1-RTT packets. This CL adds the ability to send and receive those packets; tests for those paths will follow with the handshake implementation. For golang/go#58547 Change-Id: I4e7f88f5f039baf7e01f68a53639022866786af9 Reviewed-on: https://go-review.googlesource.com/c/net/+/509017 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/conn.go | 90 ++++++- internal/quic/conn_loss.go | 46 ++++ internal/quic/conn_recv.go | 264 +++++++++++++++++++ internal/quic/conn_send.go | 255 +++++++++++++++++++ internal/quic/conn_test.go | 394 ++++++++++++++++++++++++++++- internal/quic/dgram.go | 38 +++ internal/quic/frame_debug.go | 2 +- internal/quic/packet_parser.go | 6 +- internal/quic/packet_protection.go | 4 +- internal/quic/packet_test.go | 17 ++ internal/quic/packet_writer.go | 5 +- internal/quic/ping.go | 16 ++ internal/quic/ping_test.go | 35 +++ internal/quic/quic.go | 4 + internal/quic/sent_packet.go | 2 + internal/quic/tls.go | 23 ++ 16 files changed, 1184 insertions(+), 17 deletions(-) create mode 100644 internal/quic/conn_loss.go create mode 100644 internal/quic/conn_recv.go create mode 100644 internal/quic/conn_send.go create mode 100644 internal/quic/dgram.go create mode 100644 internal/quic/ping.go create mode 100644 internal/quic/ping_test.go create mode 100644 internal/quic/tls.go diff --git a/internal/quic/conn.go b/internal/quic/conn.go index d6dbac1a93..cdf79d607c 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -9,6 +9,7 @@ package quic import ( "errors" "fmt" + "net/netip" "time" ) @@ -16,16 +17,37 @@ import ( // // Multiple goroutines may invoke methods on a Conn simultaneously. type Conn struct { + side connSide + listener connListener + testHooks connTestHooks + peerAddr netip.AddrPort + msgc chan any donec chan struct{} // closed when conn loop exits exited bool // set to make the conn loop exit immediately - testHooks connTestHooks + w packetWriter + acks [numberSpaceCount]ackState // indexed by number space + connIDState connIDState + tlsState tlsState + loss lossState // idleTimeout is the time at which the connection will be closed due to inactivity. // https://www.rfc-editor.org/rfc/rfc9000#section-10.1 maxIdleTimeout time.Duration idleTimeout time.Time + + peerAckDelayExponent int8 // -1 when unknown + + // Tests only: Send a PING in a specific number space. + testSendPingSpace numberSpace + testSendPing sentVal +} + +// The connListener is the Conn's Listener. +// Defined as an interface so we can swap it out in tests. +type connListener interface { + sendDatagram(p []byte, addr netip.AddrPort) error } // connTestHooks override conn behavior in tests. @@ -33,18 +55,41 @@ type connTestHooks interface { nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any) } -func newConn(now time.Time, hooks connTestHooks) (*Conn, error) { +func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, l connListener, hooks connTestHooks) (*Conn, error) { c := &Conn{ - donec: make(chan struct{}), - testHooks: hooks, - maxIdleTimeout: defaultMaxIdleTimeout, - idleTimeout: now.Add(defaultMaxIdleTimeout), + side: side, + listener: l, + peerAddr: peerAddr, + msgc: make(chan any, 1), + donec: make(chan struct{}), + testHooks: hooks, + maxIdleTimeout: defaultMaxIdleTimeout, + idleTimeout: now.Add(defaultMaxIdleTimeout), + peerAckDelayExponent: -1, } // A one-element buffer allows us to wake a Conn's event loop as a // non-blocking operation. c.msgc = make(chan any, 1) + if c.side == clientSide { + if err := c.connIDState.initClient(newRandomConnID); err != nil { + return nil, err + } + initialConnID = c.connIDState.dstConnID() + } else { + if err := c.connIDState.initServer(newRandomConnID, initialConnID); err != nil { + return nil, err + } + } + + // The smallest allowed maximum QUIC datagram size is 1200 bytes. + // TODO: PMTU discovery. + const maxDatagramSize = 1200 + c.loss.init(c.side, maxDatagramSize, now) + + c.tlsState.init(c.side, initialConnID) + go c.loop(now) return c, nil } @@ -76,7 +121,14 @@ func (c *Conn) loop(now time.Time) { } for !c.exited { - nextTimeout := c.idleTimeout + sendTimeout := c.maybeSend(now) // try sending + + // Note that we only need to consider the ack timer for the App Data space, + // since the Initial and Handshake spaces always ack immediately. + nextTimeout := sendTimeout + nextTimeout = firstTime(nextTimeout, c.idleTimeout) + nextTimeout = firstTime(nextTimeout, c.loss.timer) + nextTimeout = firstTime(nextTimeout, c.acks[appDataSpace].nextAck) var m any if hooks != nil { @@ -100,6 +152,9 @@ func (c *Conn) loop(now time.Time) { now = time.Now() } switch m := m.(type) { + case *datagram: + c.handleDatagram(now, m) + m.recycle() case timerEvent: // A connection timer has expired. if !now.Before(c.idleTimeout) { @@ -109,6 +164,7 @@ func (c *Conn) loop(now time.Time) { c.exited = true return } + c.loss.advance(now, c.handleAckOrLoss) case func(time.Time, *Conn): // Send a func to msgc to run it on the main Conn goroutine m(now, c) @@ -146,6 +202,12 @@ func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { return nil } +// abort terminates a connection with an error. +func (c *Conn) abort(now time.Time, err error) { + // TODO: Send CONNECTION_CLOSE frames. + c.exit() +} + // exit fully terminates a connection immediately. func (c *Conn) exit() { c.runOnLoop(func(now time.Time, c *Conn) { @@ -153,3 +215,17 @@ func (c *Conn) exit() { }) <-c.donec } + +// firstTime returns the earliest non-zero time, or zero if both times are zero. +func firstTime(a, b time.Time) time.Time { + switch { + case a.IsZero(): + return b + case b.IsZero(): + return a + case a.Before(b): + return a + default: + return b + } +} diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go new file mode 100644 index 0000000000..11ed42dbb9 --- /dev/null +++ b/internal/quic/conn_loss.go @@ -0,0 +1,46 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "fmt" + +// handleAckOrLoss deals with the final fate of a packet we sent: +// Either the peer acknowledges it, or we declare it lost. +// +// In order to handle packet loss, we must retain any information sent to the peer +// until the peer has acknowledged it. +// +// When information is acknowledged, we can discard it. +// +// When information is lost, we mark it for retransmission. +// See RFC 9000, Section 13.3 for a complete list of information which is retransmitted on loss. +// https://www.rfc-editor.org/rfc/rfc9000#section-13.3 +func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetFate) { + // The list of frames in a sent packet is marshaled into a buffer in the sentPacket + // by the packetWriter. Unmarshal that buffer here. This code must be kept in sync with + // packetWriter.append*. + // + // A sent packet meets its fate (acked or lost) only once, so it's okay to consume + // the sentPacket's buffer here. + for !sent.done() { + switch f := sent.next(); f { + default: + panic(fmt.Sprintf("BUG: unhandled lost frame type %x", f)) + case frameTypeAck: + // Unlike most information, loss of an ACK frame does not trigger + // retransmission. ACKs are sent in response to ack-eliciting packets, + // and always contain the latest information available. + // + // Acknowledgement of an ACK frame may allow us to discard information + // about older packets. + largest := packetNumber(sent.nextInt()) + if fate == packetAcked { + c.acks[space].handleAck(largest) + } + } + } +} diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go new file mode 100644 index 0000000000..d5a3b8cb0c --- /dev/null +++ b/internal/quic/conn_recv.go @@ -0,0 +1,264 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "time" +) + +func (c *Conn) handleDatagram(now time.Time, dgram *datagram) { + buf := dgram.b + c.loss.datagramReceived(now, len(buf)) + for len(buf) > 0 { + var n int + ptype := getPacketType(buf) + switch ptype { + case packetTypeInitial: + if c.side == serverSide && len(dgram.b) < minimumClientInitialDatagramSize { + // Discard client-sent Initial packets in too-short datagrams. + // https://www.rfc-editor.org/rfc/rfc9000#section-14.1-4 + return + } + n = c.handleLongHeader(now, ptype, initialSpace, buf) + case packetTypeHandshake: + n = c.handleLongHeader(now, ptype, handshakeSpace, buf) + case packetType1RTT: + n = c.handle1RTT(now, buf) + default: + return + } + if n <= 0 { + // Invalid data at the end of a datagram is ignored. + break + } + c.idleTimeout = now.Add(c.maxIdleTimeout) + buf = buf[n:] + } +} + +func (c *Conn) handleLongHeader(now time.Time, ptype packetType, space numberSpace, buf []byte) int { + if !c.tlsState.rkeys[space].isSet() { + return skipLongHeaderPacket(buf) + } + + pnumMax := c.acks[space].largestSeen() + p, n := parseLongHeaderPacket(buf, c.tlsState.rkeys[space], pnumMax) + if n < 0 { + return -1 + } + if p.reservedBits != 0 { + // https://www.rfc-editor.org/rfc/rfc9000#section-17.2-8.2.1 + c.abort(now, localTransportError(errProtocolViolation)) + return -1 + } + + if !c.acks[space].shouldProcess(p.num) { + return n + } + + c.connIDState.handlePacket(c.side, p.ptype, p.srcConnID) + ackEliciting := c.handleFrames(now, ptype, space, p.payload) + c.acks[space].receive(now, space, p.num, ackEliciting) + if p.ptype == packetTypeHandshake && c.side == serverSide { + c.loss.validateClientAddress() + + // TODO: Discard Initial keys. + // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.1-2 + } + return n +} + +func (c *Conn) handle1RTT(now time.Time, buf []byte) int { + if !c.tlsState.rkeys[appDataSpace].isSet() { + // 1-RTT packets extend to the end of the datagram, + // so skip the remainder of the datagram if we can't parse this. + return len(buf) + } + + pnumMax := c.acks[appDataSpace].largestSeen() + p, n := parse1RTTPacket(buf, c.tlsState.rkeys[appDataSpace], connIDLen, pnumMax) + if n < 0 { + return -1 + } + if p.reservedBits != 0 { + // https://www.rfc-editor.org/rfc/rfc9000#section-17.3.1-4.8.1 + c.abort(now, localTransportError(errProtocolViolation)) + return -1 + } + + if !c.acks[appDataSpace].shouldProcess(p.num) { + return len(buf) + } + + ackEliciting := c.handleFrames(now, packetType1RTT, appDataSpace, p.payload) + c.acks[appDataSpace].receive(now, appDataSpace, p.num, ackEliciting) + return len(buf) +} + +func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, payload []byte) (ackEliciting bool) { + if len(payload) == 0 { + // "An endpoint MUST treat receipt of a packet containing no frames + // as a connection error of type PROTOCOL_VIOLATION." + // https://www.rfc-editor.org/rfc/rfc9000#section-12.4-3 + c.abort(now, localTransportError(errProtocolViolation)) + return false + } + // frameOK verifies that ptype is one of the packets in mask. + frameOK := func(c *Conn, ptype, mask packetType) (ok bool) { + if ptype&mask == 0 { + // "An endpoint MUST treat receipt of a frame in a packet type + // that is not permitted as a connection error of type + // PROTOCOL_VIOLATION." + // https://www.rfc-editor.org/rfc/rfc9000#section-12.4-3 + c.abort(now, localTransportError(errProtocolViolation)) + return false + } + return true + } + // Packet masks from RFC 9000 Table 3. + // https://www.rfc-editor.org/rfc/rfc9000#table-3 + const ( + IH_1 = packetTypeInitial | packetTypeHandshake | packetType1RTT + __01 = packetType0RTT | packetType1RTT + ___1 = packetType1RTT + ) + for len(payload) > 0 { + switch payload[0] { + case frameTypePadding, frameTypeAck, frameTypeAckECN, + frameTypeConnectionCloseTransport, frameTypeConnectionCloseApplication: + default: + ackEliciting = true + } + n := -1 + switch payload[0] { + case frameTypePadding: + // PADDING is OK in all spaces. + n = 1 + case frameTypePing: + // PING is OK in all spaces. + // + // A PING frame causes us to respond with an ACK by virtue of being + // an ack-eliciting frame, but requires no other action. + n = 1 + case frameTypeAck, frameTypeAckECN: + if !frameOK(c, ptype, IH_1) { + return + } + n = c.handleAckFrame(now, space, payload) + case frameTypeResetStream: + if !frameOK(c, ptype, __01) { + return + } + _, _, _, n = consumeResetStreamFrame(payload) + case frameTypeStopSending: + if !frameOK(c, ptype, __01) { + return + } + _, _, n = consumeStopSendingFrame(payload) + case frameTypeCrypto: + if !frameOK(c, ptype, IH_1) { + return + } + _, _, n = consumeCryptoFrame(payload) + case frameTypeNewToken: + if !frameOK(c, ptype, ___1) { + return + } + _, n = consumeNewTokenFrame(payload) + case 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f: // STREAM + if !frameOK(c, ptype, __01) { + return + } + _, _, _, _, n = consumeStreamFrame(payload) + case frameTypeMaxData: + if !frameOK(c, ptype, __01) { + return + } + _, n = consumeMaxDataFrame(payload) + case frameTypeMaxStreamData: + if !frameOK(c, ptype, __01) { + return + } + _, _, n = consumeMaxStreamDataFrame(payload) + case frameTypeMaxStreamsBidi, frameTypeMaxStreamsUni: + if !frameOK(c, ptype, __01) { + return + } + _, _, n = consumeMaxStreamsFrame(payload) + case frameTypeStreamsBlockedBidi, frameTypeStreamsBlockedUni: + if !frameOK(c, ptype, __01) { + return + } + _, _, n = consumeStreamsBlockedFrame(payload) + case frameTypeStreamDataBlocked: + if !frameOK(c, ptype, __01) { + return + } + _, _, n = consumeStreamDataBlockedFrame(payload) + case frameTypeNewConnectionID: + if !frameOK(c, ptype, __01) { + return + } + _, _, _, _, n = consumeNewConnectionIDFrame(payload) + case frameTypeConnectionCloseTransport: + // CONNECTION_CLOSE is OK in all spaces. + _, _, _, n = consumeConnectionCloseTransportFrame(payload) + case frameTypeConnectionCloseApplication: + // CONNECTION_CLOSE is OK in all spaces. + _, _, n = consumeConnectionCloseApplicationFrame(payload) + case frameTypeHandshakeDone: + if !frameOK(c, ptype, ___1) { + return + } + n = 1 + } + if n < 0 { + c.abort(now, localTransportError(errFrameEncoding)) + return false + } + payload = payload[n:] + } + return ackEliciting +} + +func (c *Conn) handleAckFrame(now time.Time, space numberSpace, payload []byte) int { + c.loss.receiveAckStart() + _, ackDelay, n := consumeAckFrame(payload, func(rangeIndex int, start, end packetNumber) { + if end > c.loss.nextNumber(space) { + // Acknowledgement of a packet we never sent. + c.abort(now, localTransportError(errProtocolViolation)) + return + } + c.loss.receiveAckRange(now, space, rangeIndex, start, end, c.handleAckOrLoss) + }) + // Prior to receiving the peer's transport parameters, we cannot + // interpret the ACK Delay field because we don't know the ack_delay_exponent + // to apply. + // + // For servers, we should always know the ack_delay_exponent because the + // client's transport parameters are carried in its Initial packets and we + // won't send an ack-eliciting Initial packet until after receiving the last + // client Initial packet. + // + // For clients, we won't receive the server's transport parameters until handling + // its Handshake flight, which will probably happen after reading its ACK for our + // Initial packet(s). However, the peer's acknowledgement delay cannot reduce our + // adjusted RTT sample below min_rtt, and min_rtt is generally going to be set + // by the packet containing the ACK for our Initial flight. Therefore, the + // ACK Delay for an ACK in the Initial space is likely to be ignored anyway. + // + // Long story short, setting the delay to 0 prior to reading transport parameters + // is usually going to have no effect, will have only a minor effect in the rare + // cases when it happens, and there aren't any good alternatives anyway since we + // can't interpret the ACK Delay field without knowing the exponent. + var delay time.Duration + if c.peerAckDelayExponent >= 0 { + delay = ackDelay.Duration(uint8(c.peerAckDelayExponent)) + } + c.loss.receiveAckEnd(now, space, delay, c.handleAckOrLoss) + return n +} diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go new file mode 100644 index 0000000000..3a51ceb285 --- /dev/null +++ b/internal/quic/conn_send.go @@ -0,0 +1,255 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "time" +) + +// maybeSend sends datagrams, if possible. +// +// If sending is blocked by pacing, it returns the next time +// a datagram may be sent. +func (c *Conn) maybeSend(now time.Time) (next time.Time) { + // Assumption: The congestion window is not underutilized. + // If congestion control, pacing, and anti-amplification all permit sending, + // but we have no packet to send, then we will declare the window underutilized. + c.loss.cc.setUnderutilized(false) + + // Send one datagram on each iteration of this loop, + // until we hit a limit or run out of data to send. + // + // For each number space where we have write keys, + // attempt to construct a packet in that space. + // If the packet contains no frames (we have no data in need of sending), + // abandon the packet. + // + // Speculatively constructing packets means we don't need + // separate code paths for "do we have data to send?" and + // "send the data" that need to be kept in sync. + for { + limit, next := c.loss.sendLimit(now) + if limit == ccBlocked { + // If anti-amplification blocks sending, then no packet can be sent. + return next + } + // We may still send ACKs, even if congestion control or pacing limit sending. + + // Prepare to write a datagram of at most maxSendSize bytes. + c.w.reset(c.loss.maxSendSize()) + + // Initial packet. + pad := false + var sentInitial *sentPacket + if k := c.tlsState.wkeys[initialSpace]; k.isSet() { + pnumMaxAcked := c.acks[initialSpace].largestSeen() + pnum := c.loss.nextNumber(initialSpace) + p := longPacket{ + ptype: packetTypeInitial, + version: 1, + num: pnum, + dstConnID: c.connIDState.dstConnID(), + srcConnID: c.connIDState.srcConnID(), + } + c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) + c.appendFrames(now, initialSpace, pnum, limit) + sentInitial = c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, p) + if sentInitial != nil { + // Client initial packets need to be sent in a datagram padded to + // at least 1200 bytes. We can't add the padding yet, however, + // since we may want to coalesce additional packets with this one. + if c.side == clientSide || sentInitial.ackEliciting { + pad = true + } + } + } + + // Handshake packet. + if k := c.tlsState.wkeys[handshakeSpace]; k.isSet() { + pnumMaxAcked := c.acks[handshakeSpace].largestSeen() + pnum := c.loss.nextNumber(handshakeSpace) + p := longPacket{ + ptype: packetTypeHandshake, + version: 1, + num: pnum, + dstConnID: c.connIDState.dstConnID(), + srcConnID: c.connIDState.srcConnID(), + } + c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) + c.appendFrames(now, handshakeSpace, pnum, limit) + if sent := c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, p); sent != nil { + c.loss.packetSent(now, handshakeSpace, sent) + if c.side == clientSide { + // TODO: Discard the Initial keys. + // https://www.rfc-editor.org/rfc/rfc9001.html#section-4.9.1 + } + } + } + + // 1-RTT packet. + if k := c.tlsState.wkeys[appDataSpace]; k.isSet() { + pnumMaxAcked := c.acks[appDataSpace].largestSeen() + pnum := c.loss.nextNumber(appDataSpace) + dstConnID := c.connIDState.dstConnID() + c.w.start1RTTPacket(pnum, pnumMaxAcked, dstConnID) + c.appendFrames(now, appDataSpace, pnum, limit) + if pad && len(c.w.payload()) > 0 { + // 1-RTT packets have no length field and extend to the end + // of the datagram, so if we're sending a datagram that needs + // padding we need to add it inside the 1-RTT packet. + c.w.appendPaddingTo(minimumClientInitialDatagramSize) + pad = false + } + if sent := c.w.finish1RTTPacket(pnum, pnumMaxAcked, dstConnID, k); sent != nil { + c.loss.packetSent(now, appDataSpace, sent) + } + } + + buf := c.w.datagram() + if len(buf) == 0 { + if limit == ccOK { + // We have nothing to send, and congestion control does not + // block sending. The congestion window is underutilized. + c.loss.cc.setUnderutilized(true) + } + return next + } + + if sentInitial != nil { + if pad { + // Pad out the datagram with zeros, coalescing the Initial + // packet with invalid packets that will be ignored by the peer. + // https://www.rfc-editor.org/rfc/rfc9000.html#section-14.1-1 + for len(buf) < minimumClientInitialDatagramSize { + buf = append(buf, 0) + // Technically this padding isn't in any packet, but + // account it to the Initial packet in this datagram + // for purposes of flow control and loss recovery. + sentInitial.size++ + sentInitial.inFlight = true + } + } + if k := c.tlsState.wkeys[initialSpace]; k.isSet() { + c.loss.packetSent(now, initialSpace, sentInitial) + } + } + + c.listener.sendDatagram(buf, c.peerAddr) + } +} + +func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, limit ccLimit) { + shouldSendAck := c.acks[space].shouldSendAck(now) + if limit != ccOK { + // ACKs are not limited by congestion control. + if shouldSendAck && c.appendAckFrame(now, space) { + c.acks[space].sentAck() + } + return + } + // We want to send an ACK frame if the ack controller wants to send a frame now, + // OR if we are sending a packet anyway and have ack-eliciting packets which we + // have not yet acked. + // + // We speculatively add ACK frames here, to put them at the front of the packet + // to avoid truncation. + // + // After adding all frames, if we don't need to send an ACK frame and have not + // added any other frames, we abandon the packet. + if c.appendAckFrame(now, space) { + defer func() { + // All frames other than ACK and PADDING are ack-eliciting, + // so if the packet is ack-eliciting we've added additional + // frames to it. + if shouldSendAck || c.w.sent.ackEliciting { + // Either we are willing to send an ACK-only packet, + // or we've added additional frames. + c.acks[space].sentAck() + } else { + // There's nothing in this packet but ACK frames, and + // we don't want to send an ACK-only packet at this time. + // Abandoning the packet means we wrote an ACK frame for + // nothing, but constructing the frame is cheap. + c.w.abandonPacket() + } + }() + } + if limit != ccOK { + return + } + pto := c.loss.ptoExpired + + // TODO: Add all the other frames we can send. + + // Test-only PING frames. + if space == c.testSendPingSpace && c.testSendPing.shouldSendPTO(pto) { + if !c.w.appendPingFrame() { + return + } + c.testSendPing.setSent(pnum) + } + + // If this is a PTO probe and we haven't added an ack-eliciting frame yet, + // add a PING to make this an ack-eliciting probe. + // + // Technically, there are separate PTO timers for each number space. + // When a PTO timer expires, we MUST send an ack-eliciting packet in the + // timer's space. We SHOULD send ack-eliciting packets in every other space + // with in-flight data. (RFC 9002, section 6.2.4) + // + // What we actually do is send a single datagram containing an ack-eliciting packet + // for every space for which we have keys. + // + // We fill the PTO probe packets with new or unacknowledged data. For example, + // a PTO probe sent for the Initial space will generally retransmit previously + // sent but unacknowledged CRYPTO data. + // + // When sending a PTO probe datagram containing multiple packets, it is + // possible that an earlier packet will fill up the datagram, leaving no + // space for the remaining probe packet(s). This is not a problem in practice. + // + // A client discards Initial keys when it first sends a Handshake packet + // (RFC 9001 Section 4.9.1). Handshake keys are discarded when the handshake + // is confirmed (RFC 9001 Section 4.9.2). The PTO timer is not set for the + // Application Data packet number space until the handshake is confirmed + // (RFC 9002 Section 6.2.1). Therefore, the only times a PTO probe can fire + // while data for multiple spaces is in flight are: + // + // - a server's Initial or Handshake timers can fire while Initial and Handshake + // data is in flight; and + // + // - a client's Handshake timer can fire while Handshake and Application Data + // data is in flight. + // + // It is theoretically possible for a server's Initial CRYPTO data to overflow + // the maximum datagram size, but unlikely in practice; this space contains + // only the ServerHello TLS message, which is small. It's also unlikely that + // the Handshake PTO probe will fire while Initial data is in flight (this + // requires not just that the Initial CRYPTO data completely fill a datagram, + // but a quite specific arrangement of lost and retransmitted packets.) + // We don't bother worrying about this case here, since the worst case is + // that we send a PTO probe for the in-flight Initial data and drop the + // Handshake probe. + // + // If a client's Handshake PTO timer fires while Application Data data is in + // flight, it is possible that the resent Handshake CRYPTO data will crowd + // out the probe for the Application Data space. However, since this probe is + // optional (recall that the Application Data PTO timer is never set until + // after Handshake keys have been discarded), dropping it is acceptable. + if pto && !c.w.sent.ackEliciting { + c.w.appendPingFrame() + } +} + +func (c *Conn) appendAckFrame(now time.Time, space numberSpace) bool { + seen, delay := c.acks[space].acksToSend(now) + if len(seen) == 0 { + return false + } + d := unscaledAckDelayFromDuration(delay, ackDelayExponent) + return c.w.appendAckFrame(seen, d) +} diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index a1709958e1..6bb12e210e 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -7,7 +7,12 @@ package quic import ( + "errors" + "fmt" "math" + "net/netip" + "reflect" + "strings" "testing" "time" ) @@ -46,6 +51,52 @@ func TestConnTestConn(t *testing.T) { } } +type testDatagram struct { + packets []*testPacket + paddedSize int +} + +func (d testDatagram) String() string { + var b strings.Builder + fmt.Fprintf(&b, "datagram with %v packets", len(d.packets)) + if d.paddedSize > 0 { + fmt.Fprintf(&b, " (padded to %v bytes)", d.paddedSize) + } + b.WriteString(":") + for _, p := range d.packets { + b.WriteString("\n") + b.WriteString(p.String()) + } + return b.String() +} + +type testPacket struct { + ptype packetType + version uint32 + num packetNumber + dstConnID []byte + srcConnID []byte + frames []debugFrame +} + +func (p testPacket) String() string { + var b strings.Builder + fmt.Fprintf(&b, " %v %v", p.ptype, p.num) + if p.version != 0 { + fmt.Fprintf(&b, " version=%v", p.version) + } + if p.srcConnID != nil { + fmt.Fprintf(&b, " src={%x}", p.srcConnID) + } + if p.dstConnID != nil { + fmt.Fprintf(&b, " dst={%x}", p.dstConnID) + } + for _, f := range p.frames { + fmt.Fprintf(&b, "\n %v", f) + } + return b.String() +} + // A testConn is a Conn whose external interactions (sending and receiving packets, // setting timers) can be manipulated in tests. type testConn struct { @@ -55,6 +106,30 @@ type testConn struct { timer time.Time timerLastFired time.Time idlec chan struct{} // only accessed on the conn's loop + + // Read and write keys are distinct from the conn's keys, + // because the test may know about keys before the conn does. + // For example, when sending a datagram with coalesced + // Initial and Handshake packets to a client conn, + // we use Handshake keys to encrypt the packet. + // The client only acquires those keys when it processes + // the Initial packet. + rkeys [numberSpaceCount]keys // for packets sent to the conn + wkeys [numberSpaceCount]keys // for packets sent by the conn + + // Information about the conn's (fake) peer. + peerConnID []byte // source conn id of peer's packets + peerNextPacketNum [numberSpaceCount]packetNumber // next packet number to use + + // Datagrams, packets, and frames sent by the conn, + // but not yet processed by the test. + sentDatagrams [][]byte + sentPackets []*testPacket + sentFrames []debugFrame + sentFramePacketType packetType + + // Frame types to ignore in tests. + ignoreFrames map[byte]bool } // newTestConn creates a Conn for testing. @@ -65,17 +140,41 @@ type testConn struct { func newTestConn(t *testing.T, side connSide) *testConn { t.Helper() tc := &testConn{ - t: t, - now: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + t: t, + now: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + peerConnID: []byte{0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5}, + ignoreFrames: map[byte]bool{ + frameTypePadding: true, // ignore PADDING by default + }, } t.Cleanup(tc.cleanup) - conn, err := newConn(tc.now, (*testConnHooks)(tc)) + var initialConnID []byte + if side == serverSide { + // The initial connection ID for the server is chosen by the client. + // When creating a server-side connection, pick a random connection ID here. + var err error + initialConnID, err = newRandomConnID() + if err != nil { + tc.t.Fatal(err) + } + } + + conn, err := newConn( + tc.now, + side, + initialConnID, + netip.MustParseAddrPort("127.0.0.1:443"), + (*testConnListener)(tc), + (*testConnHooks)(tc)) if err != nil { tc.t.Fatal(err) } tc.conn = conn + tc.wkeys[initialSpace] = conn.tlsState.wkeys[initialSpace] + tc.rkeys[initialSpace] = conn.tlsState.rkeys[initialSpace] + tc.wait() return tc } @@ -108,6 +207,16 @@ func (tc *testConn) advanceToTimer() { tc.advanceTo(tc.timer) } +func (tc *testConn) timerDelay() time.Duration { + if tc.timer.IsZero() { + return math.MaxInt64 // infinite + } + if tc.timer.Before(tc.now) { + return 0 + } + return tc.timer.Sub(tc.now) +} + const infiniteDuration = time.Duration(math.MaxInt64) // timeUntilEvent returns the amount of time until the next connection event. @@ -155,6 +264,277 @@ func (tc *testConn) cleanup() { tc.conn.exit() } +// write sends the Conn a datagram. +func (tc *testConn) write(d *testDatagram) { + tc.t.Helper() + var buf []byte + for _, p := range d.packets { + space := spaceForPacketType(p.ptype) + if p.num >= tc.peerNextPacketNum[space] { + tc.peerNextPacketNum[space] = p.num + 1 + } + buf = append(buf, tc.encodeTestPacket(p)...) + } + for len(buf) < d.paddedSize { + buf = append(buf, 0) + } + tc.conn.sendMsg(&datagram{ + b: buf, + }) + tc.wait() +} + +// writeFrame sends the Conn a datagram containing the given frames. +func (tc *testConn) writeFrames(ptype packetType, frames ...debugFrame) { + tc.t.Helper() + space := spaceForPacketType(ptype) + dstConnID := tc.conn.connIDState.local[0].cid + if tc.conn.connIDState.local[0].seq == -1 && ptype != packetTypeInitial { + // Only use the transient connection ID in Initial packets. + dstConnID = tc.conn.connIDState.local[1].cid + } + d := &testDatagram{ + packets: []*testPacket{{ + ptype: ptype, + num: tc.peerNextPacketNum[space], + frames: frames, + version: 1, + dstConnID: dstConnID, + srcConnID: tc.peerConnID, + }}, + } + if ptype == packetTypeInitial && tc.conn.side == serverSide { + d.paddedSize = 1200 + } + tc.write(d) +} + +// ignoreFrame hides frames of the given type sent by the Conn. +func (tc *testConn) ignoreFrame(frameType byte) { + tc.ignoreFrames[frameType] = true +} + +// readDatagram reads the next datagram sent by the Conn. +// It returns nil if the Conn has no more datagrams to send at this time. +func (tc *testConn) readDatagram() *testDatagram { + tc.t.Helper() + tc.wait() + tc.sentPackets = nil + tc.sentFrames = nil + if len(tc.sentDatagrams) == 0 { + return nil + } + buf := tc.sentDatagrams[0] + tc.sentDatagrams = tc.sentDatagrams[1:] + return tc.parseTestDatagram(buf) +} + +// readPacket reads the next packet sent by the Conn. +// It returns nil if the Conn has no more packets to send at this time. +func (tc *testConn) readPacket() *testPacket { + tc.t.Helper() + for len(tc.sentPackets) == 0 { + d := tc.readDatagram() + if d == nil { + return nil + } + tc.sentPackets = d.packets + } + p := tc.sentPackets[0] + tc.sentPackets = tc.sentPackets[1:] + return p +} + +// readFrame reads the next frame sent by the Conn. +// It returns nil if the Conn has no more frames to send at this time. +func (tc *testConn) readFrame() (debugFrame, packetType) { + tc.t.Helper() + for len(tc.sentFrames) == 0 { + p := tc.readPacket() + if p == nil { + return nil, packetTypeInvalid + } + tc.sentFramePacketType = p.ptype + tc.sentFrames = p.frames + } + f := tc.sentFrames[0] + tc.sentFrames = tc.sentFrames[1:] + return f, tc.sentFramePacketType +} + +// wantDatagram indicates that we expect the Conn to send a datagram. +func (tc *testConn) wantDatagram(expectation string, want *testDatagram) { + tc.t.Helper() + got := tc.readDatagram() + if !reflect.DeepEqual(got, want) { + tc.t.Fatalf("%v:\ngot datagram: %v\nwant datagram: %v", expectation, got, want) + } +} + +// wantPacket indicates that we expect the Conn to send a packet. +func (tc *testConn) wantPacket(expectation string, want *testPacket) { + tc.t.Helper() + got := tc.readPacket() + if !reflect.DeepEqual(got, want) { + tc.t.Fatalf("%v:\ngot packet: %v\nwant packet: %v", expectation, got, want) + } +} + +// wantFrame indicates that we expect the Conn to send a frame. +func (tc *testConn) wantFrame(expectation string, wantType packetType, want debugFrame) { + tc.t.Helper() + got, gotType := tc.readFrame() + if got == nil { + tc.t.Fatalf("%v:\nconnection is idle\nwant %v frame: %v", expectation, wantType, want) + } + if gotType != wantType { + tc.t.Fatalf("%v:\ngot %v packet, want %v", expectation, wantType, want) + } + if !reflect.DeepEqual(got, want) { + tc.t.Fatalf("%v:\ngot frame: %v\nwant frame: %v", expectation, got, want) + } +} + +// wantIdle indicates that we expect the Conn to not send any more frames. +func (tc *testConn) wantIdle(expectation string) { + tc.t.Helper() + switch { + case len(tc.sentFrames) > 0: + tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, tc.sentFrames[0]) + case len(tc.sentPackets) > 0: + tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, tc.sentPackets[0]) + } + if f, _ := tc.readFrame(); f != nil { + tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, f) + } +} + +func (tc *testConn) encodeTestPacket(p *testPacket) []byte { + tc.t.Helper() + var w packetWriter + w.reset(1200) + var pnumMaxAcked packetNumber + if p.ptype != packetType1RTT { + w.startProtectedLongHeaderPacket(pnumMaxAcked, longPacket{ + ptype: p.ptype, + version: p.version, + num: p.num, + dstConnID: p.dstConnID, + srcConnID: p.srcConnID, + }) + } else { + w.start1RTTPacket(p.num, pnumMaxAcked, p.dstConnID) + } + for _, f := range p.frames { + f.write(&w) + } + space := spaceForPacketType(p.ptype) + if !tc.rkeys[space].isSet() { + tc.t.Fatalf("sending packet with no %v keys available", space) + return nil + } + if p.ptype != packetType1RTT { + w.finishProtectedLongHeaderPacket(pnumMaxAcked, tc.rkeys[space], longPacket{ + ptype: p.ptype, + version: p.version, + num: p.num, + dstConnID: p.dstConnID, + srcConnID: p.srcConnID, + }) + } else { + w.finish1RTTPacket(p.num, pnumMaxAcked, p.dstConnID, tc.rkeys[space]) + } + return w.datagram() +} + +func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { + tc.t.Helper() + bufSize := len(buf) + d := &testDatagram{} + for len(buf) > 0 { + if buf[0] == 0 { + d.paddedSize = bufSize + break + } + ptype := getPacketType(buf) + space := spaceForPacketType(ptype) + if !tc.wkeys[space].isSet() { + tc.t.Fatalf("no keys for space %v, packet type %v", space, ptype) + } + if isLongHeader(buf[0]) { + var pnumMax packetNumber // TODO: Track packet numbers. + p, n := parseLongHeaderPacket(buf, tc.wkeys[space], pnumMax) + if n < 0 { + tc.t.Fatalf("packet parse error") + } + frames, err := tc.parseTestFrames(p.payload) + if err != nil { + tc.t.Fatal(err) + } + d.packets = append(d.packets, &testPacket{ + ptype: p.ptype, + version: p.version, + num: p.num, + dstConnID: p.dstConnID, + srcConnID: p.srcConnID, + frames: frames, + }) + buf = buf[n:] + } else { + var pnumMax packetNumber // TODO: Track packet numbers. + p, n := parse1RTTPacket(buf, tc.wkeys[space], len(tc.peerConnID), pnumMax) + if n < 0 { + tc.t.Fatalf("packet parse error") + } + dstConnID, _ := dstConnIDForDatagram(buf) + frames, err := tc.parseTestFrames(p.payload) + if err != nil { + tc.t.Fatal(err) + } + d.packets = append(d.packets, &testPacket{ + ptype: packetType1RTT, + num: p.num, + dstConnID: dstConnID, + frames: frames, + }) + buf = buf[n:] + } + } + return d +} + +func (tc *testConn) parseTestFrames(payload []byte) ([]debugFrame, error) { + tc.t.Helper() + var frames []debugFrame + for len(payload) > 0 { + f, n := parseDebugFrame(payload) + if n < 0 { + return nil, errors.New("error parsing frames") + } + if !tc.ignoreFrames[payload[0]] { + frames = append(frames, f) + } + payload = payload[n:] + } + return frames, nil +} + +func spaceForPacketType(ptype packetType) numberSpace { + switch ptype { + case packetTypeInitial: + return initialSpace + case packetType0RTT: + panic("TODO: packetType0RTT") + case packetTypeHandshake: + return handshakeSpace + case packetTypeRetry: + panic("TODO: packetTypeRetry") + case packetType1RTT: + return appDataSpace + } + panic("unknown packet type") +} + // testConnHooks implements connTestHooks. type testConnHooks testConn @@ -186,3 +566,11 @@ func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.T m = <-msgc return tc.now, m } + +// testConnListener implements connListener. +type testConnListener testConn + +func (tc *testConnListener) sendDatagram(p []byte, addr netip.AddrPort) error { + tc.sentDatagrams = append(tc.sentDatagrams, append([]byte(nil), p...)) + return nil +} diff --git a/internal/quic/dgram.go b/internal/quic/dgram.go new file mode 100644 index 0000000000..79e6650fa4 --- /dev/null +++ b/internal/quic/dgram.go @@ -0,0 +1,38 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "net/netip" + "sync" +) + +type datagram struct { + b []byte + addr netip.AddrPort +} + +var datagramPool = sync.Pool{ + New: func() any { + return &datagram{ + b: make([]byte, maxUDPPayloadSize), + } + }, +} + +func newDatagram() *datagram { + m := datagramPool.Get().(*datagram) + m.b = m.b[:cap(m.b)] + return m +} + +func (m *datagram) recycle() { + if cap(m.b) != maxUDPPayloadSize { + return + } + datagramPool.Put(m) +} diff --git a/internal/quic/frame_debug.go b/internal/quic/frame_debug.go index 945bb9d1f7..3009a04507 100644 --- a/internal/quic/frame_debug.go +++ b/internal/quic/frame_debug.go @@ -120,7 +120,7 @@ type debugFrameAck struct { func parseDebugFrameAck(b []byte) (f debugFrameAck, n int) { f.ranges = nil - _, f.ackDelay, n = consumeAckFrame(b, func(start, end packetNumber) { + _, f.ackDelay, n = consumeAckFrame(b, func(_ int, start, end packetNumber) { f.ranges = append(f.ranges, i64range[packetNumber]{ start: start, end: end, diff --git a/internal/quic/packet_parser.go b/internal/quic/packet_parser.go index 908a82ed90..c22f031038 100644 --- a/internal/quic/packet_parser.go +++ b/internal/quic/packet_parser.go @@ -91,7 +91,7 @@ func parseLongHeaderPacket(pkt []byte, k keys, pnumMax packetNumber) (p longPack pnumOff := len(pkt) - len(b) pkt = pkt[:pnumOff+int(payLen)] - if k.initialized() { + if k.isSet() { var err error p.payload, p.num, err = k.unprotect(pkt, pnumOff, pnumMax) if err != nil { @@ -162,7 +162,7 @@ func parse1RTTPacket(pkt []byte, k keys, dstConnIDLen int, pnumMax packetNumber) // which includes both general parse failures and specific violations of frame // constraints. -func consumeAckFrame(frame []byte, f func(start, end packetNumber)) (largest packetNumber, ackDelay unscaledAckDelay, n int) { +func consumeAckFrame(frame []byte, f func(rangeIndex int, start, end packetNumber)) (largest packetNumber, ackDelay unscaledAckDelay, n int) { b := frame[1:] // type largestAck, n := consumeVarint(b) @@ -195,7 +195,7 @@ func consumeAckFrame(frame []byte, f func(start, end packetNumber)) (largest pac if rangeMin < 0 || rangeMin > rangeMax { return 0, 0, -1 } - f(rangeMin, rangeMax+1) + f(int(i), rangeMin, rangeMax+1) if i == ackRangeCount { break diff --git a/internal/quic/packet_protection.go b/internal/quic/packet_protection.go index 1f0a735e8e..18470536ff 100644 --- a/internal/quic/packet_protection.go +++ b/internal/quic/packet_protection.go @@ -163,8 +163,8 @@ func (k keys) xorIV(pnum packetNumber) { k.iv[len(k.iv)-1] ^= uint8(pnum) } -// initialized returns true if valid keys are available. -func (k keys) initialized() bool { +// isSet returns true if valid keys are available. +func (k keys) isSet() bool { return k.aead != nil } diff --git a/internal/quic/packet_test.go b/internal/quic/packet_test.go index b13a587e54..f3a8b7d570 100644 --- a/internal/quic/packet_test.go +++ b/internal/quic/packet_test.go @@ -9,10 +9,27 @@ package quic import ( "bytes" "encoding/hex" + "fmt" "strings" "testing" ) +func (p packetType) String() string { + switch p { + case packetTypeInitial: + return "Initial" + case packetType0RTT: + return "0-RTT" + case packetTypeHandshake: + return "Handshake" + case packetTypeRetry: + return "Retry" + case packetType1RTT: + return "1-RTT" + } + return fmt.Sprintf("unknown packet type %v", byte(p)) +} + func TestPacketHeader(t *testing.T) { for _, test := range []struct { name string diff --git a/internal/quic/packet_writer.go b/internal/quic/packet_writer.go index 97987e0c2f..6c4c452cdd 100644 --- a/internal/quic/packet_writer.go +++ b/internal/quic/packet_writer.go @@ -237,7 +237,10 @@ func (w *packetWriter) appendPingFrame() (added bool) { return false } w.b = append(w.b, frameTypePing) - w.sent.appendAckElicitingFrame(frameTypePing) + // Mark this packet as ack-eliciting and in-flight, + // but there's no need to record the presence of a PING frame in it. + w.sent.ackEliciting = true + w.sent.inFlight = true return true } diff --git a/internal/quic/ping.go b/internal/quic/ping.go new file mode 100644 index 0000000000..3e7d9c51bd --- /dev/null +++ b/internal/quic/ping.go @@ -0,0 +1,16 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "time" + +func (c *Conn) ping(space numberSpace) { + c.sendMsg(func(now time.Time, c *Conn) { + c.testSendPing.setUnsent() + c.testSendPingSpace = space + }) +} diff --git a/internal/quic/ping_test.go b/internal/quic/ping_test.go new file mode 100644 index 0000000000..4a732ed543 --- /dev/null +++ b/internal/quic/ping_test.go @@ -0,0 +1,35 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "testing" + +func TestPing(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.conn.ping(initialSpace) + tc.wantFrame("connection should send a PING frame", + packetTypeInitial, debugFramePing{}) + + tc.advanceToTimer() + tc.wantFrame("on PTO, connection should send another PING frame", + packetTypeInitial, debugFramePing{}) + + tc.wantIdle("after sending PTO probe, no additional frames to send") +} + +func TestAck(t *testing.T) { + tc := newTestConn(t, serverSide) + tc.writeFrames(packetTypeInitial, + debugFramePing{}, + ) + tc.wantFrame("connection should respond to ack-eliciting packet with an ACK frame", + packetTypeInitial, + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, + ) +} diff --git a/internal/quic/quic.go b/internal/quic/quic.go index c69c0b9840..9df7f7e2b1 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -41,6 +41,10 @@ const ( // https://www.rfc-editor.org/rfc/rfc9002.html#section-6.1.2-6 const timerGranularity = 1 * time.Millisecond +// Minimum size of a UDP datagram sent by a client carrying an Initial packet. +// https://www.rfc-editor.org/rfc/rfc9000#section-14.1 +const minimumClientInitialDatagramSize = 1200 + // A connSide distinguishes between the client and server sides of a connection. type connSide int8 diff --git a/internal/quic/sent_packet.go b/internal/quic/sent_packet.go index e5a80be3bb..4f11aa1368 100644 --- a/internal/quic/sent_packet.go +++ b/internal/quic/sent_packet.go @@ -29,6 +29,8 @@ type sentPacket struct { // we need to process an ack for or loss of this packet. // For example, a CRYPTO frame is recorded as the frame type (0x06), offset, and length, // but does not include the sent data. + // + // This buffer is written by packetWriter.append* and read by Conn.handleAckOrLoss. b []byte n int // read offset into b } diff --git a/internal/quic/tls.go b/internal/quic/tls.go new file mode 100644 index 0000000000..1cdb727e24 --- /dev/null +++ b/internal/quic/tls.go @@ -0,0 +1,23 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +// tlsState encapsulates interactions with TLS. +type tlsState struct { + // Encryption keys indexed by number space. + rkeys [numberSpaceCount]keys + wkeys [numberSpaceCount]keys +} + +func (s *tlsState) init(side connSide, initialConnID []byte) { + clientKeys, serverKeys := initialKeys(initialConnID) + if side == clientSide { + s.wkeys[initialSpace], s.rkeys[initialSpace] = clientKeys, serverKeys + } else { + s.wkeys[initialSpace], s.rkeys[initialSpace] = serverKeys, clientKeys + } +} From 8db2eadc7c3bda7baabb79fe1fadb8a093d5d391 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 18 Jan 2023 09:47:28 -0800 Subject: [PATCH 10/46] quic: transport parameter encoding and decoding Transport parameters are passed in the extension_data field of the quic_transport_parameters TLS extension. RFC 9000, Section 18. RFC 9001, Section 8.2. For golang/go#58547 Change-Id: I294ab6cdef19256f5db02dc269e8b417b1d5e54b Reviewed-on: https://go-review.googlesource.com/c/net/+/510575 Auto-Submit: Damien Neil Reviewed-by: Jonathan Amsterdam Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- internal/quic/transport_params.go | 277 ++++++++++++++++++ internal/quic/transport_params_test.go | 374 +++++++++++++++++++++++++ 2 files changed, 651 insertions(+) create mode 100644 internal/quic/transport_params.go create mode 100644 internal/quic/transport_params_test.go diff --git a/internal/quic/transport_params.go b/internal/quic/transport_params.go new file mode 100644 index 0000000000..416bfb8679 --- /dev/null +++ b/internal/quic/transport_params.go @@ -0,0 +1,277 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "encoding/binary" + "net/netip" + "time" +) + +// transportParameters transferred in the quic_transport_parameters TLS extension. +// https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2 +type transportParameters struct { + originalDstConnID []byte + maxIdleTimeout time.Duration + statelessResetToken []byte + maxUDPPayloadSize int64 + initialMaxData int64 + initialMaxStreamDataBidiLocal int64 + initialMaxStreamDataBidiRemote int64 + initialMaxStreamDataUni int64 + initialMaxStreamsBidi int64 + initialMaxStreamsUni int64 + ackDelayExponent uint8 + maxAckDelay time.Duration + disableActiveMigration bool + preferredAddrV4 netip.AddrPort + preferredAddrV6 netip.AddrPort + preferredAddrConnID []byte + preferredAddrResetToken []byte + activeConnIDLimit int64 + initialSrcConnID []byte + retrySrcConnID []byte +} + +const ( + defaultParamMaxUDPPayloadSize = 65527 + defaultParamAckDelayExponent = 3 + defaultParamMaxAckDelayMilliseconds = 25 + defaultParamActiveConnIDLimit = 2 +) + +// defaultTransportParameters is initialized to the RFC 9000 default values. +func defaultTransportParameters() transportParameters { + return transportParameters{ + maxUDPPayloadSize: defaultParamMaxUDPPayloadSize, + ackDelayExponent: defaultParamAckDelayExponent, + maxAckDelay: defaultParamMaxAckDelayMilliseconds * time.Millisecond, + activeConnIDLimit: defaultParamActiveConnIDLimit, + } +} + +const ( + paramOriginalDestinationConnectionID = 0x00 + paramMaxIdleTimeout = 0x01 + paramStatelessResetToken = 0x02 + paramMaxUDPPayloadSize = 0x03 + paramInitialMaxData = 0x04 + paramInitialMaxStreamDataBidiLocal = 0x05 + paramInitialMaxStreamDataBidiRemote = 0x06 + paramInitialMaxStreamDataUni = 0x07 + paramInitialMaxStreamsBidi = 0x08 + paramInitialMaxStreamsUni = 0x09 + paramAckDelayExponent = 0x0a + paramMaxAckDelay = 0x0b + paramDisableActiveMigration = 0x0c + paramPreferredAddress = 0x0d + paramActiveConnectionIDLimit = 0x0e + paramInitialSourceConnectionID = 0x0f + paramRetrySourceConnectionID = 0x10 +) + +func marshalTransportParameters(p transportParameters) []byte { + var b []byte + if v := p.originalDstConnID; v != nil { + b = appendVarint(b, paramOriginalDestinationConnectionID) + b = appendVarintBytes(b, v) + } + if v := uint64(p.maxIdleTimeout / time.Millisecond); v != 0 { + b = appendVarint(b, paramMaxIdleTimeout) + b = appendVarint(b, uint64(sizeVarint(v))) + b = appendVarint(b, uint64(v)) + } + if v := p.statelessResetToken; v != nil { + b = appendVarint(b, paramStatelessResetToken) + b = appendVarintBytes(b, v) + } + if v := p.maxUDPPayloadSize; v != defaultParamMaxUDPPayloadSize { + b = appendVarint(b, paramMaxUDPPayloadSize) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxData; v != 0 { + b = appendVarint(b, paramInitialMaxData) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxStreamDataBidiLocal; v != 0 { + b = appendVarint(b, paramInitialMaxStreamDataBidiLocal) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxStreamDataBidiRemote; v != 0 { + b = appendVarint(b, paramInitialMaxStreamDataBidiRemote) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxStreamDataUni; v != 0 { + b = appendVarint(b, paramInitialMaxStreamDataUni) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxStreamsBidi; v != 0 { + b = appendVarint(b, paramInitialMaxStreamsBidi) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialMaxStreamsUni; v != 0 { + b = appendVarint(b, paramInitialMaxStreamsUni) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.ackDelayExponent; v != defaultParamAckDelayExponent { + b = appendVarint(b, paramAckDelayExponent) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := uint64(p.maxAckDelay / time.Millisecond); v != defaultParamMaxAckDelayMilliseconds { + b = appendVarint(b, paramMaxAckDelay) + b = appendVarint(b, uint64(sizeVarint(v))) + b = appendVarint(b, v) + } + if p.disableActiveMigration { + b = appendVarint(b, paramDisableActiveMigration) + b = append(b, 0) // 0-length value + } + if p.preferredAddrConnID != nil { + b = append(b, paramPreferredAddress) + b = appendVarint(b, uint64(4+2+16+2+1+len(p.preferredAddrConnID)+16)) + b = append(b, p.preferredAddrV4.Addr().AsSlice()...) // 4 bytes + b = binary.BigEndian.AppendUint16(b, p.preferredAddrV4.Port()) // 2 bytes + b = append(b, p.preferredAddrV6.Addr().AsSlice()...) // 16 bytes + b = binary.BigEndian.AppendUint16(b, p.preferredAddrV6.Port()) // 2 bytes + b = appendUint8Bytes(b, p.preferredAddrConnID) // 1 byte + len(conn_id) + b = append(b, p.preferredAddrResetToken...) // 16 bytes + } + if v := p.activeConnIDLimit; v != defaultParamActiveConnIDLimit { + b = appendVarint(b, paramActiveConnectionIDLimit) + b = appendVarint(b, uint64(sizeVarint(uint64(v)))) + b = appendVarint(b, uint64(v)) + } + if v := p.initialSrcConnID; v != nil { + b = appendVarint(b, paramInitialSourceConnectionID) + b = appendVarintBytes(b, v) + } + if v := p.retrySrcConnID; v != nil { + b = appendVarint(b, paramRetrySourceConnectionID) + b = appendVarintBytes(b, v) + } + return b +} + +func unmarshalTransportParams(params []byte) (transportParameters, error) { + p := defaultTransportParameters() + for len(params) > 0 { + id, n := consumeVarint(params) + if n < 0 { + return p, localTransportError(errTransportParameter) + } + params = params[n:] + val, n := consumeVarintBytes(params) + if n < 0 { + return p, localTransportError(errTransportParameter) + } + params = params[n:] + n = 0 + switch id { + case paramOriginalDestinationConnectionID: + p.originalDstConnID = val + n = len(val) + case paramMaxIdleTimeout: + var v uint64 + v, n = consumeVarint(val) + // If this is unreasonably large, consider it as no timeout to avoid + // time.Duration overflows. + if v > 1<<32 { + v = 0 + } + p.maxIdleTimeout = time.Duration(v) * time.Millisecond + case paramStatelessResetToken: + if len(val) != 16 { + return p, localTransportError(errTransportParameter) + } + p.statelessResetToken = val + n = 16 + case paramMaxUDPPayloadSize: + p.maxUDPPayloadSize, n = consumeVarintInt64(val) + if p.maxUDPPayloadSize < 1200 { + return p, localTransportError(errTransportParameter) + } + case paramInitialMaxData: + p.initialMaxData, n = consumeVarintInt64(val) + case paramInitialMaxStreamDataBidiLocal: + p.initialMaxStreamDataBidiLocal, n = consumeVarintInt64(val) + case paramInitialMaxStreamDataBidiRemote: + p.initialMaxStreamDataBidiRemote, n = consumeVarintInt64(val) + case paramInitialMaxStreamDataUni: + p.initialMaxStreamDataUni, n = consumeVarintInt64(val) + case paramInitialMaxStreamsBidi: + p.initialMaxStreamsBidi, n = consumeVarintInt64(val) + case paramInitialMaxStreamsUni: + p.initialMaxStreamsUni, n = consumeVarintInt64(val) + case paramAckDelayExponent: + var v uint64 + v, n = consumeVarint(val) + if v > 20 { + return p, localTransportError(errTransportParameter) + } + p.ackDelayExponent = uint8(v) + case paramMaxAckDelay: + var v uint64 + v, n = consumeVarint(val) + if v >= 1<<14 { + return p, localTransportError(errTransportParameter) + } + p.maxAckDelay = time.Duration(v) * time.Millisecond + case paramDisableActiveMigration: + p.disableActiveMigration = true + case paramPreferredAddress: + if len(val) < 4+2+16+2+1 { + return p, localTransportError(errTransportParameter) + } + p.preferredAddrV4 = netip.AddrPortFrom( + netip.AddrFrom4(*(*[4]byte)(val[:4])), + binary.BigEndian.Uint16(val[4:][:2]), + ) + val = val[4+2:] + p.preferredAddrV6 = netip.AddrPortFrom( + netip.AddrFrom16(*(*[16]byte)(val[:16])), + binary.BigEndian.Uint16(val[16:][:2]), + ) + val = val[16+2:] + var nn int + p.preferredAddrConnID, nn = consumeUint8Bytes(val) + if nn < 0 { + return p, localTransportError(errTransportParameter) + } + val = val[nn:] + if len(val) != 16 { + return p, localTransportError(errTransportParameter) + } + p.preferredAddrResetToken = val + val = nil + case paramActiveConnectionIDLimit: + p.activeConnIDLimit, n = consumeVarintInt64(val) + if p.activeConnIDLimit < 2 { + return p, localTransportError(errTransportParameter) + } + case paramInitialSourceConnectionID: + p.initialSrcConnID = val + n = len(val) + case paramRetrySourceConnectionID: + p.retrySrcConnID = val + n = len(val) + default: + n = len(val) + } + if n != len(val) { + return p, localTransportError(errTransportParameter) + } + } + return p, nil +} diff --git a/internal/quic/transport_params_test.go b/internal/quic/transport_params_test.go new file mode 100644 index 0000000000..e1c45ca0e6 --- /dev/null +++ b/internal/quic/transport_params_test.go @@ -0,0 +1,374 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "bytes" + "math" + "net/netip" + "reflect" + "testing" + "time" +) + +func TestTransportParametersMarshalUnmarshal(t *testing.T) { + for _, test := range []struct { + params func(p *transportParameters) + enc []byte + }{{ + params: func(p *transportParameters) { + p.originalDstConnID = []byte("connid") + }, + enc: []byte{ + 0x00, // original_destination_connection_id + byte(len("connid")), + 'c', 'o', 'n', 'n', 'i', 'd', + }, + }, { + params: func(p *transportParameters) { + p.maxIdleTimeout = 10 * time.Millisecond + }, + enc: []byte{ + 0x01, // max_idle_timeout + 1, // length + 10, // varint msecs + }, + }, { + params: func(p *transportParameters) { + p.statelessResetToken = []byte("0123456789abcdef") + }, + enc: []byte{ + 0x02, // stateless_reset_token + 16, // length + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token + }, + }, { + params: func(p *transportParameters) { + p.maxUDPPayloadSize = 1200 + }, + enc: []byte{ + 0x03, // max_udp_payload_size + 2, // length + 0x44, 0xb0, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxData = 10 + }, + enc: []byte{ + 0x04, // initial_max_data + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxStreamDataBidiLocal = 10 + }, + enc: []byte{ + 0x05, // initial_max_stream_data_bidi_local + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxStreamDataBidiRemote = 10 + }, + enc: []byte{ + 0x06, // initial_max_stream_data_bidi_remote + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxStreamDataUni = 10 + }, + enc: []byte{ + 0x07, // initial_max_stream_data_uni + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxStreamsBidi = 10 + }, + enc: []byte{ + 0x08, // initial_max_streams_bidi + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialMaxStreamsUni = 10 + }, + enc: []byte{ + 0x09, // initial_max_streams_uni + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.ackDelayExponent = 4 + }, + enc: []byte{ + 0x0a, // ack_delay_exponent + 1, // length + 4, // varint value + }, + }, { + params: func(p *transportParameters) { + p.maxAckDelay = 10 * time.Millisecond + }, + enc: []byte{ + 0x0b, // max_ack_delay + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.disableActiveMigration = true + }, + enc: []byte{ + 0x0c, // disable_active_migration + 0, // length + }, + }, { + params: func(p *transportParameters) { + p.preferredAddrV4 = netip.MustParseAddrPort("127.0.0.1:80") + p.preferredAddrV6 = netip.MustParseAddrPort("[fe80::1]:1024") + p.preferredAddrConnID = []byte("connid") + p.preferredAddrResetToken = []byte("0123456789abcdef") + }, + enc: []byte{ + 0x0d, // preferred_address + byte(4 + 2 + 16 + 2 + 1 + len("connid") + 16), // length + 127, 0, 0, 1, // v4 address + 0, 80, // v4 port + 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address + 0x04, 0x00, // v6 port, + 6, // connection id length + 'c', 'o', 'n', 'n', 'i', 'd', // connection id + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token + }, + }, { + params: func(p *transportParameters) { + p.activeConnIDLimit = 10 + }, + enc: []byte{ + 0x0e, // active_connection_id_limit + 1, // length + 10, // varint value + }, + }, { + params: func(p *transportParameters) { + p.initialSrcConnID = []byte("connid") + }, + enc: []byte{ + 0x0f, // initial_source_connection_id + byte(len("connid")), + 'c', 'o', 'n', 'n', 'i', 'd', + }, + }, { + params: func(p *transportParameters) { + p.retrySrcConnID = []byte("connid") + }, + enc: []byte{ + 0x10, // retry_source_connection_id + byte(len("connid")), + 'c', 'o', 'n', 'n', 'i', 'd', + }, + }} { + wantParams := defaultTransportParameters() + test.params(&wantParams) + gotBytes := marshalTransportParameters(wantParams) + if !bytes.Equal(gotBytes, test.enc) { + t.Errorf("marshalTransportParameters(%#v):\n got: %x\nwant: %x", wantParams, gotBytes, test.enc) + } + gotParams, err := unmarshalTransportParams(test.enc) + if err != nil { + t.Errorf("unmarshalTransportParams(%x): unexpected error: %v", test.enc, err) + } else if !reflect.DeepEqual(gotParams, wantParams) { + t.Errorf("unmarshalTransportParams(%x):\n got: %#v\nwant: %#v", test.enc, gotParams, wantParams) + } + } +} + +func TestTransportParametersErrors(t *testing.T) { + for _, test := range []struct { + desc string + enc []byte + }{{ + desc: "invalid id", + enc: []byte{ + 0x40, // too short + }, + }, { + desc: "parameter too short", + enc: []byte{ + 0x00, // original_destination_connection_id + 0x04, // length + 1, 2, 3, // not enough data + }, + }, { + desc: "extra data in parameter", + enc: []byte{ + 0x01, // max_idle_timeout + 2, // length + 10, // varint msecs + 0, // extra junk + }, + }, { + desc: "invalid varint in parameter", + enc: []byte{ + 0x01, // max_idle_timeout + 1, // length + 0x40, // incomplete varint + }, + }, { + desc: "stateless_reset_token not 16 bytes", + enc: []byte{ + 0x02, // stateless_reset_token, + 15, // length + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + }, + }, { + desc: "preferred_address is too short", + enc: []byte{ + 0x0d, // preferred_address + byte(3), + 127, 0, 0, + }, + }, { + desc: "preferred_address reset token too short", + enc: []byte{ + 0x0d, // preferred_address + byte(4 + 2 + 16 + 2 + 1 + len("connid") + 15), // length + 127, 0, 0, 1, // v4 address + 0, 80, // v4 port + 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address + 0x04, 0x00, // v6 port, + 6, // connection id length + 'c', 'o', 'n', 'n', 'i', 'd', // connection id + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', // reset token, one byte too short + + }, + }, { + desc: "preferred_address conn id too long", + enc: []byte{ + 0x0d, // preferred_address + byte(4 + 2 + 16 + 2 + 1 + len("connid") + 16), // length + 127, 0, 0, 1, // v4 address + 0, 80, // v4 port + 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address + 0x04, 0x00, // v6 port, + byte(len("connid")) + 16 + 1, // connection id length, too long + 'c', 'o', 'n', 'n', 'i', 'd', // connection id + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token + + }, + }} { + _, err := unmarshalTransportParams(test.enc) + if err == nil { + t.Errorf("%v:\nunmarshalTransportParams(%x): unexpectedly succeeded", test.desc, test.enc) + } + } +} + +func TestTransportParametersRangeErrors(t *testing.T) { + for _, test := range []struct { + desc string + params func(p *transportParameters) + }{{ + desc: "max_udp_payload_size < 1200", + params: func(p *transportParameters) { + p.maxUDPPayloadSize = 1199 + }, + }, { + desc: "ack_delay_exponent > 20", + params: func(p *transportParameters) { + p.ackDelayExponent = 21 + }, + }, { + desc: "max_ack_delay > 1^14 ms", + params: func(p *transportParameters) { + p.maxAckDelay = (1 << 14) * time.Millisecond + }, + }, { + desc: "active_connection_id_limit < 2", + params: func(p *transportParameters) { + p.activeConnIDLimit = 1 + }, + }} { + p := defaultTransportParameters() + test.params(&p) + enc := marshalTransportParameters(p) + _, err := unmarshalTransportParams(enc) + if err == nil { + t.Errorf("%v: unmarshalTransportParams unexpectedly succeeded", test.desc) + } + } +} + +func TestTransportParameterMaxIdleTimeoutOverflowsDuration(t *testing.T) { + tooManyMS := 1 + (math.MaxInt64 / uint64(time.Millisecond)) + + var enc []byte + enc = appendVarint(enc, paramMaxIdleTimeout) + enc = appendVarint(enc, uint64(sizeVarint(tooManyMS))) + enc = appendVarint(enc, uint64(tooManyMS)) + + dec, err := unmarshalTransportParams(enc) + if err != nil { + t.Fatalf("unmarshalTransportParameters(enc) = %v", err) + } + if got, want := dec.maxIdleTimeout, time.Duration(0); got != want { + t.Errorf("max_idle_timeout=%v, got maxIdleTimeout=%v; want %v", tooManyMS, got, want) + } +} + +func TestTransportParametersSkipUnknownParameters(t *testing.T) { + enc := []byte{ + 0x20, // unknown transport parameter + 1, // length + 0, // varint value + + 0x04, // initial_max_data + 1, // length + 10, // varint value + + 0x21, // unknown transport parameter + 1, // length + 0, // varint value + } + dec, err := unmarshalTransportParams(enc) + if err != nil { + t.Fatalf("unmarshalTransportParameters(enc) = %v", err) + } + if got, want := dec.initialMaxData, int64(10); got != want { + t.Errorf("got initial_max_data=%v; want %v", got, want) + } +} + +func FuzzTransportParametersMarshalUnmarshal(f *testing.F) { + f.Fuzz(func(t *testing.T, in []byte) { + p1, err := unmarshalTransportParams(in) + if err != nil { + return + } + out := marshalTransportParameters(p1) + p2, err := unmarshalTransportParams(out) + if err != nil { + t.Fatalf("round trip unmarshal/remarshal: unmarshal error: %v\n%x", err, in) + } + if !reflect.DeepEqual(p1, p2) { + t.Fatalf("round trip unmarshal/remarshal: parameters differ:\n%x\n%#v\n%#v", in, p1, p2) + } + }) +} From d0912d407c27bec63dfa5df0ef9012910774f8f4 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 14 Jul 2023 12:52:21 -0700 Subject: [PATCH 11/46] quic: add pipe type Streams (including CRYPTO streams) are an ordered byte sequence. Both outgoing and incoming streams require random access to a portion of that sequence. Outbound packets may be lost, requiring us to resend the data in the lost packet. Inbound packets may arrive out of order. Add a "pipe" type as a building block for both inbound and outbound streams. A pipe is a window into a portion of a stream, permitting random read and write access within that window (unlike bufio.Reader or bufio.Writer). Pipes are implemented as a linked list of blocks. Block sizes are uniform and allocations are pooled, avoiding non-pool allocations in the steady state. Pipe memory consumption is proportional to the current window, and goes to zero when the window has been fully consumed (unlike bytes.Buffer). For golang/go#58547 Change-Id: I0c16707552c9c46f31055daea2396590a924fc60 Reviewed-on: https://go-review.googlesource.com/c/net/+/510615 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/pipe.go | 149 +++++++++++++++++++++++++++++++++++++ internal/quic/pipe_test.go | 95 +++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 internal/quic/pipe.go create mode 100644 internal/quic/pipe_test.go diff --git a/internal/quic/pipe.go b/internal/quic/pipe.go new file mode 100644 index 0000000000..978a4f3d8b --- /dev/null +++ b/internal/quic/pipe.go @@ -0,0 +1,149 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "sync" +) + +// A pipe is a byte buffer used in implementing streams. +// +// A pipe contains a window of stream data. +// Random access reads and writes are supported within the window. +// Writing past the end of the window extends it. +// Data may be discarded from the start of the pipe, advancing the window. +type pipe struct { + start int64 + end int64 + head *pipebuf + tail *pipebuf +} + +type pipebuf struct { + off int64 + b []byte + next *pipebuf +} + +func (pb *pipebuf) end() int64 { + return pb.off + int64(len(pb.b)) +} + +var pipebufPool = sync.Pool{ + New: func() any { + return &pipebuf{ + b: make([]byte, 4096), + } + }, +} + +func newPipebuf() *pipebuf { + return pipebufPool.Get().(*pipebuf) +} + +func (b *pipebuf) recycle() { + b.off = 0 + b.next = nil + pipebufPool.Put(b) +} + +// writeAt writes len(b) bytes to the pipe at offset off. +// +// Writes to offsets before p.start are discarded. +// Writes to offsets after p.end extend the pipe window. +func (p *pipe) writeAt(b []byte, off int64) { + end := off + int64(len(b)) + if end > p.end { + p.end = end + } else if end <= p.start { + return + } + + if off < p.start { + // Discard the portion of b which falls before p.start. + trim := p.start - off + b = b[trim:] + off = p.start + } + + if p.head == nil { + p.head = newPipebuf() + p.head.off = p.start + p.tail = p.head + } + pb := p.head + if off >= p.tail.off { + // Common case: Writing past the end of the pipe. + pb = p.tail + } + for { + pboff := off - pb.off + if pboff < int64(len(pb.b)) { + n := copy(pb.b[pboff:], b) + if n == len(b) { + return + } + off += int64(n) + b = b[n:] + } + if pb.next == nil { + pb.next = newPipebuf() + pb.next.off = pb.off + int64(len(pb.b)) + p.tail = pb.next + } + pb = pb.next + } +} + +// copy copies len(b) bytes into b starting from off. +// The pipe must contain [off, off+len(b)). +func (p *pipe) copy(off int64, b []byte) { + dst := b[:0] + p.read(off, len(b), func(c []byte) error { + dst = append(dst, c...) + return nil + }) +} + +// read calls f with the data in [off, off+n) +// The data may be provided sequentially across multiple calls to f. +func (p *pipe) read(off int64, n int, f func([]byte) error) error { + if off < p.start { + panic("invalid read range") + } + for pb := p.head; pb != nil && n > 0; pb = pb.next { + if off >= pb.end() { + continue + } + b := pb.b[off-pb.off:] + if len(b) > n { + b = b[:n] + } + off += int64(len(b)) + n -= len(b) + if err := f(b); err != nil { + return err + } + } + if n > 0 { + panic("invalid read range") + } + return nil +} + +// discardBefore discards all data prior to off. +func (p *pipe) discardBefore(off int64) { + for p.head != nil && p.head.end() < off { + head := p.head + p.head = p.head.next + head.recycle() + } + if p.head == nil { + p.tail = nil + } + p.start = off +} diff --git a/internal/quic/pipe_test.go b/internal/quic/pipe_test.go new file mode 100644 index 0000000000..7a05ff4d47 --- /dev/null +++ b/internal/quic/pipe_test.go @@ -0,0 +1,95 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "bytes" + "math/rand" + "testing" +) + +func TestPipeWrites(t *testing.T) { + type writeOp struct { + start, end int64 + } + type discardBeforeOp struct { + off int64 + } + type op any + src := make([]byte, 65536) + rand.New(rand.NewSource(0)).Read(src) + for _, test := range []struct { + desc string + ops []op + }{{ + desc: "sequential writes", + ops: []op{ + writeOp{0, 1024}, + writeOp{1024, 4096}, + writeOp{4096, 65536}, + }, + }, { + desc: "disordered overlapping writes", + ops: []op{ + writeOp{2000, 8000}, + writeOp{0, 3000}, + writeOp{7000, 12000}, + }, + }, { + desc: "write to discarded region", + ops: []op{ + writeOp{0, 65536}, + discardBeforeOp{32768}, + writeOp{0, 1000}, + writeOp{3000, 5000}, + writeOp{0, 32768}, + }, + }, { + desc: "write overlaps discarded region", + ops: []op{ + discardBeforeOp{10000}, + writeOp{0, 20000}, + }, + }, { + desc: "discard everything", + ops: []op{ + writeOp{0, 10000}, + discardBeforeOp{10000}, + writeOp{10000, 20000}, + }, + }} { + var p pipe + var wantset rangeset[int64] + var wantStart, wantEnd int64 + for i, o := range test.ops { + switch o := o.(type) { + case writeOp: + p.writeAt(src[o.start:o.end], o.start) + wantset.add(o.start, o.end) + wantset.sub(0, wantStart) + if o.end > wantEnd { + wantEnd = o.end + } + case discardBeforeOp: + p.discardBefore(o.off) + wantset.sub(0, o.off) + wantStart = o.off + } + if p.start != wantStart || p.end != wantEnd { + t.Errorf("%v: after %#v p contains [%v,%v), want [%v,%v)", test.desc, test.ops[:i+1], p.start, p.end, wantStart, wantEnd) + } + for _, r := range wantset { + want := src[r.start:][:r.size()] + got := make([]byte, r.size()) + p.copy(r.start, got) + if !bytes.Equal(got, want) { + t.Errorf("%v after %#v, mismatch in data in %v", test.desc, test.ops[:i+1], r) + } + } + } + } +} From dd5bc96b138a4a27c151b47f09059ce31d13ecfc Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 20 Jul 2023 16:50:08 -0700 Subject: [PATCH 12/46] internal/quic: deflake TestConnTestConn Sending a message to a connection returns an error when the connection event loop had exited. This is unreliable, since a sent to the conn's message channel can succeed after the event loop exits, writing the message to the channel buffer. Drop the error return from Conn.sendMsg; it isn't useful, since it's always possible for the connection to exit with messages still in the channel buffer. Fixes golang/go#61485 Change-Id: Ic8351f984df827af881cf7b6d93d97031d2e615c Reviewed-on: https://go-review.googlesource.com/c/net/+/511658 TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam Auto-Submit: Damien Neil Run-TryBot: Damien Neil --- internal/quic/conn.go | 11 ++++------- internal/quic/conn_test.go | 3 --- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/internal/quic/conn.go b/internal/quic/conn.go index cdf79d607c..e6375e86a8 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -176,24 +176,21 @@ func (c *Conn) loop(now time.Time) { // sendMsg sends a message to the conn's loop. // It does not wait for the message to be processed. -func (c *Conn) sendMsg(m any) error { +// The conn may close before processing the message, in which case it is lost. +func (c *Conn) sendMsg(m any) { select { case c.msgc <- m: case <-c.donec: - return errors.New("quic: connection closed") } - return nil } // runOnLoop executes a function within the conn's loop goroutine. func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { donec := make(chan struct{}) - if err := c.sendMsg(func(now time.Time, c *Conn) { + c.sendMsg(func(now time.Time, c *Conn) { defer close(donec) f(now, c) - }); err != nil { - return err - } + }) select { case <-donec: case <-c.donec: diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 6bb12e210e..fda1d4b869 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -43,9 +43,6 @@ func TestConnTestConn(t *testing.T) { tc.wait() tc.advanceToTimer() - if err := tc.conn.sendMsg(nil); err == nil { - t.Errorf("after advancing to idle timeout, sendMsg = nil, want error") - } if !tc.conn.exited { t.Errorf("after advancing to idle timeout, exited = false, want true") } From 5e678bb28c36ba4aef595a4e468e51eda5d71c12 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 31 Jan 2023 08:58:13 -0800 Subject: [PATCH 13/46] quic: CRYPTO stream handling CRYPTO frames carry TLS handshake messages. Add a cryptoStream type which manages the TLS handshake stream, including retransmission of lost data, processing out-of-order received data, etc. For golang/go#58547 Change-Id: I8defa38e22d9c1bb8753f3a44d5ae0853fa56de8 Reviewed-on: https://go-review.googlesource.com/c/net/+/510616 Reviewed-by: Jonathan Amsterdam Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- internal/quic/crypto_stream.go | 159 +++++++++++++++++ internal/quic/crypto_stream_test.go | 265 ++++++++++++++++++++++++++++ 2 files changed, 424 insertions(+) create mode 100644 internal/quic/crypto_stream.go create mode 100644 internal/quic/crypto_stream_test.go diff --git a/internal/quic/crypto_stream.go b/internal/quic/crypto_stream.go new file mode 100644 index 0000000000..6cda6578c1 --- /dev/null +++ b/internal/quic/crypto_stream.go @@ -0,0 +1,159 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +// "Implementations MUST support buffering at least 4096 bytes of data +// received in out-of-order CRYPTO frames." +// https://www.rfc-editor.org/rfc/rfc9000.html#section-7.5-2 +// +// 4096 is too small for real-world cases, however, so we allow more. +const cryptoBufferSize = 1 << 20 + +// A cryptoStream is the stream of data passed in CRYPTO frames. +// There is one cryptoStream per packet number space. +type cryptoStream struct { + // CRYPTO data received from the peer. + in pipe + inset rangeset[int64] // bytes received + + // CRYPTO data queued for transmission to the peer. + out pipe + outunsent rangeset[int64] // bytes in need of sending + outacked rangeset[int64] // bytes acked by peer +} + +// handleCrypto processes data received in a CRYPTO frame. +func (s *cryptoStream) handleCrypto(off int64, b []byte, f func([]byte) error) error { + end := off + int64(len(b)) + if end-s.inset.min() > cryptoBufferSize { + return localTransportError(errCryptoBufferExceeded) + } + s.inset.add(off, end) + if off == s.in.start { + // Fast path: This is the next chunk of data in the stream, + // so just handle it immediately. + if err := f(b); err != nil { + return err + } + s.in.discardBefore(end) + } else { + // This is either data we've already processed, + // data we can't process yet, or a mix of both. + s.in.writeAt(b, off) + } + // s.in.start is the next byte in sequence. + // If it's in s.inset, we have bytes to provide. + // If it isn't, we don't--we're either out of data, + // or only have data that comes after the next byte. + if !s.inset.contains(s.in.start) { + return nil + } + // size is the size of the first contiguous chunk of bytes + // that have not been processed yet. + size := int(s.inset[0].end - s.in.start) + if size <= 0 { + return nil + } + err := s.in.read(s.in.start, size, f) + s.in.discardBefore(s.inset[0].end) + return err +} + +// write queues data for sending to the peer. +// It does not block or limit the amount of buffered data. +// QUIC connections don't communicate the amount of CRYPTO data they are willing to buffer, +// so we send what we have and the peer can close the connection if it is too much. +func (s *cryptoStream) write(b []byte) { + start := s.out.end + s.out.writeAt(b, start) + s.outunsent.add(start, s.out.end) +} + +// ackOrLoss reports that an CRYPTO frame sent by us has been acknowledged by the peer, or lost. +func (s *cryptoStream) ackOrLoss(start, end int64, fate packetFate) { + switch fate { + case packetAcked: + s.outacked.add(start, end) + s.outunsent.sub(start, end) + // If this ack is for data at the start of the send buffer, we can now discard it. + if s.outacked.contains(s.out.start) { + s.out.discardBefore(s.outacked[0].end) + } + case packetLost: + // Mark everything lost, but not previously acked, as needing retransmission. + // We do this by adding all the lost bytes to outunsent, and then + // removing everything already acked. + s.outunsent.add(start, end) + for _, a := range s.outacked { + s.outunsent.sub(a.start, a.end) + } + } +} + +// dataToSend reports what data should be sent in CRYPTO frames to the peer. +// It calls f with each range of data to send. +// f uses sendData to get the bytes to send, and returns the number of bytes sent. +// dataToSend calls f until no data is left, or f returns 0. +// +// This function is unusually indirect (why not just return a []byte, +// or implement io.Reader?). +// +// Returning a []byte to the caller either requires that we store the +// data to send contiguously (which we don't), allocate a temporary buffer +// and copy into it (inefficient), or return less data than we have available +// (requires complexity to avoid unnecessarily breaking data across frames). +// +// Accepting a []byte from the caller (io.Reader) makes packet construction +// difficult. Since CRYPTO data is encoded with a varint length prefix, the +// location of the data depends on the length of the data. (We could hardcode +// a 2-byte length, of course.) +// +// Instead, we tell the caller how much data is, the caller figures out where +// to put it (and possibly decides that it doesn't have space for this data +// in the packet after all), and the caller then makes a separate call to +// copy the data it wants into position. +func (s *cryptoStream) dataToSend(pto bool, f func(off, size int64) (sent int64)) { + for { + var off, size int64 + if pto { + // On PTO, resend unacked data that fits in the probe packet. + // For simplicity, we send the range starting at s.out.start + // (which is definitely unacked, or else we would have discarded it) + // up to the next acked byte (if any). + // + // This may miss unacked data starting after that acked byte, + // but avoids resending data the peer has acked. + off = s.out.start + end := s.out.end + for _, r := range s.outacked { + if r.start > off { + end = r.start + break + } + } + size = end - s.out.start + } else if s.outunsent.numRanges() > 0 { + off = s.outunsent.min() + size = s.outunsent[0].size() + } + if size == 0 { + return + } + n := f(off, size) + if n == 0 || pto { + return + } + } +} + +// sendData fills b with data to send to the peer, starting at off, +// and marks the data as sent. The caller must have already ascertained +// that there is data to send in this region using dataToSend. +func (s *cryptoStream) sendData(off int64, b []byte) { + s.out.copy(off, b) + s.outunsent.sub(off, off+int64(len(b))) +} diff --git a/internal/quic/crypto_stream_test.go b/internal/quic/crypto_stream_test.go new file mode 100644 index 0000000000..a6c1e1b521 --- /dev/null +++ b/internal/quic/crypto_stream_test.go @@ -0,0 +1,265 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/rand" + "reflect" + "testing" +) + +func TestCryptoStreamReceive(t *testing.T) { + data := make([]byte, 1<<20) + rand.Read(data) // doesn't need to be crypto/rand, but non-deprecated and harmless + type frame struct { + start int64 + end int64 + want int + } + for _, test := range []struct { + name string + frames []frame + }{{ + name: "linear", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 1000, + end: 2000, + want: 2000, + }, { + // larger than any realistic packet can hold + start: 2000, + end: 1 << 20, + want: 1 << 20, + }}, + }, { + name: "out of order", + frames: []frame{{ + start: 1000, + end: 2000, + }, { + start: 2000, + end: 3000, + }, { + start: 0, + end: 1000, + want: 3000, + }}, + }, { + name: "resent", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 0, + end: 1000, + want: 1000, + }, { + start: 1000, + end: 2000, + want: 2000, + }, { + start: 0, + end: 1000, + want: 2000, + }, { + start: 1000, + end: 2000, + want: 2000, + }}, + }, { + name: "overlapping", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 3000, + end: 4000, + want: 1000, + }, { + start: 2000, + end: 3000, + want: 1000, + }, { + start: 1000, + end: 3000, + want: 4000, + }}, + }} { + t.Run(test.name, func(t *testing.T) { + var s cryptoStream + var got []byte + for _, f := range test.frames { + t.Logf("receive [%v,%v)", f.start, f.end) + s.handleCrypto( + f.start, + data[f.start:f.end], + func(b []byte) error { + t.Logf("got new bytes [%v,%v)", len(got), len(got)+len(b)) + got = append(got, b...) + return nil + }, + ) + if len(got) != f.want { + t.Fatalf("have bytes [0,%v), want [0,%v)", len(got), f.want) + } + for i := range got { + if got[i] != data[i] { + t.Fatalf("byte %v of received data = %v, want %v", i, got[i], data[i]) + } + } + } + }) + } +} + +func TestCryptoStreamSends(t *testing.T) { + data := make([]byte, 1<<20) + rand.Read(data) // doesn't need to be crypto/rand, but non-deprecated and harmless + type ( + sendOp i64range[int64] + ackOp i64range[int64] + lossOp i64range[int64] + ) + for _, test := range []struct { + name string + size int64 + ops []any + wantSend []i64range[int64] + wantPTOSend []i64range[int64] + }{{ + name: "writes with data remaining", + size: 4000, + ops: []any{ + sendOp{0, 1000}, + sendOp{1000, 2000}, + sendOp{2000, 3000}, + }, + wantSend: []i64range[int64]{ + {3000, 4000}, + }, + wantPTOSend: []i64range[int64]{ + {0, 4000}, + }, + }, { + name: "lost data is resent", + size: 4000, + ops: []any{ + sendOp{0, 1000}, + sendOp{1000, 2000}, + sendOp{2000, 3000}, + sendOp{3000, 4000}, + lossOp{1000, 2000}, + lossOp{3000, 4000}, + }, + wantSend: []i64range[int64]{ + {1000, 2000}, + {3000, 4000}, + }, + wantPTOSend: []i64range[int64]{ + {0, 4000}, + }, + }, { + name: "acked data at start of range", + size: 4000, + ops: []any{ + sendOp{0, 4000}, + ackOp{0, 1000}, + ackOp{1000, 2000}, + ackOp{2000, 3000}, + }, + wantSend: nil, + wantPTOSend: []i64range[int64]{ + {3000, 4000}, + }, + }, { + name: "acked data is not resent on pto", + size: 4000, + ops: []any{ + sendOp{0, 4000}, + ackOp{1000, 2000}, + }, + wantSend: nil, + wantPTOSend: []i64range[int64]{ + {0, 1000}, + }, + }, { + // This is an unusual, but possible scenario: + // Data is sent, resent, one of the two sends is acked, and the other is lost. + name: "acked and then lost data is not resent", + size: 4000, + ops: []any{ + sendOp{0, 4000}, + sendOp{1000, 2000}, // resent, no-op + ackOp{1000, 2000}, + lossOp{1000, 2000}, + }, + wantSend: nil, + wantPTOSend: []i64range[int64]{ + {0, 1000}, + }, + }, { + // The opposite of the above scenario: data is marked lost, and then acked + // before being resent. + name: "lost and then acked data is not resent", + size: 4000, + ops: []any{ + sendOp{0, 4000}, + sendOp{1000, 2000}, // resent, no-op + lossOp{1000, 2000}, + ackOp{1000, 2000}, + }, + wantSend: nil, + wantPTOSend: []i64range[int64]{ + {0, 1000}, + }, + }} { + t.Run(test.name, func(t *testing.T) { + var s cryptoStream + s.write(data[:test.size]) + for _, op := range test.ops { + switch op := op.(type) { + case sendOp: + t.Logf("send [%v,%v)", op.start, op.end) + b := make([]byte, op.end-op.start) + s.sendData(op.start, b) + case ackOp: + t.Logf("ack [%v,%v)", op.start, op.end) + s.ackOrLoss(op.start, op.end, packetAcked) + case lossOp: + t.Logf("loss [%v,%v)", op.start, op.end) + s.ackOrLoss(op.start, op.end, packetLost) + default: + t.Fatalf("unhandled type %T", op) + } + } + var gotSend []i64range[int64] + s.dataToSend(true, func(off, size int64) (wrote int64) { + gotSend = append(gotSend, i64range[int64]{off, off + size}) + return 0 + }) + if !reflect.DeepEqual(gotSend, test.wantPTOSend) { + t.Fatalf("got data to send on PTO: %v, want %v", gotSend, test.wantPTOSend) + } + gotSend = nil + s.dataToSend(false, func(off, size int64) (wrote int64) { + gotSend = append(gotSend, i64range[int64]{off, off + size}) + b := make([]byte, size) + s.sendData(off, b) + return int64(len(b)) + }) + if !reflect.DeepEqual(gotSend, test.wantSend) { + t.Fatalf("got data to send: %v, want %v", gotSend, test.wantSend) + } + }) + } +} From dd0aa3399ca64473636cc7074182c590dc9b4e31 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 17 Jul 2023 09:48:38 -0700 Subject: [PATCH 14/46] quic: tls handshake Exchange TLS handshake data in CRYPTO frames. Receive packet protection keys from the TLS layer. Discard packet protection keys as the handshake progresses. Send and receive HANDSHAKE_DONE frames (used by the server to inform the client of the handshake completing). Add a very minimal implementation of CONNECTION_CLOSE, just enough to let us write tests that trigger immediate close of connections. For golang/go#58547 Change-Id: I77496ca65bd72977565733739d563eaa2bb7d8d3 Reviewed-on: https://go-review.googlesource.com/c/net/+/510915 Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Auto-Submit: Damien Neil --- internal/quic/config.go | 20 ++ internal/quic/conn.go | 75 +++++- internal/quic/conn_loss.go | 7 +- internal/quic/conn_loss_test.go | 143 ++++++++++ internal/quic/conn_recv.go | 41 ++- internal/quic/conn_send.go | 72 ++++- internal/quic/conn_test.go | 150 ++++++++++- internal/quic/ping_test.go | 20 +- internal/quic/quic.go | 8 + internal/quic/tls.go | 134 +++++++++- internal/quic/tls_test.go | 421 ++++++++++++++++++++++++++++++ internal/quic/tlsconfig_test.go | 62 +++++ internal/quic/transport_params.go | 4 +- 13 files changed, 1105 insertions(+), 52 deletions(-) create mode 100644 internal/quic/config.go create mode 100644 internal/quic/conn_loss_test.go create mode 100644 internal/quic/tls_test.go create mode 100644 internal/quic/tlsconfig_test.go diff --git a/internal/quic/config.go b/internal/quic/config.go new file mode 100644 index 0000000000..7d1b7433af --- /dev/null +++ b/internal/quic/config.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/tls" +) + +// A Config structure configures a QUIC endpoint. +// A Config must not be modified after it has been passed to a QUIC function. +// A Config may be reused; the quic package will also not modify it. +type Config struct { + // TLSConfig is the endpoint's TLS configuration. + // It must be non-nil and include at least one certificate or else set GetCertificate. + TLSConfig *tls.Config +} diff --git a/internal/quic/conn.go b/internal/quic/conn.go index e6375e86a8..8130c549b9 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -7,6 +7,7 @@ package quic import ( + "crypto/tls" "errors" "fmt" "net/netip" @@ -19,6 +20,7 @@ import ( type Conn struct { side connSide listener connListener + config *Config testHooks connTestHooks peerAddr netip.AddrPort @@ -29,14 +31,27 @@ type Conn struct { w packetWriter acks [numberSpaceCount]ackState // indexed by number space connIDState connIDState - tlsState tlsState loss lossState + // errForPeer is set when the connection is being closed. + errForPeer error + connCloseSent [numberSpaceCount]bool + // idleTimeout is the time at which the connection will be closed due to inactivity. // https://www.rfc-editor.org/rfc/rfc9000#section-10.1 maxIdleTimeout time.Duration idleTimeout time.Time + // Packet protection keys, CRYPTO streams, and TLS state. + rkeys [numberSpaceCount]keys + wkeys [numberSpaceCount]keys + crypto [numberSpaceCount]cryptoStream + tls *tls.QUICConn + + // handshakeConfirmed is set when the handshake is confirmed. + // For server connections, it tracks sending HANDSHAKE_DONE. + handshakeConfirmed sentVal + peerAckDelayExponent int8 // -1 when unknown // Tests only: Send a PING in a specific number space. @@ -53,12 +68,14 @@ type connListener interface { // connTestHooks override conn behavior in tests. type connTestHooks interface { nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any) + handleTLSEvent(tls.QUICEvent) } -func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, l connListener, hooks connTestHooks) (*Conn, error) { +func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, config *Config, l connListener, hooks connTestHooks) (*Conn, error) { c := &Conn{ side: side, listener: l, + config: config, peerAddr: peerAddr, msgc: make(chan any, 1), donec: make(chan struct{}), @@ -88,12 +105,58 @@ func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip. const maxDatagramSize = 1200 c.loss.init(c.side, maxDatagramSize, now) - c.tlsState.init(c.side, initialConnID) + c.startTLS(now, initialConnID, transportParameters{ + initialSrcConnID: c.connIDState.srcConnID(), + ackDelayExponent: ackDelayExponent, + maxUDPPayloadSize: maxUDPPayloadSize, + maxAckDelay: maxAckDelay, + }) go c.loop(now) return c, nil } +// confirmHandshake is called when the handshake is confirmed. +// https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2 +func (c *Conn) confirmHandshake(now time.Time) { + // If handshakeConfirmed is unset, the handshake is not confirmed. + // If it is unsent, the handshake is confirmed and we need to send a HANDSHAKE_DONE. + // If it is sent, we have sent a HANDSHAKE_DONE. + // If it is received, the handshake is confirmed and we do not need to send anything. + if c.handshakeConfirmed.isSet() { + return // already confirmed + } + if c.side == serverSide { + // When the server confirms the handshake, it sends a HANDSHAKE_DONE. + c.handshakeConfirmed.setUnsent() + } else { + // The client never sends a HANDSHAKE_DONE, so we set handshakeConfirmed + // to the received state, indicating that the handshake is confirmed and we + // don't need to send anything. + c.handshakeConfirmed.setReceived() + } + c.loss.confirmHandshake() + // "An endpoint MUST discard its Handshake keys when the TLS handshake is confirmed" + // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.2-1 + c.discardKeys(now, handshakeSpace) +} + +// discardKeys discards unused packet protection keys. +// https://www.rfc-editor.org/rfc/rfc9001#section-4.9 +func (c *Conn) discardKeys(now time.Time, space numberSpace) { + c.rkeys[space].discard() + c.wkeys[space].discard() + c.loss.discardKeys(now, space) +} + +// receiveTransportParameters applies transport parameters sent by the peer. +func (c *Conn) receiveTransportParameters(p transportParameters) { + c.peerAckDelayExponent = p.ackDelayExponent + c.loss.setMaxAckDelay(p.maxAckDelay) + + // TODO: Many more transport parameters to come. +} + type timerEvent struct{} // loop is the connection main loop. @@ -104,6 +167,7 @@ type timerEvent struct{} // Other goroutines may examine or modify conn state by sending the loop funcs to execute. func (c *Conn) loop(now time.Time) { defer close(c.donec) + defer c.tls.Close() // The connection timer sends a message to the connection loop on expiry. // We need to give it an expiry when creating it, so set the initial timeout to @@ -201,8 +265,9 @@ func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { // abort terminates a connection with an error. func (c *Conn) abort(now time.Time, err error) { - // TODO: Send CONNECTION_CLOSE frames. - c.exit() + if c.errForPeer == nil { + c.errForPeer = err + } } // exit fully terminates a connection immediately. diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go index 11ed42dbb9..6cb459c33f 100644 --- a/internal/quic/conn_loss.go +++ b/internal/quic/conn_loss.go @@ -29,7 +29,7 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF for !sent.done() { switch f := sent.next(); f { default: - panic(fmt.Sprintf("BUG: unhandled lost frame type %x", f)) + panic(fmt.Sprintf("BUG: unhandled acked/lost frame type %x", f)) case frameTypeAck: // Unlike most information, loss of an ACK frame does not trigger // retransmission. ACKs are sent in response to ack-eliciting packets, @@ -41,6 +41,11 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF if fate == packetAcked { c.acks[space].handleAck(largest) } + case frameTypeCrypto: + start, end := sent.nextRange() + c.crypto[space].ackOrLoss(start, end, fate) + case frameTypeHandshakeDone: + c.handshakeConfirmed.ackOrLoss(sent.num, fate) } } } diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go new file mode 100644 index 0000000000..be4f5fb2c8 --- /dev/null +++ b/internal/quic/conn_loss_test.go @@ -0,0 +1,143 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/tls" + "testing" +) + +// Frames may be retransmitted either when the packet containing the frame is lost, or on PTO. +// lostFrameTest runs a test in both configurations. +func lostFrameTest(t *testing.T, f func(t *testing.T, pto bool)) { + t.Run("lost", func(t *testing.T) { + f(t, false) + }) + t.Run("pto", func(t *testing.T) { + f(t, true) + }) +} + +// triggerLossOrPTO causes the conn to declare the last sent packet lost, +// or advances to the PTO timer. +func (tc *testConn) triggerLossOrPTO(ptype packetType, pto bool) { + tc.t.Helper() + if pto { + if !tc.conn.loss.ptoTimerArmed { + tc.t.Fatalf("PTO timer not armed, expected it to be") + } + tc.advanceTo(tc.conn.loss.timer) + return + } + defer func(ignoreFrames map[byte]bool) { + tc.ignoreFrames = ignoreFrames + }(tc.ignoreFrames) + tc.ignoreFrames = map[byte]bool{ + frameTypeAck: true, + frameTypePadding: true, + } + // Send three packets containing PINGs, and then respond with an ACK for the + // last one. This puts the last packet before the PINGs outside the packet + // reordering threshold, and it will be declared lost. + const lossThreshold = 3 + var num packetNumber + for i := 0; i < lossThreshold; i++ { + tc.conn.ping(spaceForPacketType(ptype)) + d := tc.readDatagram() + if d == nil { + tc.t.Fatalf("conn is idle; want PING frame") + } + if d.packets[0].ptype != ptype { + tc.t.Fatalf("conn sent %v packet; want %v", d.packets[0].ptype, ptype) + } + num = d.packets[0].num + } + tc.writeFrames(ptype, debugFrameAck{ + ranges: []i64range[packetNumber]{ + {num, num + 1}, + }, + }) +} + +func TestLostCRYPTOFrame(t *testing.T) { + // "Data sent in CRYPTO frames is retransmitted [...] until all data has been acknowledged." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.1 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc := newTestConn(t, clientSide) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.triggerLossOrPTO(packetTypeInitial, pto) + tc.wantFrame("client resends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + + tc.wantFrame("client sends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) + tc.triggerLossOrPTO(packetTypeHandshake, pto) + tc.wantFrame("client resends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) + }) +} + +func TestLostHandshakeDoneFrame(t *testing.T) { + // "The HANDSHAKE_DONE frame MUST be retransmitted until it is acknowledged." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.16 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc := newTestConn(t, serverSide) + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("server sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("server sends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + + tc.wantFrame("server sends HANDSHAKE_DONE after handshake completes", + packetType1RTT, debugFrameHandshakeDone{}) + tc.wantFrame("server sends session ticket in CRYPTO frame", + packetType1RTT, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelApplication], + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("server resends HANDSHAKE_DONE", + packetType1RTT, debugFrameHandshakeDone{}) + tc.wantFrame("server resends session ticket", + packetType1RTT, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelApplication], + }) + }) +} diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index d5a3b8cb0c..7eb03e7279 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -41,12 +41,12 @@ func (c *Conn) handleDatagram(now time.Time, dgram *datagram) { } func (c *Conn) handleLongHeader(now time.Time, ptype packetType, space numberSpace, buf []byte) int { - if !c.tlsState.rkeys[space].isSet() { + if !c.rkeys[space].isSet() { return skipLongHeaderPacket(buf) } pnumMax := c.acks[space].largestSeen() - p, n := parseLongHeaderPacket(buf, c.tlsState.rkeys[space], pnumMax) + p, n := parseLongHeaderPacket(buf, c.rkeys[space], pnumMax) if n < 0 { return -1 } @@ -66,21 +66,23 @@ func (c *Conn) handleLongHeader(now time.Time, ptype packetType, space numberSpa if p.ptype == packetTypeHandshake && c.side == serverSide { c.loss.validateClientAddress() - // TODO: Discard Initial keys. + // "[...] a server MUST discard Initial keys when it first successfully + // processes a Handshake packet [...]" // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.1-2 + c.discardKeys(now, initialSpace) } return n } func (c *Conn) handle1RTT(now time.Time, buf []byte) int { - if !c.tlsState.rkeys[appDataSpace].isSet() { + if !c.rkeys[appDataSpace].isSet() { // 1-RTT packets extend to the end of the datagram, // so skip the remainder of the datagram if we can't parse this. return len(buf) } pnumMax := c.acks[appDataSpace].largestSeen() - p, n := parse1RTTPacket(buf, c.tlsState.rkeys[appDataSpace], connIDLen, pnumMax) + p, n := parse1RTTPacket(buf, c.rkeys[appDataSpace], connIDLen, pnumMax) if n < 0 { return -1 } @@ -163,7 +165,7 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, if !frameOK(c, ptype, IH_1) { return } - _, _, n = consumeCryptoFrame(payload) + n = c.handleCryptoFrame(now, space, payload) case frameTypeNewToken: if !frameOK(c, ptype, ___1) { return @@ -207,14 +209,18 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, case frameTypeConnectionCloseTransport: // CONNECTION_CLOSE is OK in all spaces. _, _, _, n = consumeConnectionCloseTransportFrame(payload) + // TODO: https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2 + c.abort(now, localTransportError(errNo)) case frameTypeConnectionCloseApplication: // CONNECTION_CLOSE is OK in all spaces. _, _, n = consumeConnectionCloseApplicationFrame(payload) + // TODO: https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2 + c.abort(now, localTransportError(errNo)) case frameTypeHandshakeDone: if !frameOK(c, ptype, ___1) { return } - n = 1 + n = c.handleHandshakeDoneFrame(now, space, payload) } if n < 0 { c.abort(now, localTransportError(errFrameEncoding)) @@ -262,3 +268,24 @@ func (c *Conn) handleAckFrame(now time.Time, space numberSpace, payload []byte) c.loss.receiveAckEnd(now, space, delay, c.handleAckOrLoss) return n } + +func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byte) int { + off, data, n := consumeCryptoFrame(payload) + err := c.handleCrypto(now, space, off, data) + if err != nil { + c.abort(now, err) + return -1 + } + return n +} + +func (c *Conn) handleHandshakeDoneFrame(now time.Time, space numberSpace, payload []byte) int { + if c.side == serverSide { + // Clients should never send HANDSHAKE_DONE. + // https://www.rfc-editor.org/rfc/rfc9000#section-19.20-4 + c.abort(now, localTransportError(errProtocolViolation)) + return -1 + } + c.confirmHandshake(now) + return 1 +} diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index 3a51ceb285..71d24e6f0f 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -7,6 +7,8 @@ package quic import ( + "crypto/tls" + "errors" "time" ) @@ -45,7 +47,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { // Initial packet. pad := false var sentInitial *sentPacket - if k := c.tlsState.wkeys[initialSpace]; k.isSet() { + if k := c.wkeys[initialSpace]; k.isSet() { pnumMaxAcked := c.acks[initialSpace].largestSeen() pnum := c.loss.nextNumber(initialSpace) p := longPacket{ @@ -62,14 +64,14 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { // Client initial packets need to be sent in a datagram padded to // at least 1200 bytes. We can't add the padding yet, however, // since we may want to coalesce additional packets with this one. - if c.side == clientSide || sentInitial.ackEliciting { + if c.side == clientSide { pad = true } } } // Handshake packet. - if k := c.tlsState.wkeys[handshakeSpace]; k.isSet() { + if k := c.wkeys[handshakeSpace]; k.isSet() { pnumMaxAcked := c.acks[handshakeSpace].largestSeen() pnum := c.loss.nextNumber(handshakeSpace) p := longPacket{ @@ -84,14 +86,16 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { if sent := c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, p); sent != nil { c.loss.packetSent(now, handshakeSpace, sent) if c.side == clientSide { - // TODO: Discard the Initial keys. - // https://www.rfc-editor.org/rfc/rfc9001.html#section-4.9.1 + // "[...] a client MUST discard Initial keys when it first + // sends a Handshake packet [...]" + // https://www.rfc-editor.org/rfc/rfc9001.html#section-4.9.1-2 + c.discardKeys(now, initialSpace) } } } // 1-RTT packet. - if k := c.tlsState.wkeys[appDataSpace]; k.isSet() { + if k := c.wkeys[appDataSpace]; k.isSet() { pnumMaxAcked := c.acks[appDataSpace].largestSeen() pnum := c.loss.nextNumber(appDataSpace) dstConnID := c.connIDState.dstConnID() @@ -133,7 +137,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { sentInitial.inFlight = true } } - if k := c.tlsState.wkeys[initialSpace]; k.isSet() { + if k := c.wkeys[initialSpace]; k.isSet() { c.loss.packetSent(now, initialSpace, sentInitial) } } @@ -143,6 +147,26 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { } func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, limit ccLimit) { + if c.errForPeer != nil { + // This is the bare minimum required to send a CONNECTION_CLOSE frame + // when closing a connection immediately, for example in response to a + // protocol error. + // + // This does not handle the closing and draining states + // (https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2), + // but it's enough to let us write tests that result in a CONNECTION_CLOSE, + // and have those tests still pass when we finish implementing + // connection shutdown. + // + // TODO: Finish implementing connection shutdown. + if !c.connCloseSent[space] { + c.exited = true + c.appendConnectionCloseFrame(c.errForPeer) + c.connCloseSent[space] = true + } + return + } + shouldSendAck := c.acks[space].shouldSendAck(now) if limit != ccOK { // ACKs are not limited by congestion control. @@ -185,6 +209,21 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, // TODO: Add all the other frames we can send. + // HANDSHAKE_DONE + if c.handshakeConfirmed.shouldSendPTO(pto) { + if !c.w.appendHandshakeDoneFrame() { + return + } + c.handshakeConfirmed.setSent(pnum) + } + + // CRYPTO + c.crypto[space].dataToSend(pto, func(off, size int64) int64 { + b, _ := c.w.appendCryptoFrame(off, int(size)) + c.crypto[space].sendData(off, b) + return int64(len(b)) + }) + // Test-only PING frames. if space == c.testSendPingSpace && c.testSendPing.shouldSendPTO(pto) { if !c.w.appendPingFrame() { @@ -253,3 +292,22 @@ func (c *Conn) appendAckFrame(now time.Time, space numberSpace) bool { d := unscaledAckDelayFromDuration(delay, ackDelayExponent) return c.w.appendAckFrame(seen, d) } + +func (c *Conn) appendConnectionCloseFrame(err error) { + // TODO: Send application errors. + switch e := err.(type) { + case localTransportError: + c.w.appendConnectionCloseTransportFrame(transportError(e), 0, "") + default: + // TLS alerts are sent using error codes [0x0100,0x01ff). + // https://www.rfc-editor.org/rfc/rfc9000#section-20.1-2.36.1 + var alert tls.AlertError + if errors.As(err, &alert) { + // tls.AlertError is a uint8, so this can't exceed 0x01ff. + code := errTLSBase + transportError(alert) + c.w.appendConnectionCloseTransportFrame(code, 0, "") + return + } + c.w.appendConnectionCloseTransportFrame(errInternal, 0, "") + } +} diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index fda1d4b869..511fb97a0a 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -7,6 +7,9 @@ package quic import ( + "bytes" + "context" + "crypto/tls" "errors" "fmt" "math" @@ -111,8 +114,22 @@ type testConn struct { // we use Handshake keys to encrypt the packet. // The client only acquires those keys when it processes // the Initial packet. - rkeys [numberSpaceCount]keys // for packets sent to the conn - wkeys [numberSpaceCount]keys // for packets sent by the conn + rkeys [numberSpaceCount]keyData // for packets sent to the conn + wkeys [numberSpaceCount]keyData // for packets sent by the conn + + // testConn uses a test hook to snoop on the conn's TLS events. + // CRYPTO data produced by the conn's QUICConn is placed in + // cryptoDataOut. + // + // The peerTLSConn is is a QUICConn representing the peer. + // CRYPTO data produced by the conn is written to peerTLSConn, + // and data produced by peerTLSConn is placed in cryptoDataIn. + cryptoDataOut map[tls.QUICEncryptionLevel][]byte + cryptoDataIn map[tls.QUICEncryptionLevel][]byte + peerTLSConn *tls.QUICConn + + localConnID []byte + transientConnID []byte // Information about the conn's (fake) peer. peerConnID []byte // source conn id of peer's packets @@ -129,12 +146,18 @@ type testConn struct { ignoreFrames map[byte]bool } +type keyData struct { + suite uint16 + secret []byte + k keys +} + // newTestConn creates a Conn for testing. // // The Conn's event loop is controlled by the test, // allowing test code to access Conn state directly // by first ensuring the loop goroutine is idle. -func newTestConn(t *testing.T, side connSide) *testConn { +func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { t.Helper() tc := &testConn{ t: t, @@ -143,9 +166,24 @@ func newTestConn(t *testing.T, side connSide) *testConn { ignoreFrames: map[byte]bool{ frameTypePadding: true, // ignore PADDING by default }, + cryptoDataOut: make(map[tls.QUICEncryptionLevel][]byte), + cryptoDataIn: make(map[tls.QUICEncryptionLevel][]byte), } t.Cleanup(tc.cleanup) + config := &Config{ + TLSConfig: newTestTLSConfig(side), + } + peerProvidedParams := defaultTransportParameters() + for _, o := range opts { + switch o := o.(type) { + case func(*tls.Config): + o(config.TLSConfig) + default: + t.Fatalf("unknown newTestConn option %T", o) + } + } + var initialConnID []byte if side == serverSide { // The initial connection ID for the server is chosen by the client. @@ -157,11 +195,21 @@ func newTestConn(t *testing.T, side connSide) *testConn { } } + peerQUICConfig := &tls.QUICConfig{TLSConfig: newTestTLSConfig(side.peer())} + if side == clientSide { + tc.peerTLSConn = tls.QUICServer(peerQUICConfig) + } else { + tc.peerTLSConn = tls.QUICClient(peerQUICConfig) + } + tc.peerTLSConn.SetTransportParameters(marshalTransportParameters(peerProvidedParams)) + tc.peerTLSConn.Start(context.Background()) + conn, err := newConn( tc.now, side, initialConnID, netip.MustParseAddrPort("127.0.0.1:443"), + config, (*testConnListener)(tc), (*testConnHooks)(tc)) if err != nil { @@ -169,8 +217,16 @@ func newTestConn(t *testing.T, side connSide) *testConn { } tc.conn = conn - tc.wkeys[initialSpace] = conn.tlsState.wkeys[initialSpace] - tc.rkeys[initialSpace] = conn.tlsState.rkeys[initialSpace] + if side == serverSide { + tc.transientConnID = tc.conn.connIDState.local[0].cid + tc.localConnID = tc.conn.connIDState.local[1].cid + } else if side == clientSide { + tc.transientConnID = tc.conn.connIDState.remote[0].cid + tc.localConnID = tc.conn.connIDState.local[0].cid + } + + tc.wkeys[initialSpace].k = conn.wkeys[initialSpace] + tc.rkeys[initialSpace].k = conn.rkeys[initialSpace] tc.wait() return tc @@ -385,7 +441,7 @@ func (tc *testConn) wantFrame(expectation string, wantType packetType, want debu tc.t.Fatalf("%v:\nconnection is idle\nwant %v frame: %v", expectation, wantType, want) } if gotType != wantType { - tc.t.Fatalf("%v:\ngot %v packet, want %v", expectation, wantType, want) + tc.t.Fatalf("%v:\ngot %v packet, want %v\ngot frame: %v", expectation, gotType, wantType, got) } if !reflect.DeepEqual(got, want) { tc.t.Fatalf("%v:\ngot frame: %v\nwant frame: %v", expectation, got, want) @@ -426,12 +482,12 @@ func (tc *testConn) encodeTestPacket(p *testPacket) []byte { f.write(&w) } space := spaceForPacketType(p.ptype) - if !tc.rkeys[space].isSet() { + if !tc.rkeys[space].k.isSet() { tc.t.Fatalf("sending packet with no %v keys available", space) return nil } if p.ptype != packetType1RTT { - w.finishProtectedLongHeaderPacket(pnumMaxAcked, tc.rkeys[space], longPacket{ + w.finishProtectedLongHeaderPacket(pnumMaxAcked, tc.rkeys[space].k, longPacket{ ptype: p.ptype, version: p.version, num: p.num, @@ -439,7 +495,7 @@ func (tc *testConn) encodeTestPacket(p *testPacket) []byte { srcConnID: p.srcConnID, }) } else { - w.finish1RTTPacket(p.num, pnumMaxAcked, p.dstConnID, tc.rkeys[space]) + w.finish1RTTPacket(p.num, pnumMaxAcked, p.dstConnID, tc.rkeys[space].k) } return w.datagram() } @@ -455,12 +511,12 @@ func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { } ptype := getPacketType(buf) space := spaceForPacketType(ptype) - if !tc.wkeys[space].isSet() { + if !tc.wkeys[space].k.isSet() { tc.t.Fatalf("no keys for space %v, packet type %v", space, ptype) } if isLongHeader(buf[0]) { var pnumMax packetNumber // TODO: Track packet numbers. - p, n := parseLongHeaderPacket(buf, tc.wkeys[space], pnumMax) + p, n := parseLongHeaderPacket(buf, tc.wkeys[space].k, pnumMax) if n < 0 { tc.t.Fatalf("packet parse error") } @@ -479,11 +535,10 @@ func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { buf = buf[n:] } else { var pnumMax packetNumber // TODO: Track packet numbers. - p, n := parse1RTTPacket(buf, tc.wkeys[space], len(tc.peerConnID), pnumMax) + p, n := parse1RTTPacket(buf, tc.wkeys[space].k, len(tc.peerConnID), pnumMax) if n < 0 { tc.t.Fatalf("packet parse error") } - dstConnID, _ := dstConnIDForDatagram(buf) frames, err := tc.parseTestFrames(p.payload) if err != nil { tc.t.Fatal(err) @@ -491,7 +546,7 @@ func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { d.packets = append(d.packets, &testPacket{ ptype: packetType1RTT, num: p.num, - dstConnID: dstConnID, + dstConnID: buf[1:][:len(tc.peerConnID)], frames: frames, }) buf = buf[n:] @@ -535,6 +590,73 @@ func spaceForPacketType(ptype packetType) numberSpace { // testConnHooks implements connTestHooks. type testConnHooks testConn +// handleTLSEvent processes TLS events generated by +// the connection under test's tls.QUICConn. +// +// We maintain a second tls.QUICConn representing the peer, +// and feed the TLS handshake data into it. +// +// We stash TLS handshake data from both sides in the testConn, +// where it can be used by tests. +// +// We snoop packet protection keys out of the tls.QUICConns, +// and verify that both sides of the connection are getting +// matching keys. +func (tc *testConnHooks) handleTLSEvent(e tls.QUICEvent) { + setKey := func(keys *[numberSpaceCount]keyData, e tls.QUICEvent) { + k, err := newKeys(e.Suite, e.Data) + if err != nil { + tc.t.Errorf("newKeys: %v", err) + return + } + var space numberSpace + switch { + case e.Level == tls.QUICEncryptionLevelHandshake: + space = handshakeSpace + case e.Level == tls.QUICEncryptionLevelApplication: + space = appDataSpace + default: + tc.t.Errorf("unexpected encryption level %v", e.Level) + return + } + s := "read" + if keys == &tc.wkeys { + s = "write" + } + if keys[space].k.isSet() { + if keys[space].suite != e.Suite || !bytes.Equal(keys[space].secret, e.Data) { + tc.t.Errorf("%v key mismatch for level for level %v", s, e.Level) + } + return + } + keys[space].suite = e.Suite + keys[space].secret = append([]byte{}, e.Data...) + keys[space].k = k + } + switch e.Kind { + case tls.QUICSetReadSecret: + setKey(&tc.rkeys, e) + case tls.QUICSetWriteSecret: + setKey(&tc.wkeys, e) + case tls.QUICWriteData: + tc.cryptoDataOut[e.Level] = append(tc.cryptoDataOut[e.Level], e.Data...) + tc.peerTLSConn.HandleData(e.Level, e.Data) + } + for { + e := tc.peerTLSConn.NextEvent() + switch e.Kind { + case tls.QUICNoEvent: + return + case tls.QUICSetReadSecret: + setKey(&tc.wkeys, e) + case tls.QUICSetWriteSecret: + setKey(&tc.rkeys, e) + case tls.QUICWriteData: + tc.cryptoDataIn[e.Level] = append(tc.cryptoDataIn[e.Level], e.Data...) + } + } +} + // nextMessage is called by the Conn's event loop to request its next event. func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.Time, m any) { tc.timer = timer diff --git a/internal/quic/ping_test.go b/internal/quic/ping_test.go index 4a732ed543..c370aaf1d8 100644 --- a/internal/quic/ping_test.go +++ b/internal/quic/ping_test.go @@ -10,26 +10,34 @@ import "testing" func TestPing(t *testing.T) { tc := newTestConn(t, clientSide) - tc.conn.ping(initialSpace) + tc.handshake() + + tc.conn.ping(appDataSpace) tc.wantFrame("connection should send a PING frame", - packetTypeInitial, debugFramePing{}) + packetType1RTT, debugFramePing{}) tc.advanceToTimer() tc.wantFrame("on PTO, connection should send another PING frame", - packetTypeInitial, debugFramePing{}) + packetType1RTT, debugFramePing{}) tc.wantIdle("after sending PTO probe, no additional frames to send") } func TestAck(t *testing.T) { tc := newTestConn(t, serverSide) - tc.writeFrames(packetTypeInitial, + tc.handshake() + + // Send two packets, to trigger an immediate ACK. + tc.writeFrames(packetType1RTT, + debugFramePing{}, + ) + tc.writeFrames(packetType1RTT, debugFramePing{}, ) tc.wantFrame("connection should respond to ack-eliciting packet with an ACK frame", - packetTypeInitial, + packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{0, 1}}, + ranges: []i64range[packetNumber]{{0, 3}}, }, ) } diff --git a/internal/quic/quic.go b/internal/quic/quic.go index 9df7f7e2b1..a61c91f16b 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -64,6 +64,14 @@ func (s connSide) String() string { } } +func (s connSide) peer() connSide { + if s == clientSide { + return serverSide + } else { + return clientSide + } +} + // A numberSpace is the context in which a packet number applies. // https://www.rfc-editor.org/rfc/rfc9000.html#section-12.3-7 type numberSpace byte diff --git a/internal/quic/tls.go b/internal/quic/tls.go index 1cdb727e24..4306a3e46d 100644 --- a/internal/quic/tls.go +++ b/internal/quic/tls.go @@ -6,18 +6,132 @@ package quic -// tlsState encapsulates interactions with TLS. -type tlsState struct { - // Encryption keys indexed by number space. - rkeys [numberSpaceCount]keys - wkeys [numberSpaceCount]keys -} +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "time" +) -func (s *tlsState) init(side connSide, initialConnID []byte) { +// startTLS starts the TLS handshake. +func (c *Conn) startTLS(now time.Time, initialConnID []byte, params transportParameters) error { clientKeys, serverKeys := initialKeys(initialConnID) - if side == clientSide { - s.wkeys[initialSpace], s.rkeys[initialSpace] = clientKeys, serverKeys + if c.side == clientSide { + c.wkeys[initialSpace], c.rkeys[initialSpace] = clientKeys, serverKeys } else { - s.wkeys[initialSpace], s.rkeys[initialSpace] = serverKeys, clientKeys + c.wkeys[initialSpace], c.rkeys[initialSpace] = serverKeys, clientKeys + } + + qconfig := &tls.QUICConfig{TLSConfig: c.config.TLSConfig} + if c.side == clientSide { + c.tls = tls.QUICClient(qconfig) + } else { + c.tls = tls.QUICServer(qconfig) + } + c.tls.SetTransportParameters(marshalTransportParameters(params)) + // TODO: We don't need or want a context for cancelation here, + // but users can use a context to plumb values through to hooks defined + // in the tls.Config. Pass through a context. + if err := c.tls.Start(context.TODO()); err != nil { + return err + } + return c.handleTLSEvents(now) +} + +func (c *Conn) handleTLSEvents(now time.Time) error { + for { + e := c.tls.NextEvent() + if c.testHooks != nil { + c.testHooks.handleTLSEvent(e) + } + switch e.Kind { + case tls.QUICNoEvent: + return nil + case tls.QUICSetReadSecret: + space, k, err := tlsKey(e) + if err != nil { + return err + } + c.rkeys[space] = k + case tls.QUICSetWriteSecret: + space, k, err := tlsKey(e) + if err != nil { + return err + } + c.wkeys[space] = k + case tls.QUICWriteData: + space, err := spaceForLevel(e.Level) + if err != nil { + return err + } + c.crypto[space].write(e.Data) + case tls.QUICHandshakeDone: + if c.side == serverSide { + // "[...] the TLS handshake is considered confirmed + // at the server when the handshake completes." + // https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2-1 + c.confirmHandshake(now) + if !c.config.TLSConfig.SessionTicketsDisabled { + if err := c.tls.SendSessionTicket(false); err != nil { + return err + } + } + } + case tls.QUICTransportParameters: + params, err := unmarshalTransportParams(e.Data) + if err != nil { + return err + } + c.receiveTransportParameters(params) + } + } +} + +// tlsKey returns the keys in a QUICSetReadSecret or QUICSetWriteSecret event. +func tlsKey(e tls.QUICEvent) (numberSpace, keys, error) { + space, err := spaceForLevel(e.Level) + if err != nil { + return 0, keys{}, err + } + k, err := newKeys(e.Suite, e.Data) + if err != nil { + return 0, keys{}, err + } + return space, k, nil +} + +func spaceForLevel(level tls.QUICEncryptionLevel) (numberSpace, error) { + switch level { + case tls.QUICEncryptionLevelInitial: + return initialSpace, nil + case tls.QUICEncryptionLevelHandshake: + return handshakeSpace, nil + case tls.QUICEncryptionLevelApplication: + return appDataSpace, nil + default: + return 0, fmt.Errorf("quic: internal error: write handshake data at level %v", level) + } +} + +// handleCrypto processes data received in a CRYPTO frame. +func (c *Conn) handleCrypto(now time.Time, space numberSpace, off int64, data []byte) error { + var level tls.QUICEncryptionLevel + switch space { + case initialSpace: + level = tls.QUICEncryptionLevelInitial + case handshakeSpace: + level = tls.QUICEncryptionLevelHandshake + case appDataSpace: + level = tls.QUICEncryptionLevelApplication + default: + return errors.New("quic: internal error: received CRYPTO frame in unexpected number space") + } + err := c.crypto[space].handleCrypto(off, data, func(b []byte) error { + return c.tls.HandleData(level, b) + }) + if err != nil { + return err } + return c.handleTLSEvents(now) } diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go new file mode 100644 index 0000000000..df07820086 --- /dev/null +++ b/internal/quic/tls_test.go @@ -0,0 +1,421 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "reflect" + "testing" + "time" +) + +// handshake executes the handshake. +func (tc *testConn) handshake() { + tc.t.Helper() + defer func(saved map[byte]bool) { + tc.ignoreFrames = saved + }(tc.ignoreFrames) + tc.ignoreFrames = nil + t := tc.t + dgrams := handshakeDatagrams(tc) + i := 0 + for { + if i == len(dgrams)-1 { + if tc.conn.side == clientSide { + want := tc.now.Add(maxAckDelay - timerGranularity) + if !tc.timer.Equal(want) { + t.Fatalf("want timer = %v (max_ack_delay), got %v", want, tc.timer) + } + if got := tc.readDatagram(); got != nil { + t.Fatalf("client unexpectedly sent: %v", got) + } + } + tc.advance(maxAckDelay) + } + + // Check that we're sending exactly the data we expect. + // Any variation from the norm here should be intentional. + got := tc.readDatagram() + var want *testDatagram + if !(tc.conn.side == serverSide && i == 0) && i < len(dgrams) { + want = dgrams[i] + fillCryptoFrames(want, tc.cryptoDataOut) + i++ + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("dgram %v:\ngot %v\n\nwant %v", i, got, want) + } + if i >= len(dgrams) { + break + } + + fillCryptoFrames(dgrams[i], tc.cryptoDataIn) + tc.write(dgrams[i]) + i++ + } +} + +func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { + var ( + clientConnID []byte + serverConnID []byte + ) + if tc.conn.side == clientSide { + clientConnID = tc.localConnID + serverConnID = tc.peerConnID + } else { + clientConnID = tc.peerConnID + serverConnID = tc.localConnID + } + return []*testDatagram{{ + // Client Initial + packets: []*testPacket{{ + ptype: packetTypeInitial, + num: 0, + version: 1, + srcConnID: clientConnID, + dstConnID: tc.transientConnID, + frames: []debugFrame{ + debugFrameCrypto{}, + }, + }}, + paddedSize: 1200, + }, { + // Server Initial + Handshake + packets: []*testPacket{{ + ptype: packetTypeInitial, + num: 0, + version: 1, + srcConnID: serverConnID, + dstConnID: clientConnID, + frames: []debugFrame{ + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, + debugFrameCrypto{}, + }, + }, { + ptype: packetTypeHandshake, + num: 0, + version: 1, + srcConnID: serverConnID, + dstConnID: clientConnID, + frames: []debugFrame{ + debugFrameCrypto{}, + }, + }}, + }, { + // Client Handshake + packets: []*testPacket{{ + ptype: packetTypeInitial, + num: 1, + version: 1, + srcConnID: clientConnID, + dstConnID: serverConnID, + frames: []debugFrame{ + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, + }, + }, { + ptype: packetTypeHandshake, + num: 0, + version: 1, + srcConnID: clientConnID, + dstConnID: serverConnID, + frames: []debugFrame{ + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, + debugFrameCrypto{}, + }, + }}, + paddedSize: 1200, + }, { + // Server HANDSHAKE_DONE and session ticket + packets: []*testPacket{{ + ptype: packetType1RTT, + num: 0, + dstConnID: clientConnID, + frames: []debugFrame{ + debugFrameHandshakeDone{}, + debugFrameCrypto{}, + }, + }}, + }, { + // Client ack (after max_ack_delay) + packets: []*testPacket{{ + ptype: packetType1RTT, + num: 0, + dstConnID: serverConnID, + frames: []debugFrame{ + debugFrameAck{ + ackDelay: unscaledAckDelayFromDuration( + maxAckDelay, ackDelayExponent), + ranges: []i64range[packetNumber]{{0, 1}}, + }, + }, + }}, + }} +} + +func fillCryptoFrames(d *testDatagram, data map[tls.QUICEncryptionLevel][]byte) { + for _, p := range d.packets { + var level tls.QUICEncryptionLevel + switch p.ptype { + case packetTypeInitial: + level = tls.QUICEncryptionLevelInitial + case packetTypeHandshake: + level = tls.QUICEncryptionLevelHandshake + case packetType1RTT: + level = tls.QUICEncryptionLevelApplication + default: + continue + } + for i := range p.frames { + c, ok := p.frames[i].(debugFrameCrypto) + if !ok { + continue + } + c.data = data[level] + data[level] = nil + p.frames[i] = c + } + } +} + +func TestConnClientHandshake(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.handshake() + tc.advance(1 * time.Second) + tc.wantIdle("no packets should be sent by an idle conn after the handshake") +} + +func TestConnServerHandshake(t *testing.T) { + tc := newTestConn(t, serverSide) + tc.handshake() + tc.advance(1 * time.Second) + tc.wantIdle("no packets should be sent by an idle conn after the handshake") +} + +func TestConnKeysDiscardedClient(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrame("client sends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) + + // The client discards Initial keys after sending a Handshake packet. + tc.writeFrames(packetTypeInitial, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantIdle("client has discarded Initial keys, cannot read CONNECTION_CLOSE") + + // The client discards Handshake keys after receiving a HANDSHAKE_DONE frame. + tc.writeFrames(packetType1RTT, + debugFrameHandshakeDone{}) + tc.writeFrames(packetTypeHandshake, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantIdle("client has discarded Handshake keys, cannot read CONNECTION_CLOSE") + + tc.writeFrames(packetType1RTT, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantFrame("client closes connection after 1-RTT CONNECTION_CLOSE", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errNo, + }) +} + +func TestConnKeysDiscardedServer(t *testing.T) { + tc := newTestConn(t, serverSide, func(c *tls.Config) { + c.SessionTicketsDisabled = true + }) + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("server sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("server sends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) + + // The server discards Initial keys after receiving a Handshake packet. + // The Handshake packet contains only the start of the client's CRYPTO flight here, + // to avoids completing the handshake yet. + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][:1], + }) + tc.writeFrames(packetTypeInitial, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantIdle("server has discarded Initial keys, cannot read CONNECTION_CLOSE") + + // The server discards Handshake keys after sending a HANDSHAKE_DONE frame. + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + off: 1, + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][1:], + }) + tc.wantFrame("server sends HANDSHAKE_DONE after handshake completes", + packetType1RTT, debugFrameHandshakeDone{}) + tc.writeFrames(packetTypeHandshake, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantIdle("server has discarded Handshake keys, cannot read CONNECTION_CLOSE") + + tc.writeFrames(packetType1RTT, + debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantFrame("server closes connection after 1-RTT CONNECTION_CLOSE", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errNo, + }) +} + +func TestConnInvalidCryptoData(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + + // Render the server's response invalid. + // + // The client closes the connection with CRYPTO_ERROR. + // + // Changing the first byte will change the TLS message type, + // so we can reasonably assume that this is an unexpected_message alert (10). + tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][0] ^= 0x1 + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrame("client closes connection due to TLS handshake error", + packetTypeInitial, debugFrameConnectionCloseTransport{ + code: errTLSBase + 10, + }) +} + +func TestConnInvalidPeerCertificate(t *testing.T) { + tc := newTestConn(t, clientSide, func(c *tls.Config) { + c.VerifyPeerCertificate = func([][]byte, [][]*x509.Certificate) error { + return errors.New("I will not buy this certificate. It is scratched.") + } + }) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrame("client closes connection due to rejecting server certificate", + packetTypeInitial, debugFrameConnectionCloseTransport{ + code: errTLSBase + 42, // 42: bad_certificate + }) +} + +func TestConnHandshakeDoneSentToServer(t *testing.T) { + tc := newTestConn(t, serverSide) + tc.handshake() + + tc.writeFrames(packetType1RTT, + debugFrameHandshakeDone{}) + tc.wantFrame("server closes connection when client sends a HANDSHAKE_DONE frame", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) +} + +func TestConnCryptoDataOutOfOrder(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.wantIdle("client is idle, server Handshake flight has not arrived") + + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + off: 15, + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][15:], + }) + tc.wantIdle("client is idle, server Handshake flight is not complete") + + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + off: 1, + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][1:20], + }) + tc.wantIdle("client is idle, server Handshake flight is still not complete") + + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][0:1], + }) + tc.wantFrame("client sends Handshake CRYPTO frame", + packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], + }) +} + +func TestConnCryptoBufferSizeExceeded(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.ignoreFrame(frameTypeAck) + + tc.wantFrame("client sends Initial CRYPTO frame", + packetTypeInitial, debugFrameCrypto{ + data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + off: cryptoBufferSize, + data: []byte{0}, + }) + tc.wantFrame("client closes connection after server exceeds CRYPTO buffer", + packetTypeInitial, debugFrameConnectionCloseTransport{ + code: errCryptoBufferExceeded, + }) +} diff --git a/internal/quic/tlsconfig_test.go b/internal/quic/tlsconfig_test.go new file mode 100644 index 0000000000..47bfb05983 --- /dev/null +++ b/internal/quic/tlsconfig_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "crypto/tls" + "strings" +) + +func newTestTLSConfig(side connSide) *tls.Config { + config := &tls.Config{ + InsecureSkipVerify: true, + CipherSuites: []uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }, + MinVersion: tls.VersionTLS13, + } + if side == serverSide { + config.Certificates = []tls.Certificate{testCert} + } + return config +} + +var testCert = func() tls.Certificate { + cert, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + panic(err) + } + return cert +}() + +// localhostCert is a PEM-encoded TLS cert with SAN IPs +// "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT. +// generated from src/crypto/tls: +// go run generate_cert.go --ecdsa-curve P256 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +var localhostCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBrDCCAVKgAwIBAgIPCvPhO+Hfv+NW76kWxULUMAoGCCqGSM49BAMCMBIxEDAO +BgNVBAoTB0FjbWUgQ28wIBcNNzAwMTAxMDAwMDAwWhgPMjA4NDAxMjkxNjAwMDBa +MBIxEDAOBgNVBAoTB0FjbWUgQ28wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARh +WRF8p8X9scgW7JjqAwI9nYV8jtkdhqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGms +PyfMPe5Jrha/LmjgR1G9o4GIMIGFMA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAK +BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOJri/wLQxq6oC +Y6ZImms/STbTljAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAA +AAAAAAAAAAAAATAKBggqhkjOPQQDAgNIADBFAiBUguxsW6TGhixBAdORmVNnkx40 +HjkKwncMSDbUaeL9jQIhAJwQ8zV9JpQvYpsiDuMmqCuW35XXil3cQ6Drz82c+fvE +-----END CERTIFICATE-----`) + +// localhostKey is the private key for localhostCert. +var localhostKey = []byte(testingKey(`-----BEGIN TESTING KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgY1B1eL/Bbwf/MDcs +rnvvWhFNr1aGmJJR59PdCN9lVVqhRANCAARhWRF8p8X9scgW7JjqAwI9nYV8jtkd +hqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGmsPyfMPe5Jrha/LmjgR1G9 +-----END TESTING KEY-----`)) + +// testingKey helps keep security scanners from getting excited about a private key in this file. +func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/internal/quic/transport_params.go b/internal/quic/transport_params.go index 416bfb8679..89ea69fb97 100644 --- a/internal/quic/transport_params.go +++ b/internal/quic/transport_params.go @@ -25,7 +25,7 @@ type transportParameters struct { initialMaxStreamDataUni int64 initialMaxStreamsBidi int64 initialMaxStreamsUni int64 - ackDelayExponent uint8 + ackDelayExponent int8 maxAckDelay time.Duration disableActiveMigration bool preferredAddrV4 netip.AddrPort @@ -220,7 +220,7 @@ func unmarshalTransportParams(params []byte) (transportParameters, error) { if v > 20 { return p, localTransportError(errTransportParameter) } - p.ackDelayExponent = uint8(v) + p.ackDelayExponent = int8(v) case paramMaxAckDelay: var v uint64 v, n = consumeVarint(val) From 08001ccbedb025e618cc203ed6d33973d5839a2f Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 21 Jul 2023 13:21:18 -0700 Subject: [PATCH 15/46] quic: debug logging of packets Log every packet sent and received to stdout when GODEBUG=quiclogpackets=1. For golang/go#58547 Change-Id: I904336017ea646ad6459557b44702bfe4c732ba9 Reviewed-on: https://go-review.googlesource.com/c/net/+/513439 Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- internal/quic/conn.go | 4 +++ internal/quic/conn_recv.go | 6 ++++ internal/quic/conn_send.go | 9 +++++ internal/quic/log.go | 69 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 internal/quic/log.go diff --git a/internal/quic/conn.go b/internal/quic/conn.go index 8130c549b9..ff03bd7f87 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -116,6 +116,10 @@ func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip. return c, nil } +func (c *Conn) String() string { + return fmt.Sprintf("quic.Conn(%v,->%v)", c.side, c.peerAddr) +} + // confirmHandshake is called when the handshake is confirmed. // https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2 func (c *Conn) confirmHandshake(now time.Time) { diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 7eb03e7279..3baa79a0cc 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -60,6 +60,9 @@ func (c *Conn) handleLongHeader(now time.Time, ptype packetType, space numberSpa return n } + if logPackets { + logInboundLongPacket(c, p) + } c.connIDState.handlePacket(c.side, p.ptype, p.srcConnID) ackEliciting := c.handleFrames(now, ptype, space, p.payload) c.acks[space].receive(now, space, p.num, ackEliciting) @@ -96,6 +99,9 @@ func (c *Conn) handle1RTT(now time.Time, buf []byte) int { return len(buf) } + if logPackets { + logInboundShortPacket(c, p) + } ackEliciting := c.handleFrames(now, packetType1RTT, appDataSpace, p.payload) c.acks[appDataSpace].receive(now, appDataSpace, p.num, ackEliciting) return len(buf) diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index 71d24e6f0f..62c9b62ec7 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -59,6 +59,9 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { } c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) c.appendFrames(now, initialSpace, pnum, limit) + if logPackets { + logSentPacket(c, packetTypeInitial, pnum, p.srcConnID, p.dstConnID, c.w.payload()) + } sentInitial = c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, p) if sentInitial != nil { // Client initial packets need to be sent in a datagram padded to @@ -83,6 +86,9 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { } c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) c.appendFrames(now, handshakeSpace, pnum, limit) + if logPackets { + logSentPacket(c, packetTypeHandshake, pnum, p.srcConnID, p.dstConnID, c.w.payload()) + } if sent := c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, p); sent != nil { c.loss.packetSent(now, handshakeSpace, sent) if c.side == clientSide { @@ -108,6 +114,9 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { c.w.appendPaddingTo(minimumClientInitialDatagramSize) pad = false } + if logPackets { + logSentPacket(c, packetType1RTT, pnum, nil, dstConnID, c.w.payload()) + } if sent := c.w.finish1RTTPacket(pnum, pnumMaxAcked, dstConnID, k); sent != nil { c.loss.packetSent(now, appDataSpace, sent) } diff --git a/internal/quic/log.go b/internal/quic/log.go new file mode 100644 index 0000000000..d7248343b0 --- /dev/null +++ b/internal/quic/log.go @@ -0,0 +1,69 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "fmt" + "os" + "strings" +) + +var logPackets bool + +// Parse GODEBUG settings. +// +// GODEBUG=quiclogpackets=1 -- log every packet sent and received. +func init() { + s := os.Getenv("GODEBUG") + for len(s) > 0 { + var opt string + opt, s, _ = strings.Cut(s, ",") + switch opt { + case "quiclogpackets=1": + logPackets = true + } + } +} + +func logInboundLongPacket(c *Conn, p longPacket) { + if !logPackets { + return + } + prefix := c.String() + fmt.Printf("%v recv %v %v\n", prefix, p.ptype, p.num) + logFrames(prefix+" <- ", p.payload) +} + +func logInboundShortPacket(c *Conn, p shortPacket) { + if !logPackets { + return + } + prefix := c.String() + fmt.Printf("%v recv 1-RTT %v\n", prefix, p.num) + logFrames(prefix+" <- ", p.payload) +} + +func logSentPacket(c *Conn, ptype packetType, pnum packetNumber, src, dst, payload []byte) { + if !logPackets || len(payload) == 0 { + return + } + prefix := c.String() + fmt.Printf("%v send %v %v\n", prefix, ptype, pnum) + logFrames(prefix+" -> ", payload) +} + +func logFrames(prefix string, payload []byte) { + for len(payload) > 0 { + f, n := parseDebugFrame(payload) + if n < 0 { + fmt.Printf("%vBAD DATA\n", prefix) + break + } + payload = payload[n:] + fmt.Printf("%v%v\n", prefix, f) + } +} From bd8ac9ecf8d3c89c10b91a3b40cb5f536a99635b Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 20 Jul 2023 16:45:15 -0700 Subject: [PATCH 16/46] quic: fill out connection id handling Add support for sending and receiving NEW_CONNECTION_ID and RETIRE_CONNECTION_ID frames. Keep the peer supplied with up to 4 connection IDs. Retire connection IDs as required by the peer. Support connection IDs provided in the preferred_address transport parameter. RFC 9000, Section 5.1. For golang/go#58547 Change-Id: I015a69b94c40a6396e9f117a92c88acaf83c594e Reviewed-on: https://go-review.googlesource.com/c/net/+/513440 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam --- internal/quic/conn.go | 32 ++- internal/quic/conn_id.go | 238 +++++++++++++++++- internal/quic/conn_id_test.go | 422 +++++++++++++++++++++++++++++++- internal/quic/conn_loss.go | 6 + internal/quic/conn_loss_test.go | 65 +++++ internal/quic/conn_recv.go | 29 ++- internal/quic/conn_send.go | 19 +- internal/quic/conn_test.go | 74 ++++-- internal/quic/frame_debug.go | 5 +- internal/quic/packet_parser.go | 4 +- internal/quic/packet_writer.go | 7 +- internal/quic/ping_test.go | 2 +- internal/quic/quic.go | 10 + internal/quic/tls.go | 4 +- internal/quic/tls_test.go | 156 ++++++++++-- 15 files changed, 998 insertions(+), 75 deletions(-) diff --git a/internal/quic/conn.go b/internal/quic/conn.go index ff03bd7f87..77ecea0d62 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -69,6 +69,7 @@ type connListener interface { type connTestHooks interface { nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any) handleTLSEvent(tls.QUICEvent) + newConnID(seq int64) ([]byte, error) } func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, config *Config, l connListener, hooks connTestHooks) (*Conn, error) { @@ -90,12 +91,12 @@ func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip. c.msgc = make(chan any, 1) if c.side == clientSide { - if err := c.connIDState.initClient(newRandomConnID); err != nil { + if err := c.connIDState.initClient(c.newConnIDFunc()); err != nil { return nil, err } - initialConnID = c.connIDState.dstConnID() + initialConnID, _ = c.connIDState.dstConnID() } else { - if err := c.connIDState.initServer(newRandomConnID, initialConnID); err != nil { + if err := c.connIDState.initServer(c.newConnIDFunc(), initialConnID); err != nil { return nil, err } } @@ -154,11 +155,27 @@ func (c *Conn) discardKeys(now time.Time, space numberSpace) { } // receiveTransportParameters applies transport parameters sent by the peer. -func (c *Conn) receiveTransportParameters(p transportParameters) { +func (c *Conn) receiveTransportParameters(p transportParameters) error { c.peerAckDelayExponent = p.ackDelayExponent c.loss.setMaxAckDelay(p.maxAckDelay) + if err := c.connIDState.setPeerActiveConnIDLimit(p.activeConnIDLimit, c.newConnIDFunc()); err != nil { + return err + } + if p.preferredAddrConnID != nil { + var ( + seq int64 = 1 // sequence number of this conn id is 1 + retirePriorTo int64 = 0 // retire nothing + resetToken [16]byte + ) + copy(resetToken[:], p.preferredAddrResetToken) + if err := c.connIDState.handleNewConnID(seq, retirePriorTo, p.preferredAddrConnID, resetToken); err != nil { + return err + } + } // TODO: Many more transport parameters to come. + + return nil } type timerEvent struct{} @@ -295,3 +312,10 @@ func firstTime(a, b time.Time) time.Time { return b } } + +func (c *Conn) newConnIDFunc() newConnIDFunc { + if c.testHooks != nil { + return c.testHooks.newConnID + } + return newRandomConnID +} diff --git a/internal/quic/conn_id.go b/internal/quic/conn_id.go index deea70d326..561dea2c14 100644 --- a/internal/quic/conn_id.go +++ b/internal/quic/conn_id.go @@ -7,6 +7,7 @@ package quic import ( + "bytes" "crypto/rand" ) @@ -18,8 +19,16 @@ type connIDState struct { // Local IDs are usually issued by us, and remote IDs by the peer. // The exception is the transient destination connection ID sent in // a client's Initial packets, which is chosen by the client. + // + // These are []connID rather than []*connID to minimize allocations. local []connID remote []connID + + nextLocalSeq int64 + retireRemotePriorTo int64 // largest Retire Prior To value sent by the peer + peerActiveConnIDLimit int64 // peer's active_connection_id_limit transport parameter + + needSend bool } // A connID is a connection ID and associated metadata. @@ -32,12 +41,24 @@ type connID struct { // // For the transient destination ID in a client's Initial packet, this is -1. seq int64 + + // retired is set when the connection ID is retired. + retired bool + + // send is set when the connection ID's state needs to be sent to the peer. + // + // For local IDs, this indicates a new ID that should be sent + // in a NEW_CONNECTION_ID frame. + // + // For remote IDs, this indicates a retired ID that should be sent + // in a RETIRE_CONNECTION_ID frame. + send sentVal } func (s *connIDState) initClient(newID newConnIDFunc) error { // Client chooses its initial connection ID, and sends it // in the Source Connection ID field of the first Initial packet. - locid, err := newID() + locid, err := newID(0) if err != nil { return err } @@ -45,10 +66,11 @@ func (s *connIDState) initClient(newID newConnIDFunc) error { seq: 0, cid: locid, }) + s.nextLocalSeq = 1 // Client chooses an initial, transient connection ID for the server, // and sends it in the Destination Connection ID field of the first Initial packet. - remid, err := newID() + remid, err := newID(-1) if err != nil { return err } @@ -70,7 +92,7 @@ func (s *connIDState) initServer(newID newConnIDFunc, dstConnID []byte) error { // Server chooses a connection ID, and sends it in the Source Connection ID of // the response to the clent. - locid, err := newID() + locid, err := newID(0) if err != nil { return err } @@ -78,6 +100,7 @@ func (s *connIDState) initServer(newID newConnIDFunc, dstConnID []byte) error { seq: 0, cid: locid, }) + s.nextLocalSeq = 1 return nil } @@ -91,8 +114,44 @@ func (s *connIDState) srcConnID() []byte { } // dstConnID is the Destination Connection ID to use in a sent packet. -func (s *connIDState) dstConnID() []byte { - return s.remote[0].cid +func (s *connIDState) dstConnID() (cid []byte, ok bool) { + for i := range s.remote { + if !s.remote[i].retired { + return s.remote[i].cid, true + } + } + return nil, false +} + +// setPeerActiveConnIDLimit sets the active_connection_id_limit +// transport parameter received from the peer. +func (s *connIDState) setPeerActiveConnIDLimit(lim int64, newID newConnIDFunc) error { + s.peerActiveConnIDLimit = lim + return s.issueLocalIDs(newID) +} + +func (s *connIDState) issueLocalIDs(newID newConnIDFunc) error { + toIssue := min(int(s.peerActiveConnIDLimit), maxPeerActiveConnIDLimit) + for i := range s.local { + if s.local[i].seq != -1 && !s.local[i].retired { + toIssue-- + } + } + for toIssue > 0 { + cid, err := newID(s.nextLocalSeq) + if err != nil { + return err + } + s.local = append(s.local, connID{ + seq: s.nextLocalSeq, + cid: cid, + }) + s.local[len(s.local)-1].send.setUnsent() + s.nextLocalSeq++ + s.needSend = true + toIssue-- + } + return nil } // handlePacket updates the connection ID state during the handshake @@ -128,19 +187,184 @@ func (s *connIDState) handlePacket(side connSide, ptype packetType, srcConnID [] } } +func (s *connIDState) handleNewConnID(seq, retire int64, cid []byte, resetToken [16]byte) error { + if len(s.remote[0].cid) == 0 { + // "An endpoint that is sending packets with a zero-length + // Destination Connection ID MUST treat receipt of a NEW_CONNECTION_ID + // frame as a connection error of type PROTOCOL_VIOLATION." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.15-6 + return localTransportError(errProtocolViolation) + } + + if retire > s.retireRemotePriorTo { + s.retireRemotePriorTo = retire + } + + have := false // do we already have this connection ID? + active := 0 + for i := range s.remote { + rcid := &s.remote[i] + if !rcid.retired && rcid.seq < s.retireRemotePriorTo { + s.retireRemote(rcid) + } + if !rcid.retired { + active++ + } + if rcid.seq == seq { + if !bytes.Equal(rcid.cid, cid) { + return localTransportError(errProtocolViolation) + } + have = true // yes, we've seen this sequence number + } + } + + if !have { + // This is a new connection ID that we have not seen before. + // + // We could take steps to keep the list of remote connection IDs + // sorted by sequence number, but there's no particular need + // so we don't bother. + s.remote = append(s.remote, connID{ + seq: seq, + cid: cloneBytes(cid), + }) + if seq < s.retireRemotePriorTo { + // This ID was already retired by a previous NEW_CONNECTION_ID frame. + s.retireRemote(&s.remote[len(s.remote)-1]) + } else { + active++ + } + } + + if active > activeConnIDLimit { + // Retired connection IDs (including newly-retired ones) do not count + // against the limit. + // https://www.rfc-editor.org/rfc/rfc9000.html#section-5.1.1-5 + return localTransportError(errConnectionIDLimit) + } + + // "An endpoint SHOULD limit the number of connection IDs it has retired locally + // for which RETIRE_CONNECTION_ID frames have not yet been acknowledged." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-6 + // + // Set a limit of four times the active_connection_id_limit for + // the total number of remote connection IDs we keep state for locally. + if len(s.remote) > 4*activeConnIDLimit { + return localTransportError(errConnectionIDLimit) + } + + return nil +} + +// retireRemote marks a remote connection ID as retired. +func (s *connIDState) retireRemote(rcid *connID) { + rcid.retired = true + rcid.send.setUnsent() + s.needSend = true +} + +func (s *connIDState) handleRetireConnID(seq int64, newID newConnIDFunc) error { + if seq >= s.nextLocalSeq { + return localTransportError(errProtocolViolation) + } + for i := range s.local { + if s.local[i].seq == seq { + s.local = append(s.local[:i], s.local[i+1:]...) + break + } + } + s.issueLocalIDs(newID) + return nil +} + +func (s *connIDState) ackOrLossNewConnectionID(pnum packetNumber, seq int64, fate packetFate) { + for i := range s.local { + if s.local[i].seq != seq { + continue + } + s.local[i].send.ackOrLoss(pnum, fate) + if fate != packetAcked { + s.needSend = true + } + return + } +} + +func (s *connIDState) ackOrLossRetireConnectionID(pnum packetNumber, seq int64, fate packetFate) { + for i := 0; i < len(s.remote); i++ { + if s.remote[i].seq != seq { + continue + } + if fate == packetAcked { + // We have retired this connection ID, and the peer has acked. + // Discard its state completely. + s.remote = append(s.remote[:i], s.remote[i+1:]...) + } else { + // RETIRE_CONNECTION_ID frame was lost, mark for retransmission. + s.needSend = true + s.remote[i].send.ackOrLoss(pnum, fate) + } + return + } +} + +// appendFrames appends NEW_CONNECTION_ID and RETIRE_CONNECTION_ID frames +// to the current packet. +// +// It returns true if no more frames need appending, +// false if not everything fit in the current packet. +func (s *connIDState) appendFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + if !s.needSend && !pto { + // Fast path: We don't need to send anything. + return true + } + retireBefore := int64(0) + if s.local[0].seq != -1 { + retireBefore = s.local[0].seq + } + for i := range s.local { + if !s.local[i].send.shouldSendPTO(pto) { + continue + } + if !w.appendNewConnectionIDFrame( + s.local[i].seq, + retireBefore, + s.local[i].cid, + [16]byte{}, // TODO: stateless reset token + ) { + return false + } + s.local[i].send.setSent(pnum) + } + for i := range s.remote { + if !s.remote[i].send.shouldSendPTO(pto) { + continue + } + if !w.appendRetireConnectionIDFrame(s.remote[i].seq) { + return false + } + s.remote[i].send.setSent(pnum) + } + s.needSend = false + return true +} + func cloneBytes(b []byte) []byte { n := make([]byte, len(b)) copy(n, b) return n } -type newConnIDFunc func() ([]byte, error) +type newConnIDFunc func(seq int64) ([]byte, error) -func newRandomConnID() ([]byte, error) { +func newRandomConnID(_ int64) ([]byte, error) { // It is not necessary for connection IDs to be cryptographically secure, // but it doesn't hurt. id := make([]byte, connIDLen) if _, err := rand.Read(id); err != nil { + // TODO: Surface this error as a metric or log event or something. + // rand.Read really shouldn't ever fail, but if it does, we should + // have a way to inform the user. return nil, err } return id, nil diff --git a/internal/quic/conn_id_test.go b/internal/quic/conn_id_test.go index 7c31e9d560..74905578dc 100644 --- a/internal/quic/conn_id_test.go +++ b/internal/quic/conn_id_test.go @@ -7,7 +7,10 @@ package quic import ( + "bytes" + "crypto/tls" "fmt" + "net/netip" "reflect" "testing" ) @@ -22,14 +25,16 @@ func TestConnIDClientHandshake(t *testing.T) { if got, want := string(s.srcConnID()), "local-1"; got != want { t.Errorf("after initClient: srcConnID = %q, want %q", got, want) } - if got, want := string(s.dstConnID()), "local-2"; got != want { + dstConnID, _ := s.dstConnID() + if got, want := string(dstConnID), "local-2"; got != want { t.Errorf("after initClient: dstConnID = %q, want %q", got, want) } // The server's first Initial packet provides the client with a // non-transient remote connection ID. s.handlePacket(clientSide, packetTypeInitial, []byte("remote-1")) - if got, want := string(s.dstConnID()), "remote-1"; got != want { + dstConnID, _ = s.dstConnID() + if got, want := string(dstConnID), "remote-1"; got != want { t.Errorf("after receiving Initial: dstConnID = %q, want %q", got, want) } @@ -59,7 +64,8 @@ func TestConnIDServerHandshake(t *testing.T) { if got, want := string(s.srcConnID()), "local-1"; got != want { t.Errorf("after initClient: srcConnID = %q, want %q", got, want) } - if got, want := string(s.dstConnID()), "remote-1"; got != want { + dstConnID, _ := s.dstConnID() + if got, want := string(dstConnID), "remote-1"; got != want { t.Errorf("after initClient: dstConnID = %q, want %q", got, want) } @@ -95,15 +101,421 @@ func TestConnIDServerHandshake(t *testing.T) { func newConnIDSequence() newConnIDFunc { var n uint64 - return func() ([]byte, error) { + return func(_ int64) ([]byte, error) { n++ return []byte(fmt.Sprintf("local-%v", n)), nil } } func TestNewRandomConnID(t *testing.T) { - cid, err := newRandomConnID() + cid, err := newRandomConnID(0) if len(cid) != connIDLen || err != nil { t.Fatalf("newConnID() = %x, %v; want %v bytes", cid, connIDLen, err) } } + +func TestConnIDPeerRequestsManyIDs(t *testing.T) { + // "An endpoint SHOULD ensure that its peer has a sufficient number + // of available and unused connection IDs." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4 + // + // "An endpoint MAY limit the total number of connection IDs + // issued for each connection [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-6 + // + // Peer requests 100 connection IDs. + // We give them 4 in total. + tc := newTestConn(t, serverSide, func(p *transportParameters) { + p.activeConnIDLimit = 100 + }) + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeCrypto) + + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("provide additional connection ID 1", + packetType1RTT, debugFrameNewConnectionID{ + seq: 1, + connID: testLocalConnID(1), + }) + tc.wantFrame("provide additional connection ID 2", + packetType1RTT, debugFrameNewConnectionID{ + seq: 2, + connID: testLocalConnID(2), + }) + tc.wantFrame("provide additional connection ID 3", + packetType1RTT, debugFrameNewConnectionID{ + seq: 3, + connID: testLocalConnID(3), + }) + tc.wantIdle("connection ID limit reached, no more to provide") +} + +func TestConnIDPeerProvidesTooManyIDs(t *testing.T) { + // "An endpoint MUST NOT provide more connection IDs than the peer's limit." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4 + tc := newTestConn(t, serverSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + connID: testLocalConnID(2), + }) + tc.wantFrame("peer provided 3 connection IDs, our limit is 2", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errConnectionIDLimit, + }) +} + +func TestConnIDPeerTemporarilyExceedsActiveConnIDLimit(t *testing.T) { + // "An endpoint MAY send connection IDs that temporarily exceed a peer's limit + // if the NEW_CONNECTION_ID frame also requires the retirement of any excess [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4 + tc := newTestConn(t, serverSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + retirePriorTo: 2, + seq: 2, + connID: testPeerConnID(2), + }, debugFrameNewConnectionID{ + retirePriorTo: 2, + seq: 3, + connID: testPeerConnID(3), + }) + tc.wantFrame("peer requested we retire conn id 0", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + tc.wantFrame("peer requested we retire conn id 1", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 1, + }) +} + +func TestConnIDPeerRetiresConnID(t *testing.T) { + // "An endpoint SHOULD supply a new connection ID when the peer retires a connection ID." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-6 + for _, side := range []connSide{ + clientSide, + serverSide, + } { + t.Run(side.String(), func(t *testing.T) { + tc := newTestConn(t, side) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameRetireConnectionID{ + seq: 0, + }) + tc.wantFrame("provide replacement connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testLocalConnID(2), + }) + }) + } +} + +func TestConnIDPeerWithZeroLengthConnIDSendsNewConnectionID(t *testing.T) { + // An endpoint that selects a zero-length connection ID during the handshake + // cannot issue a new connection ID." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-8 + tc := newTestConn(t, clientSide, func(c *tls.Config) { + c.SessionTicketsDisabled = true + }) + tc.peerConnID = []byte{} + tc.ignoreFrame(frameTypeAck) + tc.uncheckedHandshake() + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 1, + connID: testPeerConnID(1), + }) + tc.wantFrame("invalid NEW_CONNECTION_ID: previous conn id is zero-length", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) +} + +func TestConnIDPeerRequestsRetirement(t *testing.T) { + // "Upon receipt of an increased Retire Prior To field, the peer MUST + // stop using the corresponding connection IDs and retire them with + // RETIRE_CONNECTION_ID frames [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-5 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testPeerConnID(2), + }) + tc.wantFrame("peer asked for conn id 0 to be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + if got, want := tc.sentFramePacket.dstConnID, testPeerConnID(1); !bytes.Equal(got, want) { + t.Fatalf("used destination conn id {%x}, want {%x}", got, want) + } +} + +func TestConnIDPeerDoesNotAcknowledgeRetirement(t *testing.T) { + // "An endpoint SHOULD limit the number of connection IDs it has retired locally + // for which RETIRE_CONNECTION_ID frames have not yet been acknowledged." + // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-6 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeRetireConnectionID) + + // Send a number of NEW_CONNECTION_ID frames, each retiring an old one. + for seq := int64(0); seq < 7; seq++ { + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: seq + 2, + retirePriorTo: seq + 1, + connID: testPeerConnID(seq + 2), + }) + // We're ignoring the RETIRE_CONNECTION_ID frames. + } + tc.wantFrame("number of retired, unacked conn ids is too large", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errConnectionIDLimit, + }) +} + +func TestConnIDRepeatedNewConnectionIDFrame(t *testing.T) { + // "Receipt of the same [NEW_CONNECTION_ID] frame multiple times + // MUST NOT be treated as a connection error. + // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-7 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + for i := 0; i < 4; i++ { + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testPeerConnID(2), + }) + } + tc.wantFrame("peer asked for conn id to be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + tc.wantIdle("repeated NEW_CONNECTION_ID frames are not an error") +} + +func TestConnIDForSequenceNumberChanges(t *testing.T) { + // "[...] if a sequence number is used for different connection IDs, + // the endpoint MAY treat that receipt as a connection error + // of type PROTOCOL_VIOLATION." + // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-8 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeRetireConnectionID) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testPeerConnID(2), + }) + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testPeerConnID(3), + }) + tc.wantFrame("connection ID for sequence 0 has changed", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) +} + +func TestConnIDRetirePriorToAfterNewConnID(t *testing.T) { + // "Receiving a value in the Retire Prior To field that is greater than + // that in the Sequence Number field MUST be treated as a connection error + // of type FRAME_ENCODING_ERROR. + // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-9 + tc := newTestConn(t, serverSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + retirePriorTo: 3, + seq: 2, + connID: testPeerConnID(2), + }) + tc.wantFrame("invalid NEW_CONNECTION_ID: retired the new conn id", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errFrameEncoding, + }) +} + +func TestConnIDAlreadyRetired(t *testing.T) { + // "An endpoint that receives a NEW_CONNECTION_ID frame with a + // sequence number smaller than the Retire Prior To field of a + // previously received NEW_CONNECTION_ID frame MUST send a + // corresponding RETIRE_CONNECTION_ID frame [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-11 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 4, + retirePriorTo: 3, + connID: testPeerConnID(4), + }) + tc.wantFrame("peer asked for conn id to be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + tc.wantFrame("peer asked for conn id to be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 1, + }) + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 0, + connID: testPeerConnID(2), + }) + tc.wantFrame("NEW_CONNECTION_ID was for an already-retired ID", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 2, + }) +} + +func TestConnIDRepeatedRetireConnectionIDFrame(t *testing.T) { + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + for i := 0; i < 4; i++ { + tc.writeFrames(packetType1RTT, + debugFrameRetireConnectionID{ + seq: 0, + }) + } + tc.wantFrame("issue new conn id after peer retires one", + packetType1RTT, debugFrameNewConnectionID{ + retirePriorTo: 1, + seq: 2, + connID: testLocalConnID(2), + }) + tc.wantIdle("repeated RETIRE_CONNECTION_ID frames are not an error") +} + +func TestConnIDRetiredUnsent(t *testing.T) { + // "Receipt of a RETIRE_CONNECTION_ID frame containing a sequence number + // greater than any previously sent to the peer MUST be treated as a + // connection error of type PROTOCOL_VIOLATION." + // https://www.rfc-editor.org/rfc/rfc9000#section-19.16-7 + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameRetireConnectionID{ + seq: 2, + }) + tc.wantFrame("invalid NEW_CONNECTION_ID: previous conn id is zero-length", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) +} + +func TestConnIDUsePreferredAddressConnID(t *testing.T) { + // Peer gives us a connection ID in the preferred address transport parameter. + // We don't use the preferred address at this time, but we should use the + // connection ID. (It isn't tied to any specific address.) + // + // This test will probably need updating if/when we start using the preferred address. + cid := testPeerConnID(10) + tc := newTestConn(t, serverSide, func(p *transportParameters) { + p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0") + p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0") + p.preferredAddrConnID = cid + p.preferredAddrResetToken = make([]byte, 16) + }) + tc.uncheckedHandshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: []byte{0xff}, + }) + tc.wantFrame("peer asked for conn id 0 to be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + if got, want := tc.sentFramePacket.dstConnID, cid; !bytes.Equal(got, want) { + t.Fatalf("used destination conn id {%x}, want {%x} from preferred address transport parameter", got, want) + } +} + +func TestConnIDPeerProvidesPreferredAddrAndTooManyConnIDs(t *testing.T) { + // Peer gives us more conn ids than our advertised limit, + // including a conn id in the preferred address transport parameter. + cid := testPeerConnID(10) + tc := newTestConn(t, serverSide, func(p *transportParameters) { + p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0") + p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0") + p.preferredAddrConnID = cid + p.preferredAddrResetToken = make([]byte, 16) + }) + tc.uncheckedHandshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 0, + connID: testPeerConnID(2), + }) + tc.wantFrame("peer provided 3 connection IDs, our limit is 2", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errConnectionIDLimit, + }) +} + +func TestConnIDPeerWithZeroLengthIDProvidesPreferredAddr(t *testing.T) { + // Peer gives us more conn ids than our advertised limit, + // including a conn id in the preferred address transport parameter. + tc := newTestConn(t, serverSide, func(p *transportParameters) { + p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0") + p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0") + p.preferredAddrConnID = testPeerConnID(1) + p.preferredAddrResetToken = make([]byte, 16) + }) + tc.peerConnID = []byte{} + + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.wantFrame("peer with zero-length connection ID tried to provide another in transport parameters", + packetTypeInitial, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) +} diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go index 6cb459c33f..57570d0861 100644 --- a/internal/quic/conn_loss.go +++ b/internal/quic/conn_loss.go @@ -44,6 +44,12 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF case frameTypeCrypto: start, end := sent.nextRange() c.crypto[space].ackOrLoss(start, end, fate) + case frameTypeNewConnectionID: + seq := int64(sent.nextInt()) + c.connIDState.ackOrLossNewConnectionID(sent.num, seq, fate) + case frameTypeRetireConnectionID: + seq := int64(sent.nextInt()) + c.connIDState.ackOrLossRetireConnectionID(sent.num, seq, fate) case frameTypeHandshakeDone: c.handshakeConfirmed.ackOrLoss(sent.num, fate) } diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index be4f5fb2c8..021c86c876 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -93,6 +93,11 @@ func TestLostCRYPTOFrame(t *testing.T) { packetTypeHandshake, debugFrameCrypto{ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], }) + tc.wantFrame("client provides server with an additional connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 1, + connID: testLocalConnID(1), + }) tc.triggerLossOrPTO(packetTypeHandshake, pto) tc.wantFrame("client resends Handshake CRYPTO frame", packetTypeHandshake, debugFrameCrypto{ @@ -101,6 +106,61 @@ func TestLostCRYPTOFrame(t *testing.T) { }) } +func TestLostNewConnectionIDFrame(t *testing.T) { + // "New connection IDs are [...] retransmitted if the packet containing them is lost." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.13 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc := newTestConn(t, serverSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameRetireConnectionID{ + seq: 1, + }) + tc.wantFrame("provide a new connection ID after peer retires old one", + packetType1RTT, debugFrameNewConnectionID{ + seq: 2, + connID: testLocalConnID(2), + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resend new connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 2, + connID: testLocalConnID(2), + }) + }) +} + +func TestLostRetireConnectionIDFrame(t *testing.T) { + // "[...] retired connection IDs are [...] retransmitted + // if the packet containing them is lost." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.13 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc := newTestConn(t, clientSide) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + tc.writeFrames(packetType1RTT, + debugFrameNewConnectionID{ + seq: 2, + retirePriorTo: 1, + connID: testPeerConnID(2), + }) + tc.wantFrame("peer requested connection id be retired", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resend RETIRE_CONNECTION_ID", + packetType1RTT, debugFrameRetireConnectionID{ + seq: 0, + }) + }) +} + func TestLostHandshakeDoneFrame(t *testing.T) { // "The HANDSHAKE_DONE frame MUST be retransmitted until it is acknowledged." // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.16 @@ -120,6 +180,11 @@ func TestLostHandshakeDoneFrame(t *testing.T) { packetTypeHandshake, debugFrameCrypto{ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], }) + tc.wantFrame("server provides an additional connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 1, + connID: testLocalConnID(1), + }) tc.writeFrames(packetTypeHandshake, debugFrameCrypto{ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 3baa79a0cc..7992a619f4 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -211,7 +211,12 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, if !frameOK(c, ptype, __01) { return } - _, _, _, _, n = consumeNewConnectionIDFrame(payload) + n = c.handleNewConnectionIDFrame(now, space, payload) + case frameTypeRetireConnectionID: + if !frameOK(c, ptype, __01) { + return + } + n = c.handleRetireConnectionIDFrame(now, space, payload) case frameTypeConnectionCloseTransport: // CONNECTION_CLOSE is OK in all spaces. _, _, _, n = consumeConnectionCloseTransportFrame(payload) @@ -285,6 +290,28 @@ func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byt return n } +func (c *Conn) handleNewConnectionIDFrame(now time.Time, space numberSpace, payload []byte) int { + seq, retire, connID, resetToken, n := consumeNewConnectionIDFrame(payload) + if n < 0 { + return -1 + } + if err := c.connIDState.handleNewConnID(seq, retire, connID, resetToken); err != nil { + c.abort(now, err) + } + return n +} + +func (c *Conn) handleRetireConnectionIDFrame(now time.Time, space numberSpace, payload []byte) int { + seq, n := consumeRetireConnectionIDFrame(payload) + if n < 0 { + return -1 + } + if err := c.connIDState.handleRetireConnID(seq, c.newConnIDFunc()); err != nil { + c.abort(now, err) + } + return n +} + func (c *Conn) handleHandshakeDoneFrame(now time.Time, space numberSpace, payload []byte) int { if c.side == serverSide { // Clients should never send HANDSHAKE_DONE. diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index 62c9b62ec7..d410548a98 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -44,6 +44,13 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { // Prepare to write a datagram of at most maxSendSize bytes. c.w.reset(c.loss.maxSendSize()) + dstConnID, ok := c.connIDState.dstConnID() + if !ok { + // It is currently not possible for us to end up without a connection ID, + // but handle the case anyway. + return time.Time{} + } + // Initial packet. pad := false var sentInitial *sentPacket @@ -54,7 +61,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { ptype: packetTypeInitial, version: 1, num: pnum, - dstConnID: c.connIDState.dstConnID(), + dstConnID: dstConnID, srcConnID: c.connIDState.srcConnID(), } c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) @@ -81,7 +88,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { ptype: packetTypeHandshake, version: 1, num: pnum, - dstConnID: c.connIDState.dstConnID(), + dstConnID: dstConnID, srcConnID: c.connIDState.srcConnID(), } c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p) @@ -104,7 +111,6 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { if k := c.wkeys[appDataSpace]; k.isSet() { pnumMaxAcked := c.acks[appDataSpace].largestSeen() pnum := c.loss.nextNumber(appDataSpace) - dstConnID := c.connIDState.dstConnID() c.w.start1RTTPacket(pnum, pnumMaxAcked, dstConnID) c.appendFrames(now, appDataSpace, pnum, limit) if pad && len(c.w.payload()) > 0 { @@ -233,6 +239,13 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, return int64(len(b)) }) + // NEW_CONNECTION_ID, RETIRE_CONNECTION_ID + if space == appDataSpace { + if !c.connIDState.appendFrames(&c.w, pnum, pto) { + return + } + } + // Test-only PING frames. if space == c.testSendPingSpace && c.testSendPing.shouldSendPTO(pto) { if !c.w.appendPingFrame() { diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 511fb97a0a..317ca8f81f 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -128,19 +128,16 @@ type testConn struct { cryptoDataIn map[tls.QUICEncryptionLevel][]byte peerTLSConn *tls.QUICConn - localConnID []byte - transientConnID []byte - // Information about the conn's (fake) peer. peerConnID []byte // source conn id of peer's packets peerNextPacketNum [numberSpaceCount]packetNumber // next packet number to use // Datagrams, packets, and frames sent by the conn, // but not yet processed by the test. - sentDatagrams [][]byte - sentPackets []*testPacket - sentFrames []debugFrame - sentFramePacketType packetType + sentDatagrams [][]byte + sentPackets []*testPacket + sentFrames []debugFrame + sentFramePacket *testPacket // Frame types to ignore in tests. ignoreFrames map[byte]bool @@ -162,7 +159,7 @@ func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { tc := &testConn{ t: t, now: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), - peerConnID: []byte{0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5}, + peerConnID: testPeerConnID(0), ignoreFrames: map[byte]bool{ frameTypePadding: true, // ignore PADDING by default }, @@ -179,6 +176,8 @@ func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { switch o := o.(type) { case func(*tls.Config): o(config.TLSConfig) + case func(p *transportParameters): + o(&peerProvidedParams) default: t.Fatalf("unknown newTestConn option %T", o) } @@ -189,7 +188,7 @@ func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { // The initial connection ID for the server is chosen by the client. // When creating a server-side connection, pick a random connection ID here. var err error - initialConnID, err = newRandomConnID() + initialConnID, err = newRandomConnID(0) if err != nil { tc.t.Fatal(err) } @@ -217,14 +216,6 @@ func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { } tc.conn = conn - if side == serverSide { - tc.transientConnID = tc.conn.connIDState.local[0].cid - tc.localConnID = tc.conn.connIDState.local[1].cid - } else if side == clientSide { - tc.transientConnID = tc.conn.connIDState.remote[0].cid - tc.localConnID = tc.conn.connIDState.local[0].cid - } - tc.wkeys[initialSpace].k = conn.wkeys[initialSpace] tc.rkeys[initialSpace].k = conn.rkeys[initialSpace] @@ -326,7 +317,11 @@ func (tc *testConn) write(d *testDatagram) { if p.num >= tc.peerNextPacketNum[space] { tc.peerNextPacketNum[space] = p.num + 1 } - buf = append(buf, tc.encodeTestPacket(p)...) + pad := 0 + if p.ptype == packetType1RTT { + pad = d.paddedSize + } + buf = append(buf, tc.encodeTestPacket(p, pad)...) } for len(buf) < d.paddedSize { buf = append(buf, 0) @@ -407,12 +402,12 @@ func (tc *testConn) readFrame() (debugFrame, packetType) { if p == nil { return nil, packetTypeInvalid } - tc.sentFramePacketType = p.ptype + tc.sentFramePacket = p tc.sentFrames = p.frames } f := tc.sentFrames[0] tc.sentFrames = tc.sentFrames[1:] - return f, tc.sentFramePacketType + return f, tc.sentFramePacket.ptype } // wantDatagram indicates that we expect the Conn to send a datagram. @@ -462,7 +457,7 @@ func (tc *testConn) wantIdle(expectation string) { } } -func (tc *testConn) encodeTestPacket(p *testPacket) []byte { +func (tc *testConn) encodeTestPacket(p *testPacket, pad int) []byte { tc.t.Helper() var w packetWriter w.reset(1200) @@ -486,6 +481,7 @@ func (tc *testConn) encodeTestPacket(p *testPacket) []byte { tc.t.Fatalf("sending packet with no %v keys available", space) return nil } + w.appendPaddingTo(pad) if p.ptype != packetType1RTT { w.finishProtectedLongHeaderPacket(pnumMaxAcked, tc.rkeys[space].k, longPacket{ ptype: p.ptype, @@ -504,6 +500,7 @@ func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { tc.t.Helper() bufSize := len(buf) d := &testDatagram{} + size := len(buf) for len(buf) > 0 { if buf[0] == 0 { d.paddedSize = bufSize @@ -552,6 +549,20 @@ func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram { buf = buf[n:] } } + // This is rather hackish: If the last frame in the last packet + // in the datagram is PADDING, then remove it and record + // the padded size in the testDatagram.paddedSize. + // + // This makes it easier to write a test that expects a datagram + // padded to 1200 bytes. + if len(d.packets) > 0 && len(d.packets[len(d.packets)-1].frames) > 0 { + p := d.packets[len(d.packets)-1] + f := p.frames[len(p.frames)-1] + if _, ok := f.(debugFramePadding); ok { + p.frames = p.frames[:len(p.frames)-1] + d.paddedSize = size + } + } return d } @@ -686,6 +697,27 @@ func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.T return tc.now, m } +func (tc *testConnHooks) newConnID(seq int64) ([]byte, error) { + return testLocalConnID(seq), nil +} + +// testLocalConnID returns the connection ID with a given sequence number +// used by a Conn under test. +func testLocalConnID(seq int64) []byte { + cid := make([]byte, connIDLen) + copy(cid, []byte{0xc0, 0xff, 0xee}) + cid[len(cid)-1] = byte(seq) + return cid +} + +// testPeerConnID returns the connection ID with a given sequence number +// used by the fake peer of a Conn under test. +func testPeerConnID(seq int64) []byte { + // Use a different length than we choose for our own conn ids, + // to help catch any bad assumptions. + return []byte{0xbe, 0xee, 0xff, byte(seq)} +} + // testConnListener implements connListener. type testConnListener testConn diff --git a/internal/quic/frame_debug.go b/internal/quic/frame_debug.go index 3009a04507..7a5aee57b1 100644 --- a/internal/quic/frame_debug.go +++ b/internal/quic/frame_debug.go @@ -386,10 +386,7 @@ func (f debugFrameNewConnectionID) write(w *packetWriter) bool { // debugFrameRetireConnectionID is a NEW_CONNECTION_ID frame. type debugFrameRetireConnectionID struct { - seq uint64 - retirePriorTo uint64 - connID []byte - token [16]byte + seq int64 } func parseDebugFrameRetireConnectionID(b []byte) (f debugFrameRetireConnectionID, n int) { diff --git a/internal/quic/packet_parser.go b/internal/quic/packet_parser.go index c22f031038..0520078974 100644 --- a/internal/quic/packet_parser.go +++ b/internal/quic/packet_parser.go @@ -454,10 +454,10 @@ func consumeNewConnectionIDFrame(b []byte) (seq, retire int64, connID []byte, re return seq, retire, connID, resetToken, n } -func consumeRetireConnectionIDFrame(b []byte) (seq uint64, n int) { +func consumeRetireConnectionIDFrame(b []byte) (seq int64, n int) { n = 1 var nn int - seq, nn = consumeVarint(b[n:]) + seq, nn = consumeVarintInt64(b[n:]) if nn < 0 { return 0, -1 } diff --git a/internal/quic/packet_writer.go b/internal/quic/packet_writer.go index 6c4c452cdd..a80b4711ee 100644 --- a/internal/quic/packet_writer.go +++ b/internal/quic/packet_writer.go @@ -482,13 +482,14 @@ func (w *packetWriter) appendNewConnectionIDFrame(seq, retirePriorTo int64, conn return true } -func (w *packetWriter) appendRetireConnectionIDFrame(seq uint64) (added bool) { - if w.avail() < 1+sizeVarint(seq) { +func (w *packetWriter) appendRetireConnectionIDFrame(seq int64) (added bool) { + if w.avail() < 1+sizeVarint(uint64(seq)) { return false } w.b = append(w.b, frameTypeRetireConnectionID) - w.b = appendVarint(w.b, seq) + w.b = appendVarint(w.b, uint64(seq)) w.sent.appendAckElicitingFrame(frameTypeRetireConnectionID) + w.sent.appendInt(uint64(seq)) return true } diff --git a/internal/quic/ping_test.go b/internal/quic/ping_test.go index c370aaf1d8..a8fdf2567e 100644 --- a/internal/quic/ping_test.go +++ b/internal/quic/ping_test.go @@ -37,7 +37,7 @@ func TestAck(t *testing.T) { tc.wantFrame("connection should respond to ack-eliciting packet with an ACK frame", packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{0, 3}}, + ranges: []i64range[packetNumber]{{0, 4}}, }, ) } diff --git a/internal/quic/quic.go b/internal/quic/quic.go index a61c91f16b..84ce2bda13 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -35,6 +35,16 @@ const ( ackDelayExponent = 3 // ack_delay_exponent maxAckDelay = 25 * time.Millisecond // max_ack_delay + + // The active_conn_id_limit transport parameter is the maximum + // number of connection IDs from the peer we're willing to store. + // + // maxPeerActiveConnIDLimit is the maximum number of connection IDs + // we're willing to send to the peer. + // + // https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2-6.2.1 + activeConnIDLimit = 2 + maxPeerActiveConnIDLimit = 4 ) // Local timer granularity. diff --git a/internal/quic/tls.go b/internal/quic/tls.go index 4306a3e46d..ed848c6a13 100644 --- a/internal/quic/tls.go +++ b/internal/quic/tls.go @@ -83,7 +83,9 @@ func (c *Conn) handleTLSEvents(now time.Time) error { if err != nil { return err } - c.receiveTransportParameters(params) + if err := c.receiveTransportParameters(params); err != nil { + return err + } } } } diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go index df07820086..3768dc0c07 100644 --- a/internal/quic/tls_test.go +++ b/internal/quic/tls_test.go @@ -63,15 +63,26 @@ func (tc *testConn) handshake() { func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { var ( - clientConnID []byte - serverConnID []byte + clientConnIDs [][]byte + serverConnIDs [][]byte + transientConnID []byte ) + localConnIDs := [][]byte{ + testLocalConnID(0), + testLocalConnID(1), + } + peerConnIDs := [][]byte{ + testPeerConnID(0), + testPeerConnID(1), + } if tc.conn.side == clientSide { - clientConnID = tc.localConnID - serverConnID = tc.peerConnID + clientConnIDs = localConnIDs + serverConnIDs = peerConnIDs + transientConnID = testLocalConnID(-1) } else { - clientConnID = tc.peerConnID - serverConnID = tc.localConnID + clientConnIDs = peerConnIDs + serverConnIDs = localConnIDs + transientConnID = []byte{0xde, 0xad, 0xbe, 0xef} } return []*testDatagram{{ // Client Initial @@ -79,21 +90,21 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { ptype: packetTypeInitial, num: 0, version: 1, - srcConnID: clientConnID, - dstConnID: tc.transientConnID, + srcConnID: clientConnIDs[0], + dstConnID: transientConnID, frames: []debugFrame{ debugFrameCrypto{}, }, }}, paddedSize: 1200, }, { - // Server Initial + Handshake + // Server Initial + Handshake + 1-RTT packets: []*testPacket{{ ptype: packetTypeInitial, num: 0, version: 1, - srcConnID: serverConnID, - dstConnID: clientConnID, + srcConnID: serverConnIDs[0], + dstConnID: clientConnIDs[0], frames: []debugFrame{ debugFrameAck{ ranges: []i64range[packetNumber]{{0, 1}}, @@ -104,20 +115,30 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { ptype: packetTypeHandshake, num: 0, version: 1, - srcConnID: serverConnID, - dstConnID: clientConnID, + srcConnID: serverConnIDs[0], + dstConnID: clientConnIDs[0], frames: []debugFrame{ debugFrameCrypto{}, }, + }, { + ptype: packetType1RTT, + num: 0, + dstConnID: clientConnIDs[0], + frames: []debugFrame{ + debugFrameNewConnectionID{ + seq: 1, + connID: serverConnIDs[1], + }, + }, }}, }, { - // Client Handshake + // Client Initial + Handshake + 1-RTT packets: []*testPacket{{ ptype: packetTypeInitial, num: 1, version: 1, - srcConnID: clientConnID, - dstConnID: serverConnID, + srcConnID: clientConnIDs[0], + dstConnID: serverConnIDs[0], frames: []debugFrame{ debugFrameAck{ ranges: []i64range[packetNumber]{{0, 1}}, @@ -127,23 +148,39 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { ptype: packetTypeHandshake, num: 0, version: 1, - srcConnID: clientConnID, - dstConnID: serverConnID, + srcConnID: clientConnIDs[0], + dstConnID: serverConnIDs[0], frames: []debugFrame{ debugFrameAck{ ranges: []i64range[packetNumber]{{0, 1}}, }, debugFrameCrypto{}, }, + }, { + ptype: packetType1RTT, + num: 0, + dstConnID: serverConnIDs[0], + frames: []debugFrame{ + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, + debugFrameNewConnectionID{ + seq: 1, + connID: clientConnIDs[1], + }, + }, }}, paddedSize: 1200, }, { // Server HANDSHAKE_DONE and session ticket packets: []*testPacket{{ ptype: packetType1RTT, - num: 0, - dstConnID: clientConnID, + num: 1, + dstConnID: clientConnIDs[0], frames: []debugFrame{ + debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 1}}, + }, debugFrameHandshakeDone{}, debugFrameCrypto{}, }, @@ -152,13 +189,13 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { // Client ack (after max_ack_delay) packets: []*testPacket{{ ptype: packetType1RTT, - num: 0, - dstConnID: serverConnID, + num: 1, + dstConnID: serverConnIDs[0], frames: []debugFrame{ debugFrameAck{ ackDelay: unscaledAckDelayFromDuration( maxAckDelay, ackDelayExponent), - ranges: []i64range[packetNumber]{{0, 1}}, + ranges: []i64range[packetNumber]{{0, 2}}, }, }, }}, @@ -190,6 +227,69 @@ func fillCryptoFrames(d *testDatagram, data map[tls.QUICEncryptionLevel][]byte) } } +// uncheckedHandshake executes the handshake. +// +// Unlike testConn.handshake, it sends nothing unnecessary +// (in particular, no NEW_CONNECTION_ID frames), +// and does not validate the conn's responses. +// +// Useful for testing scenarios where configuration has +// changed the handshake responses in some way. +func (tc *testConn) uncheckedHandshake() { + defer func(saved map[byte]bool) { + tc.ignoreFrames = saved + }(tc.ignoreFrames) + tc.ignoreFrames = map[byte]bool{ + frameTypeAck: true, + frameTypeCrypto: true, + frameTypeNewConnectionID: true, + } + if tc.conn.side == serverSide { + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrame("send HANDSHAKE_DONE after handshake completes", + packetType1RTT, debugFrameHandshakeDone{}) + tc.writeFrames(packetType1RTT, + debugFrameAck{ + ackDelay: unscaledAckDelayFromDuration( + maxAckDelay, ackDelayExponent), + ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, + }) + } else { + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantIdle("don't expect any frames we aren't ignoring") + // Send the next two frames in separate packets, so the client sends an + // ack immediately without delay. We want to consume that ack here, rather + // than returning with a delayed ack waiting to be sent. + tc.ignoreFrames = nil + tc.writeFrames(packetType1RTT, + debugFrameHandshakeDone{}) + tc.writeFrames(packetType1RTT, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrame("client ACKs server's first 1-RTT packet", + packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, 2}}, + }) + + } + tc.wantIdle("handshake is done") +} + func TestConnClientHandshake(t *testing.T) { tc := newTestConn(t, clientSide) tc.handshake() @@ -224,6 +324,11 @@ func TestConnKeysDiscardedClient(t *testing.T) { packetTypeHandshake, debugFrameCrypto{ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake], }) + tc.wantFrame("client provides an additional connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 1, + connID: testLocalConnID(1), + }) // The client discards Initial keys after sending a Handshake packet. tc.writeFrames(packetTypeInitial, @@ -273,6 +378,11 @@ func TestConnKeysDiscardedServer(t *testing.T) { }) tc.writeFrames(packetTypeInitial, debugFrameConnectionCloseTransport{code: errInternal}) + tc.wantFrame("server provides an additional connection ID", + packetType1RTT, debugFrameNewConnectionID{ + seq: 1, + connID: testLocalConnID(1), + }) tc.wantIdle("server has discarded Initial keys, cannot read CONNECTION_CLOSE") // The server discards Handshake keys after sending a HANDSHAKE_DONE frame. From 63fe334ad57133b911a1422472a28b11de828c89 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 31 Jan 2023 08:58:02 -0800 Subject: [PATCH 17/46] quic: gate and queue synchronization primitives Add a form of monitor (in the sense of the synchronization primitive) for controlling access to queues and streams. We call this a "gate". A gate acts as a mutex and condition variable with one bit of state. A gate may be locked and unlocked. Lock operations may optionally block on the gate condition being set. Unlock operations always record the new value of the condition. Gates play nicely with contexts. Unlike traditional condition variables, gates do not suffer from spurious wakeups: A goroutine waiting for a gate condition is not woken before the condition is set. Gates are inspired by the queue design from Bryan Mills's talk, Rethinking Classical Concurrency Patterns. Add a queue implemented with a gate. For golang/go#58547 Change-Id: Ibec6d1f29a2c03a7184fca7392ed5639f96b6485 Reviewed-on: https://go-review.googlesource.com/c/net/+/513955 TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam Run-TryBot: Damien Neil --- internal/quic/gate.go | 104 ++++++++++++++++++++++++++ internal/quic/gate_test.go | 142 ++++++++++++++++++++++++++++++++++++ internal/quic/queue.go | 65 +++++++++++++++++ internal/quic/queue_test.go | 59 +++++++++++++++ 4 files changed, 370 insertions(+) create mode 100644 internal/quic/gate.go create mode 100644 internal/quic/gate_test.go create mode 100644 internal/quic/queue.go create mode 100644 internal/quic/queue_test.go diff --git a/internal/quic/gate.go b/internal/quic/gate.go new file mode 100644 index 0000000000..efb28daf8f --- /dev/null +++ b/internal/quic/gate.go @@ -0,0 +1,104 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "context" + +// An gate is a monitor (mutex + condition variable) with one bit of state. +// +// The condition may be either set or unset. +// Lock operations may be unconditional, or wait for the condition to be set. +// Unlock operations record the new state of the condition. +type gate struct { + // When unlocked, exactly one of set or unset contains a value. + // When locked, neither chan contains a value. + set chan struct{} + unset chan struct{} +} + +func newGate() gate { + g := gate{ + set: make(chan struct{}, 1), + unset: make(chan struct{}, 1), + } + g.unset <- struct{}{} + return g +} + +// lock acquires the gate unconditionally. +// It reports whether the condition is set. +func (g *gate) lock() (set bool) { + select { + case <-g.set: + return true + case <-g.unset: + return false + } +} + +// waitAndLock waits until the condition is set before acquiring the gate. +func (g *gate) waitAndLock() { + <-g.set +} + +// waitAndLockContext waits until the condition is set before acquiring the gate. +// If the context expires, waitAndLockContext returns an error and does not acquire the gate. +func (g *gate) waitAndLockContext(ctx context.Context) error { + select { + case <-g.set: + return nil + default: + } + select { + case <-g.set: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// waitWithLock releases an acquired gate until the condition is set. +// The caller must have previously acquired the gate. +// Upon return from waitWithLock, the gate will still be held. +// If waitWithLock returns nil, the condition is set. +func (g *gate) waitWithLock(ctx context.Context) error { + g.unlock(false) + err := g.waitAndLockContext(ctx) + if err != nil { + if g.lock() { + // The condition was set in between the context expiring + // and us reacquiring the gate. + err = nil + } + } + return err +} + +// lockIfSet acquires the gate if and only if the condition is set. +func (g *gate) lockIfSet() (acquired bool) { + select { + case <-g.set: + return true + default: + return false + } +} + +// unlock sets the condition and releases the gate. +func (g *gate) unlock(set bool) { + if set { + g.set <- struct{}{} + } else { + g.unset <- struct{}{} + } +} + +// unlock sets the condition to the result of f and releases the gate. +// Useful in defers. +func (g *gate) unlockFunc(f func() bool) { + g.unlock(f()) +} diff --git a/internal/quic/gate_test.go b/internal/quic/gate_test.go new file mode 100644 index 0000000000..0122e39865 --- /dev/null +++ b/internal/quic/gate_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "testing" + "time" +) + +func TestGateLockAndUnlock(t *testing.T) { + g := newGate() + if set := g.lock(); set { + t.Errorf("g.lock() of never-locked gate: true, want false") + } + unlockedc := make(chan struct{}) + donec := make(chan struct{}) + go func() { + defer close(donec) + set := g.lock() + select { + case <-unlockedc: + default: + t.Errorf("g.lock() succeeded while gate was held") + } + if !set { + t.Errorf("g.lock() of set gate: false, want true") + } + g.unlock(false) + }() + time.Sleep(1 * time.Millisecond) + close(unlockedc) + g.unlock(true) + <-donec + if set := g.lock(); set { + t.Errorf("g.lock() of unset gate: true, want false") + } +} + +func TestGateWaitAndLock(t *testing.T) { + g := newGate() + set := false + go func() { + for i := 0; i < 3; i++ { + g.lock() + g.unlock(false) + time.Sleep(1 * time.Millisecond) + } + g.lock() + set = true + g.unlock(true) + }() + g.waitAndLock() + if !set { + t.Errorf("g.waitAndLock() returned before gate was set") + } +} + +func TestGateWaitAndLockContext(t *testing.T) { + g := newGate() + // waitAndLockContext is canceled + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1 * time.Millisecond) + cancel() + }() + if err := g.waitAndLockContext(ctx); err != context.Canceled { + t.Errorf("g.waitAndLockContext() = %v, want context.Canceled", err) + } + // waitAndLockContext succeeds + set := false + go func() { + time.Sleep(1 * time.Millisecond) + g.lock() + set = true + g.unlock(true) + }() + if err := g.waitAndLockContext(context.Background()); err != nil { + t.Errorf("g.waitAndLockContext() = %v, want nil", err) + } + if !set { + t.Errorf("g.waitAndLockContext() returned before gate was set") + } + g.unlock(true) + // waitAndLockContext succeeds when the gate is set and the context is canceled + if err := g.waitAndLockContext(ctx); err != nil { + t.Errorf("g.waitAndLockContext() = %v, want nil", err) + } +} + +func TestGateWaitWithLock(t *testing.T) { + g := newGate() + // waitWithLock is canceled + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1 * time.Millisecond) + cancel() + }() + g.lock() + if err := g.waitWithLock(ctx); err != context.Canceled { + t.Errorf("g.waitWithLock() = %v, want context.Canceled", err) + } + // waitWithLock succeeds + set := false + go func() { + g.lock() + set = true + g.unlock(true) + }() + time.Sleep(1 * time.Millisecond) + if err := g.waitWithLock(context.Background()); err != nil { + t.Errorf("g.waitWithLock() = %v, want nil", err) + } + if !set { + t.Errorf("g.waitWithLock() returned before gate was set") + } +} + +func TestGateLockIfSet(t *testing.T) { + g := newGate() + if locked := g.lockIfSet(); locked { + t.Errorf("g.lockIfSet() of unset gate = %v, want false", locked) + } + g.lock() + g.unlock(true) + if locked := g.lockIfSet(); !locked { + t.Errorf("g.lockIfSet() of set gate = %v, want true", locked) + } +} + +func TestGateUnlockFunc(t *testing.T) { + g := newGate() + go func() { + g.lock() + defer g.unlockFunc(func() bool { return true }) + }() + g.waitAndLock() +} diff --git a/internal/quic/queue.go b/internal/quic/queue.go new file mode 100644 index 0000000000..9bb71ca3f4 --- /dev/null +++ b/internal/quic/queue.go @@ -0,0 +1,65 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "context" + +// A queue is an unbounded queue of some item (new connections and streams). +type queue[T any] struct { + // The gate condition is set if the queue is non-empty or closed. + gate gate + err error + q []T +} + +func newQueue[T any]() queue[T] { + return queue[T]{gate: newGate()} +} + +// close closes the queue, causing pending and future pop operations +// to return immediately with err. +func (q *queue[T]) close(err error) { + q.gate.lock() + defer q.unlock() + if q.err == nil { + q.err = err + } +} + +// put appends an item to the queue. +// It returns true if the item was added, false if the queue is closed. +func (q *queue[T]) put(v T) bool { + q.gate.lock() + defer q.unlock() + if q.err != nil { + return false + } + q.q = append(q.q, v) + return true +} + +// get removes the first item from the queue, blocking until ctx is done, an item is available, +// or the queue is closed. +func (q *queue[T]) get(ctx context.Context) (T, error) { + var zero T + if err := q.gate.waitAndLockContext(ctx); err != nil { + return zero, err + } + defer q.unlock() + if q.err != nil { + return zero, q.err + } + v := q.q[0] + copy(q.q[:], q.q[1:]) + q.q[len(q.q)-1] = zero + q.q = q.q[:len(q.q)-1] + return v, nil +} + +func (q *queue[T]) unlock() { + q.gate.unlock(q.err != nil || len(q.q) > 0) +} diff --git a/internal/quic/queue_test.go b/internal/quic/queue_test.go new file mode 100644 index 0000000000..8debeff110 --- /dev/null +++ b/internal/quic/queue_test.go @@ -0,0 +1,59 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "io" + "testing" + "time" +) + +func TestQueue(t *testing.T) { + nonblocking, cancel := context.WithCancel(context.Background()) + cancel() + + q := newQueue[int]() + if got, err := q.get(nonblocking); err != context.Canceled { + t.Fatalf("q.get() = %v, %v, want nil, contex.Canceled", got, err) + } + + if !q.put(1) { + t.Fatalf("q.put(1) = false, want true") + } + if !q.put(2) { + t.Fatalf("q.put(2) = false, want true") + } + if got, err := q.get(nonblocking); got != 1 || err != nil { + t.Fatalf("q.get() = %v, %v, want 1, nil", got, err) + } + if got, err := q.get(nonblocking); got != 2 || err != nil { + t.Fatalf("q.get() = %v, %v, want 2, nil", got, err) + } + if got, err := q.get(nonblocking); err != context.Canceled { + t.Fatalf("q.get() = %v, %v, want nil, contex.Canceled", got, err) + } + + go func() { + time.Sleep(1 * time.Millisecond) + q.put(3) + }() + if got, err := q.get(context.Background()); got != 3 || err != nil { + t.Fatalf("q.get() = %v, %v, want 3, nil", got, err) + } + + if !q.put(4) { + t.Fatalf("q.put(2) = false, want true") + } + q.close(io.EOF) + if got, err := q.get(context.Background()); got != 0 || err != io.EOF { + t.Fatalf("q.get() = %v, %v, want 0, io.EOF", got, err) + } + if q.put(5) { + t.Fatalf("q.put(5) = true, want false") + } +} From 8ffa475fbdb33da97e8bf79cc5791ee8751fca5e Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Thu, 6 Jul 2023 10:25:47 -0700 Subject: [PATCH 18/46] html: only render content literally in the HTML namespace Per the WHATWG HTML specification, section 13.3, only append the literal content of a text node if we are in the HTML namespace. Thanks to Mohammad Thoriq Aziz for reporting this issue. Fixes golang/go#61615 Fixes CVE-2023-3978 Change-Id: I332152904d4e7646bd2441602bcbe591fc655fa4 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1942896 Reviewed-by: Tatiana Bradley Run-TryBot: Roland Shoemaker Reviewed-by: Damien Neil TryBot-Result: Security TryBots Reviewed-on: https://go-review.googlesource.com/c/net/+/514896 Reviewed-by: Roland Shoemaker TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- html/render.go | 28 +++++++++++++++++++---- html/render_test.go | 56 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/html/render.go b/html/render.go index 8b28031905..e8c1233455 100644 --- a/html/render.go +++ b/html/render.go @@ -194,9 +194,8 @@ func render1(w writer, n *Node) error { } } - // Render any child nodes. - switch n.Data { - case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "xmp": + // Render any child nodes + if childTextNodesAreLiteral(n) { for c := n.FirstChild; c != nil; c = c.NextSibling { if c.Type == TextNode { if _, err := w.WriteString(c.Data); err != nil { @@ -213,7 +212,7 @@ func render1(w writer, n *Node) error { // last element in the file, with no closing tag. return plaintextAbort } - default: + } else { for c := n.FirstChild; c != nil; c = c.NextSibling { if err := render1(w, c); err != nil { return err @@ -231,6 +230,27 @@ func render1(w writer, n *Node) error { return w.WriteByte('>') } +func childTextNodesAreLiteral(n *Node) bool { + // Per WHATWG HTML 13.3, if the parent of the current node is a style, + // script, xmp, iframe, noembed, noframes, or plaintext element, and the + // current node is a text node, append the value of the node's data + // literally. The specification is not explicit about it, but we only + // enforce this if we are in the HTML namespace (i.e. when the namespace is + // ""). + // NOTE: we also always include noscript elements, although the + // specification states that they should only be rendered as such if + // scripting is enabled for the node (which is not something we track). + if n.Namespace != "" { + return false + } + switch n.Data { + case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "xmp": + return true + default: + return false + } +} + // writeQuoted writes s to w surrounded by quotes. Normally it will use double // quotes, but if s contains a double quote, it will use single quotes. // It is used for writing the identifiers in a doctype declaration. diff --git a/html/render_test.go b/html/render_test.go index 08e592be27..22d08641a0 100644 --- a/html/render_test.go +++ b/html/render_test.go @@ -6,6 +6,8 @@ package html import ( "bytes" + "fmt" + "strings" "testing" ) @@ -108,16 +110,16 @@ func TestRenderer(t *testing.T) { // just commentary. The "0:" prefixes are for easy cross-reference with // the nodes array. treeAsText := [...]string{ - 0: ``, - 1: `. `, - 2: `. `, - 3: `. . "0<1"`, - 4: `. .

`, - 5: `. . . "2"`, - 6: `. . . `, - 7: `. . . . "3"`, - 8: `. . . `, - 9: `. . . . "&4"`, + 0: ``, + 1: `. `, + 2: `. `, + 3: `. . "0<1"`, + 4: `. .

`, + 5: `. . . "2"`, + 6: `. . . `, + 7: `. . . . "3"`, + 8: `. . . `, + 9: `. . . . "&4"`, 10: `. . "5"`, 11: `. .

`, 12: `. .
`, @@ -169,3 +171,37 @@ func TestRenderer(t *testing.T) { t.Errorf("got vs want:\n%s\n%s\n", got, want) } } + +func TestRenderTextNodes(t *testing.T) { + elements := []string{"style", "script", "xmp", "iframe", "noembed", "noframes", "plaintext", "noscript"} + for _, namespace := range []string{ + "", // html + "svg", + "math", + } { + for _, e := range elements { + var namespaceOpen, namespaceClose string + if namespace != "" { + namespaceOpen, namespaceClose = fmt.Sprintf("<%s>", namespace), fmt.Sprintf("", namespace) + } + doc := fmt.Sprintf(`%s<%s>&%s`, namespaceOpen, e, e, namespaceClose) + n, err := Parse(strings.NewReader(doc)) + if err != nil { + t.Fatal(err) + } + b := bytes.NewBuffer(nil) + if err := Render(b, n); err != nil { + t.Fatal(err) + } + + expected := doc + if namespace != "" { + expected = strings.Replace(expected, "&", "&", 1) + } + + if b.String() != expected { + t.Errorf("unexpected output: got %q, want %q", b.String(), expected) + } + } + } +} From 167593b38cf631be267ebcd8d612b7c58138d8c4 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 Jul 2023 11:46:09 -0400 Subject: [PATCH 19/46] quic: create and accept streams Add minimal API surface for creating streams, basic loop for sending stream-related frames. No limits, data, or lifetime management yet. RFC 9000, Sections 2 and 3. For golang/go#58547 Change-Id: I2c167b9363d0121b8a8776309d165b0f47f8f090 Reviewed-on: https://go-review.googlesource.com/c/net/+/514115 Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- internal/quic/conn.go | 17 ++- internal/quic/conn_loss.go | 10 ++ internal/quic/conn_loss_test.go | 32 +++++ internal/quic/conn_recv.go | 15 +- internal/quic/conn_send.go | 7 + internal/quic/conn_streams.go | 215 +++++++++++++++++++++++++++++ internal/quic/conn_streams_test.go | 144 +++++++++++++++++++ internal/quic/conn_test.go | 11 ++ internal/quic/packet_parser.go | 3 + internal/quic/quic.go | 1 + internal/quic/stream.go | 151 ++++++++++++++++++++ internal/quic/stream_test.go | 33 +++++ 12 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 internal/quic/conn_streams.go create mode 100644 internal/quic/conn_streams_test.go create mode 100644 internal/quic/stream.go create mode 100644 internal/quic/stream_test.go diff --git a/internal/quic/conn.go b/internal/quic/conn.go index 77ecea0d62..5601b989e3 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -32,6 +32,7 @@ type Conn struct { acks [numberSpaceCount]ackState // indexed by number space connIDState connIDState loss lossState + streams streamsState // errForPeer is set when the connection is being closed. errForPeer error @@ -105,6 +106,7 @@ func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip. // TODO: PMTU discovery. const maxDatagramSize = 1200 c.loss.init(c.side, maxDatagramSize, now) + c.streamsInit() c.startTLS(now, initialConnID, transportParameters{ initialSrcConnID: c.connIDState.srcConnID(), @@ -178,7 +180,10 @@ func (c *Conn) receiveTransportParameters(p transportParameters) error { return nil } -type timerEvent struct{} +type ( + timerEvent struct{} + wakeEvent struct{} +) // loop is the connection main loop. // @@ -250,6 +255,8 @@ func (c *Conn) loop(now time.Time) { return } c.loss.advance(now, c.handleAckOrLoss) + case wakeEvent: + // We're being woken up to try sending some frames. case func(time.Time, *Conn): // Send a func to msgc to run it on the main Conn goroutine m(now, c) @@ -269,6 +276,14 @@ func (c *Conn) sendMsg(m any) { } } +// wake wakes up the conn's loop. +func (c *Conn) wake() { + select { + case c.msgc <- wakeEvent{}: + default: + } +} + // runOnLoop executes a function within the conn's loop goroutine. func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { donec := make(chan struct{}) diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go index 57570d0861..ca178089d2 100644 --- a/internal/quic/conn_loss.go +++ b/internal/quic/conn_loss.go @@ -44,6 +44,16 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF case frameTypeCrypto: start, end := sent.nextRange() c.crypto[space].ackOrLoss(start, end, fate) + case frameTypeStreamBase, + frameTypeStreamBase | streamFinBit: + id := streamID(sent.nextInt()) + start, end := sent.nextRange() + s := c.streamForID(id) + if s == nil { + continue + } + fin := f&streamFinBit != 0 + s.ackOrLossData(sent.num, start, end, fin, fate) case frameTypeNewConnectionID: seq := int64(sent.nextInt()) c.connIDState.ackOrLossNewConnectionID(sent.num, seq, fate) diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index 021c86c876..3c9e6149ab 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -106,6 +106,38 @@ func TestLostCRYPTOFrame(t *testing.T) { }) } +func TestLostStreamFrameEmpty(t *testing.T) { + // A STREAM frame opening a stream, but containing no stream data, should + // be retransmitted if lost. + lostFrameTest(t, func(t *testing.T, pto bool) { + ctx := canceledContext() + tc := newTestConn(t, clientSide, func(p *transportParameters) { + p.initialMaxStreamDataBidiRemote = 100 + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + c, err := tc.conn.NewStream(ctx) + if err != nil { + t.Fatalf("NewStream: %v", err) + } + c.Write(nil) // open the stream + tc.wantFrame("created bidirectional stream 0", + packetType1RTT, debugFrameStream{ + id: newStreamID(clientSide, bidiStream, 0), + data: []byte{}, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resent stream frame", + packetType1RTT, debugFrameStream{ + id: newStreamID(clientSide, bidiStream, 0), + data: []byte{}, + }) + }) + +} + func TestLostNewConnectionIDFrame(t *testing.T) { // "New connection IDs are [...] retransmitted if the packet containing them is lost." // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.13 diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 7992a619f4..45ef3844e8 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -181,7 +181,7 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, if !frameOK(c, ptype, __01) { return } - _, _, _, _, n = consumeStreamFrame(payload) + n = c.handleStreamFrame(now, space, payload) case frameTypeMaxData: if !frameOK(c, ptype, __01) { return @@ -290,6 +290,19 @@ func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byt return n } +func (c *Conn) handleStreamFrame(now time.Time, space numberSpace, payload []byte) int { + id, off, fin, b, n := consumeStreamFrame(payload) + if n < 0 { + return -1 + } + if s := c.streamForFrame(now, id, recvStream); s != nil { + if err := s.handleData(off, b, fin); err != nil { + c.abort(now, err) + } + } + return n +} + func (c *Conn) handleNewConnectionIDFrame(now time.Time, space numberSpace, payload []byte) int { seq, retire, connID, resetToken, n := consumeNewConnectionIDFrame(payload) if n < 0 { diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index d410548a98..6e6fbc5857 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -254,6 +254,13 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, c.testSendPing.setSent(pnum) } + // All stream-related frames. This should come last in the packet, + // so large amounts of STREAM data don't crowd out other frames + // we may need to send. + if !c.appendStreamFrames(&c.w, pnum, pto) { + return + } + // If this is a PTO probe and we haven't added an ack-eliciting frame yet, // add a PING to make this an ack-eliciting probe. // diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go new file mode 100644 index 0000000000..82e9028609 --- /dev/null +++ b/internal/quic/conn_streams.go @@ -0,0 +1,215 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +type streamsState struct { + queue queue[*Stream] // new, peer-created streams + + streamsMu sync.Mutex + streams map[streamID]*Stream + opened [streamTypeCount]int64 // number of streams opened by us + + // Streams with frames to send are stored in a circular linked list. + // sendHead is the next stream to write, or nil if there are no streams + // with data to send. sendTail is the last stream to write. + needSend atomic.Bool + sendMu sync.Mutex + sendHead *Stream + sendTail *Stream +} + +func (c *Conn) streamsInit() { + c.streams.streams = make(map[streamID]*Stream) + c.streams.queue = newQueue[*Stream]() +} + +// AcceptStream waits for and returns the next stream created by the peer. +func (c *Conn) AcceptStream(ctx context.Context) (*Stream, error) { + return c.streams.queue.get(ctx) +} + +// NewStream creates a stream. +// +// If the peer's maximum stream limit for the connection has been reached, +// NewStream blocks until the limit is increased or the context expires. +func (c *Conn) NewStream(ctx context.Context) (*Stream, error) { + return c.newLocalStream(ctx, bidiStream) +} + +// NewSendOnlyStream creates a unidirectional, send-only stream. +// +// If the peer's maximum stream limit for the connection has been reached, +// NewSendOnlyStream blocks until the limit is increased or the context expires. +func (c *Conn) NewSendOnlyStream(ctx context.Context) (*Stream, error) { + return c.newLocalStream(ctx, uniStream) +} + +func (c *Conn) newLocalStream(ctx context.Context, typ streamType) (*Stream, error) { + // TODO: Stream limits. + c.streams.streamsMu.Lock() + defer c.streams.streamsMu.Unlock() + + num := c.streams.opened[typ] + c.streams.opened[typ]++ + + s := newStream(c, newStreamID(c.side, typ, num)) + c.streams.streams[s.id] = s + return s, nil +} + +// streamFrameType identifies which direction of a stream, +// from the local perspective, a frame is associated with. +// +// For example, STREAM is a recvStream frame, +// because it carries data from the peer to us. +type streamFrameType uint8 + +const ( + sendStream = streamFrameType(iota) // for example, MAX_DATA + recvStream // for example, STREAM_DATA_BLOCKED +) + +// streamForID returns the stream with the given id. +// If the stream does not exist, it returns nil. +func (c *Conn) streamForID(id streamID) *Stream { + c.streams.streamsMu.Lock() + defer c.streams.streamsMu.Unlock() + return c.streams.streams[id] +} + +// streamForFrame returns the stream with the given id. +// If the stream does not exist, it may be created. +// +// streamForFrame aborts the connection if the stream id, state, and frame type don't align. +// For example, it aborts the connection with a STREAM_STATE error if a MAX_DATA frame +// is received for a receive-only stream, or if the peer attempts to create a stream that +// should be originated locally. +// +// streamForFrame returns nil if the stream no longer exists or if an error occurred. +func (c *Conn) streamForFrame(now time.Time, id streamID, ftype streamFrameType) *Stream { + if id.streamType() == uniStream { + if (id.initiator() == c.side) != (ftype == sendStream) { + // Received an invalid frame for unidirectional stream. + // For example, a RESET_STREAM frame for a send-only stream. + c.abort(now, localTransportError(errStreamState)) + return nil + } + } + + c.streams.streamsMu.Lock() + defer c.streams.streamsMu.Unlock() + if s := c.streams.streams[id]; s != nil { + return s + } + // TODO: Check for closed streams, once we support closing streams. + if id.initiator() == c.side { + c.abort(now, localTransportError(errStreamState)) + return nil + } + s := newStream(c, id) + c.streams.streams[id] = s + c.streams.queue.put(s) + return s +} + +// queueStreamForSend marks a stream as containing frames that need sending. +func (c *Conn) queueStreamForSend(s *Stream) { + c.streams.sendMu.Lock() + defer c.streams.sendMu.Unlock() + if s.next != nil { + // Already in the queue. + return + } + if c.streams.sendHead == nil { + // The queue was empty. + c.streams.sendHead = s + c.streams.sendTail = s + s.next = s + } else { + // Insert this stream at the end of the queue. + c.streams.sendTail.next = s + c.streams.sendTail = s + } + c.streams.needSend.Store(true) + c.wake() +} + +// appendStreamFrames writes stream-related frames to the current packet. +// +// It returns true if no more frames need appending, +// false if not everything fit in the current packet. +func (c *Conn) appendStreamFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + if pto { + return c.appendStreamFramesPTO(w, pnum) + } + if !c.streams.needSend.Load() { + return true + } + c.streams.sendMu.Lock() + defer c.streams.sendMu.Unlock() + for { + s := c.streams.sendHead + const pto = false + if !s.appendInFrames(w, pnum, pto) { + return false + } + avail := w.avail() + if !s.appendOutFrames(w, pnum, pto) { + // We've sent some data for this stream, but it still has more to send. + // If the stream got a reasonable chance to put data in a packet, + // advance sendHead to the next stream in line, to avoid starvation. + // We'll come back to this stream after going through the others. + // + // If the packet was already mostly out of space, leave sendHead alone + // and come back to this stream again on the next packet. + if avail > 512 { + c.streams.sendHead = s.next + c.streams.sendTail = s + } + return false + } + s.next = nil + if s == c.streams.sendTail { + // This was the last stream. + c.streams.sendHead = nil + c.streams.sendTail = nil + c.streams.needSend.Store(false) + return true + } + // We've sent all data for this stream, so remove it from the list. + c.streams.sendTail.next = s.next + c.streams.sendHead = s.next + s.next = nil + } +} + +// appendStreamFramesPTO writes stream-related frames to the current packet +// for a PTO probe. +// +// It returns true if no more frames need appending, +// false if not everything fit in the current packet. +func (c *Conn) appendStreamFramesPTO(w *packetWriter, pnum packetNumber) bool { + c.streams.sendMu.Lock() + defer c.streams.sendMu.Unlock() + for _, s := range c.streams.streams { + const pto = true + if !s.appendInFrames(w, pnum, pto) { + return false + } + if !s.appendOutFrames(w, pnum, pto) { + return false + } + } + return true +} diff --git a/internal/quic/conn_streams_test.go b/internal/quic/conn_streams_test.go new file mode 100644 index 0000000000..8481a604c5 --- /dev/null +++ b/internal/quic/conn_streams_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "testing" +) + +func TestStreamsCreate(t *testing.T) { + ctx := canceledContext() + tc := newTestConn(t, clientSide, func(p *transportParameters) { + p.initialMaxStreamDataBidiLocal = 100 + p.initialMaxStreamDataBidiRemote = 100 + }) + tc.handshake() + + c, err := tc.conn.NewStream(ctx) + if err != nil { + t.Fatalf("NewStream: %v", err) + } + c.Write(nil) // open the stream + tc.wantFrame("created bidirectional stream 0", + packetType1RTT, debugFrameStream{ + id: 0, // client-initiated, bidi, number 0 + data: []byte{}, + }) + + c, err = tc.conn.NewSendOnlyStream(ctx) + if err != nil { + t.Fatalf("NewStream: %v", err) + } + c.Write(nil) // open the stream + tc.wantFrame("created unidirectional stream 0", + packetType1RTT, debugFrameStream{ + id: 2, // client-initiated, uni, number 0 + data: []byte{}, + }) + + c, err = tc.conn.NewStream(ctx) + if err != nil { + t.Fatalf("NewStream: %v", err) + } + c.Write(nil) // open the stream + tc.wantFrame("created bidirectional stream 1", + packetType1RTT, debugFrameStream{ + id: 4, // client-initiated, uni, number 4 + data: []byte{}, + }) +} + +func TestStreamsAccept(t *testing.T) { + ctx := canceledContext() + tc := newTestConn(t, serverSide) + tc.handshake() + + tc.writeFrames(packetType1RTT, + debugFrameStream{ + id: 0, // client-initiated, bidi, number 0 + }, + debugFrameStream{ + id: 2, // client-initiated, uni, number 0 + }, + debugFrameStream{ + id: 4, // client-initiated, bidi, number 1 + }) + + for _, accept := range []struct { + id streamID + readOnly bool + }{ + {0, false}, + {2, true}, + {4, false}, + } { + s, err := tc.conn.AcceptStream(ctx) + if err != nil { + t.Fatalf("conn.AcceptStream() = %v, want stream %v", err, accept.id) + } + if got, want := s.id, accept.id; got != want { + t.Fatalf("conn.AcceptStream() = stream %v, want %v", got, want) + } + if got, want := s.IsReadOnly(), accept.readOnly; got != want { + t.Fatalf("stream %v: s.IsReadOnly() = %v, want %v", accept.id, got, want) + } + } + + _, err := tc.conn.AcceptStream(ctx) + if err != context.Canceled { + t.Fatalf("conn.AcceptStream() = %v, want context.Canceled", err) + } +} + +func TestStreamsStreamNotCreated(t *testing.T) { + // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR + // if it receives a STREAM frame for a locally initiated stream that has + // not yet been created [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3 + tc := newTestConn(t, serverSide) + tc.handshake() + + tc.writeFrames(packetType1RTT, + debugFrameStream{ + id: 1, // server-initiated, bidi, number 0 + }) + tc.wantFrame("peer sent STREAM frame for an uncreated local stream", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) +} + +func TestStreamsStreamSendOnly(t *testing.T) { + // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR + // if it receives a STREAM frame for a locally initiated stream that has + // not yet been created [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3 + ctx := canceledContext() + tc := newTestConn(t, serverSide) + tc.handshake() + + c, err := tc.conn.NewSendOnlyStream(ctx) + if err != nil { + t.Fatalf("NewStream: %v", err) + } + c.Write(nil) // open the stream + tc.wantFrame("created unidirectional stream 0", + packetType1RTT, debugFrameStream{ + id: 3, // server-initiated, uni, number 0 + data: []byte{}, + }) + + tc.writeFrames(packetType1RTT, + debugFrameStream{ + id: 3, // server-initiated, bidi, number 0 + }) + tc.wantFrame("peer sent STREAM frame for a send-only stream", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) +} diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 317ca8f81f..1fe1e7b84a 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -725,3 +725,14 @@ func (tc *testConnListener) sendDatagram(p []byte, addr netip.AddrPort) error { tc.sentDatagrams = append(tc.sentDatagrams, append([]byte(nil), p...)) return nil } + +// canceledContext returns a canceled Context. +// +// Functions which take a context preference progress over cancelation. +// For example, a read with a canceled context will return data if any is available. +// Tests use canceled contexts to perform non-blocking operations. +func canceledContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} diff --git a/internal/quic/packet_parser.go b/internal/quic/packet_parser.go index 0520078974..9a00da7560 100644 --- a/internal/quic/packet_parser.go +++ b/internal/quic/packet_parser.go @@ -330,6 +330,9 @@ func consumeStreamFrame(b []byte) (id streamID, off int64, fin bool, data []byte data = b[n:] n += len(data) } + if off+int64(len(data)) >= 1<<62 { + return 0, 0, false, nil, -1 + } return streamID(idInt), off, fin, data, n } diff --git a/internal/quic/quic.go b/internal/quic/quic.go index 84ce2bda13..8cd61aed08 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -112,6 +112,7 @@ type streamType uint8 const ( bidiStream = streamType(iota) uniStream + streamTypeCount ) func (s streamType) String() string { diff --git a/internal/quic/stream.go b/internal/quic/stream.go new file mode 100644 index 0000000000..b55f927e02 --- /dev/null +++ b/internal/quic/stream.go @@ -0,0 +1,151 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "errors" +) + +type Stream struct { + id streamID + conn *Conn + + // outgate's lock guards all send-related state. + // + // The gate condition is set if a write to the stream will not block, + // either because the stream has available flow control or because + // the write will fail. + outgate gate + outopened sentVal // set if we should open the stream + + prev, next *Stream // guarded by streamsState.sendMu +} + +func newStream(c *Conn, id streamID) *Stream { + s := &Stream{ + conn: c, + id: id, + outgate: newGate(), + } + + // Lock and unlock outgate to update the stream writability state. + s.outgate.lock() + s.outUnlock() + + return s +} + +// IsReadOnly reports whether the stream is read-only +// (a unidirectional stream created by the peer). +func (s *Stream) IsReadOnly() bool { + return s.id.streamType() == uniStream && s.id.initiator() != s.conn.side +} + +// IsWriteOnly reports whether the stream is write-only +// (a unidirectional stream created locally). +func (s *Stream) IsWriteOnly() bool { + return s.id.streamType() == uniStream && s.id.initiator() == s.conn.side +} + +// Read reads data from the stream. +// See ReadContext for more details. +func (s *Stream) Read(b []byte) (n int, err error) { + return s.ReadContext(context.Background(), b) +} + +// ReadContext reads data from the stream. +// +// ReadContext returns as soon as at least one byte of data is available. +// +// If the peer closes the stream cleanly, ReadContext returns io.EOF after +// returning all data sent by the peer. +// If the peer terminates reads abruptly, ReadContext returns StreamResetError. +func (s *Stream) ReadContext(ctx context.Context, b []byte) (n int, err error) { + // TODO: implement + return 0, errors.New("unimplemented") +} + +// Write writes data to the stream. +// See WriteContext for more details. +func (s *Stream) Write(b []byte) (n int, err error) { + return s.WriteContext(context.Background(), b) +} + +// WriteContext writes data to the stream. +// +// WriteContext writes data to the stream write buffer. +// Buffered data is only sent when the buffer is sufficiently full. +// Call the Flush method to ensure buffered data is sent. +// +// If the peer aborts reads on the stream, ReadContext returns StreamResetError. +func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) { + if s.IsReadOnly() { + return 0, errors.New("write to read-only stream") + } + if len(b) > 0 { + // TODO: implement + return 0, errors.New("unimplemented") + } + if err := s.outgate.waitAndLockContext(ctx); err != nil { + return 0, err + } + defer s.outUnlock() + + // Set outopened to send a STREAM frame with no data, + // opening the stream on the peer. + s.outopened.set() + + return n, nil +} + +// outUnlock unlocks s.outgate. +// It sets the gate condition if writes to s will not block. +// If s has frames to write, it notifies the Conn. +func (s *Stream) outUnlock() { + if s.outopened.shouldSend() { + s.conn.queueStreamForSend(s) + } + canSend := true // TODO: set sendability status based on flow control + s.outgate.unlock(canSend) +} + +// handleData handles data received in a STREAM frame. +func (s *Stream) handleData(off int64, b []byte, fin bool) error { + // TODO + return nil +} + +// ackOrLossData handles the fate of a STREAM frame. +func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fate packetFate) { + s.outgate.lock() + defer s.outUnlock() + s.outopened.ackOrLoss(pnum, fate) +} + +func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + // TODO: STOP_SENDING + // TODO: MAX_STREAM_DATA + return true +} + +func (s *Stream) appendOutFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + // TODO: RESET_STREAM + // TODO: STREAM_DATA_BLOCKED + // TODO: STREAM frames with data + if s.outopened.shouldSendPTO(pto) { + off := int64(0) + size := 0 + fin := false + _, added := w.appendStreamFrame(s.id, off, size, fin) + if !added { + return false + } + s.outopened.setSent(pnum) + } + return true +} diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go new file mode 100644 index 0000000000..8ae9dbc825 --- /dev/null +++ b/internal/quic/stream_test.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "reflect" + "testing" +) + +func TestStreamOffsetTooLarge(t *testing.T) { + // "Receipt of a frame that exceeds [2^62-1] MUST be treated as a + // connection error of type FRAME_ENCODING_ERROR or FLOW_CONTROL_ERROR." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-9 + tc := newTestConn(t, serverSide) + tc.handshake() + + tc.writeFrames(packetType1RTT, + debugFrameStream{ + id: newStreamID(clientSide, bidiStream, 0), + off: 1<<62 - 1, + data: []byte{0}, + }) + got, _ := tc.readFrame() + want1 := debugFrameConnectionCloseTransport{code: errFrameEncoding} + want2 := debugFrameConnectionCloseTransport{code: errFlowControl} + if !reflect.DeepEqual(got, want1) && !reflect.DeepEqual(got, want2) { + t.Fatalf("STREAM offset exceeds 2^62-1\ngot: %v\nwant: %v\n or: %v", got, want1, want2) + } +} From a7da556f067cc43c881288fc0577d0000a6ad619 Mon Sep 17 00:00:00 2001 From: David Fu Date: Mon, 10 Jul 2023 07:36:53 +0000 Subject: [PATCH 20/46] http2: optimize buffer allocation in transport We have identified a high memory usage problem in our production service, which utilizes Traefik as a gRPC proxy. This service handles a substantial volume of gRPC bi-directional streaming requests that can persist for extended periods, spanning many days. Currently, there exists only a single global buffer pool in the http2 package. The allocated buffers, regardless of their sizes, are shared among requests with vastly different characteristics. For instance, gRPC streaming requests typically require smaller buffer sizes and occupy buffers for significant durations. Conversely, general HTTP requests may necessitate larger buffer sizes but only retain them temporarily. Unfortunately, the large buffers allocated by HTTP requests are can be chosen for subsequent gRPC streaming requests, resulting in numerous large buffers being unable to be recycled. In our production environment, which processes approximately 1 million gRPC streaming requests, memory usage can soar to an excessive 800 GiB. This is a substantial waste of resources. To address this challenge, we propose implementing a multi-layered buffer pool mechanism. This mechanism allows requests with varying characteristics to select buffers of appropriate sizes, optimizing resource allocation and recycling. Change-Id: I834f7c08d90fd298aac7971ad45dc1a36251788b GitHub-Last-Rev: 477197698f27f55a1cffe6864fcb84582f80c7a7 GitHub-Pull-Request: golang/net#182 Reviewed-on: https://go-review.googlesource.com/c/net/+/508415 Run-TryBot: Damien Neil Reviewed-by: David Chase Reviewed-by: Brad Fitzpatrick TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Reviewed-by: Damien Neil --- http2/transport.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/http2/transport.go b/http2/transport.go index b20c749171..b0d482f9f4 100644 --- a/http2/transport.go +++ b/http2/transport.go @@ -19,6 +19,7 @@ import ( "io/fs" "log" "math" + "math/bits" mathrand "math/rand" "net" "net/http" @@ -1680,7 +1681,27 @@ func (cs *clientStream) frameScratchBufferLen(maxFrameSize int) int { return int(n) // doesn't truncate; max is 512K } -var bufPool sync.Pool // of *[]byte +// Seven bufPools manage different frame sizes. This helps to avoid scenarios where long-running +// streaming requests using small frame sizes occupy large buffers initially allocated for prior +// requests needing big buffers. The size ranges are as follows: +// {0 KB, 16 KB], {16 KB, 32 KB], {32 KB, 64 KB], {64 KB, 128 KB], {128 KB, 256 KB], +// {256 KB, 512 KB], {512 KB, infinity} +// In practice, the maximum scratch buffer size should not exceed 512 KB due to +// frameScratchBufferLen(maxFrameSize), thus the "infinity pool" should never be used. +// It exists mainly as a safety measure, for potential future increases in max buffer size. +var bufPools [7]sync.Pool // of *[]byte +func bufPoolIndex(size int) int { + if size <= 16384 { + return 0 + } + size -= 1 + bits := bits.Len(uint(size)) + index := bits - 14 + if index >= len(bufPools) { + return len(bufPools) - 1 + } + return index +} func (cs *clientStream) writeRequestBody(req *http.Request) (err error) { cc := cs.cc @@ -1698,12 +1719,13 @@ func (cs *clientStream) writeRequestBody(req *http.Request) (err error) { // Scratch buffer for reading into & writing from. scratchLen := cs.frameScratchBufferLen(maxFrameSize) var buf []byte - if bp, ok := bufPool.Get().(*[]byte); ok && len(*bp) >= scratchLen { - defer bufPool.Put(bp) + index := bufPoolIndex(scratchLen) + if bp, ok := bufPools[index].Get().(*[]byte); ok && len(*bp) >= scratchLen { + defer bufPools[index].Put(bp) buf = *bp } else { buf = make([]byte, scratchLen) - defer bufPool.Put(&buf) + defer bufPools[index].Put(&buf) } var sawEOF bool From 60ae793a0dde26dc7ddd0a789e7b53e263e9ef33 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 1 Aug 2023 16:13:12 -0700 Subject: [PATCH 21/46] quic: don't send session tickets The crypto/tls QUIC session ticket API may change prior to the go1.21 release (see golang/go#60107). Drop session tickets entirely for now. We can revisit this when adding 0-RTT support later, which will also need to interact with session tickets. For golang/go#58547 Change-Id: Ib24c456508e39ed11fa284ca3832ba61dc5121f3 Reviewed-on: https://go-review.googlesource.com/c/net/+/514999 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Roland Shoemaker --- internal/quic/conn_id_test.go | 4 +--- internal/quic/conn_loss_test.go | 8 -------- internal/quic/tls.go | 5 ----- internal/quic/tls_test.go | 7 ++----- 4 files changed, 3 insertions(+), 21 deletions(-) diff --git a/internal/quic/conn_id_test.go b/internal/quic/conn_id_test.go index 74905578dc..04baf0edaf 100644 --- a/internal/quic/conn_id_test.go +++ b/internal/quic/conn_id_test.go @@ -229,9 +229,7 @@ func TestConnIDPeerWithZeroLengthConnIDSendsNewConnectionID(t *testing.T) { // An endpoint that selects a zero-length connection ID during the handshake // cannot issue a new connection ID." // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-8 - tc := newTestConn(t, clientSide, func(c *tls.Config) { - c.SessionTicketsDisabled = true - }) + tc := newTestConn(t, clientSide) tc.peerConnID = []byte{} tc.ignoreFrame(frameTypeAck) tc.uncheckedHandshake() diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index 3c9e6149ab..2e30b5af6c 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -224,17 +224,9 @@ func TestLostHandshakeDoneFrame(t *testing.T) { tc.wantFrame("server sends HANDSHAKE_DONE after handshake completes", packetType1RTT, debugFrameHandshakeDone{}) - tc.wantFrame("server sends session ticket in CRYPTO frame", - packetType1RTT, debugFrameCrypto{ - data: tc.cryptoDataOut[tls.QUICEncryptionLevelApplication], - }) tc.triggerLossOrPTO(packetType1RTT, pto) tc.wantFrame("server resends HANDSHAKE_DONE", packetType1RTT, debugFrameHandshakeDone{}) - tc.wantFrame("server resends session ticket", - packetType1RTT, debugFrameCrypto{ - data: tc.cryptoDataOut[tls.QUICEncryptionLevelApplication], - }) }) } diff --git a/internal/quic/tls.go b/internal/quic/tls.go index ed848c6a13..584316f0e4 100644 --- a/internal/quic/tls.go +++ b/internal/quic/tls.go @@ -72,11 +72,6 @@ func (c *Conn) handleTLSEvents(now time.Time) error { // at the server when the handshake completes." // https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2-1 c.confirmHandshake(now) - if !c.config.TLSConfig.SessionTicketsDisabled { - if err := c.tls.SendSessionTicket(false); err != nil { - return err - } - } } case tls.QUICTransportParameters: params, err := unmarshalTransportParams(e.Data) diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go index 3768dc0c07..45ed2517e9 100644 --- a/internal/quic/tls_test.go +++ b/internal/quic/tls_test.go @@ -172,7 +172,7 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { }}, paddedSize: 1200, }, { - // Server HANDSHAKE_DONE and session ticket + // Server HANDSHAKE_DONE packets: []*testPacket{{ ptype: packetType1RTT, num: 1, @@ -182,7 +182,6 @@ func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) { ranges: []i64range[packetNumber]{{0, 1}}, }, debugFrameHandshakeDone{}, - debugFrameCrypto{}, }, }}, }, { @@ -351,9 +350,7 @@ func TestConnKeysDiscardedClient(t *testing.T) { } func TestConnKeysDiscardedServer(t *testing.T) { - tc := newTestConn(t, serverSide, func(c *tls.Config) { - c.SessionTicketsDisabled = true - }) + tc := newTestConn(t, serverSide) tc.ignoreFrame(frameTypeAck) tc.writeFrames(packetTypeInitial, From 464865166c04e207ce296d9f3534c7bf5a224d0e Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 2 Aug 2023 12:43:07 -0700 Subject: [PATCH 22/46] quic: add -vv flag for more verbose tests Add a -vv flag to make tests log each packet sent/received. Disable logging of packets generally not relevant to the test, namely the handshake and the series of pings and acks used to trigger loss detection in loss tests. For golang/go#58547 Change-Id: I69c7f6743436648c2c2f202e38c3f6fb2c73c802 Reviewed-on: https://go-review.googlesource.com/c/net/+/515339 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam --- internal/quic/conn_loss_test.go | 10 ++++++++++ internal/quic/conn_test.go | 31 ++++++++++++++++++++++++++++++- internal/quic/tls_test.go | 7 +++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index 2e30b5af6c..e3d16a7baa 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -30,9 +30,19 @@ func (tc *testConn) triggerLossOrPTO(ptype packetType, pto bool) { if !tc.conn.loss.ptoTimerArmed { tc.t.Fatalf("PTO timer not armed, expected it to be") } + if *testVV { + tc.t.Logf("advancing to PTO timer") + } tc.advanceTo(tc.conn.loss.timer) return } + if *testVV { + *testVV = false + defer func() { + tc.t.Logf("cause conn to declare last packet lost") + *testVV = true + }() + } defer func(ignoreFrames map[byte]bool) { tc.ignoreFrames = ignoreFrames }(tc.ignoreFrames) diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 1fe1e7b84a..110b0a9f90 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -11,6 +11,7 @@ import ( "context" "crypto/tls" "errors" + "flag" "fmt" "math" "net/netip" @@ -20,6 +21,8 @@ import ( "time" ) +var testVV = flag.Bool("vv", false, "even more verbose test output") + func TestConnTestConn(t *testing.T) { tc := newTestConn(t, serverSide) if got, want := tc.timeUntilEvent(), defaultMaxIdleTimeout; got != want { @@ -308,10 +311,34 @@ func (tc *testConn) cleanup() { tc.conn.exit() } +func (tc *testConn) logDatagram(text string, d *testDatagram) { + tc.t.Helper() + if !*testVV { + return + } + pad := "" + if d.paddedSize > 0 { + pad = fmt.Sprintf(" (padded to %v)", d.paddedSize) + } + tc.t.Logf("%v datagram%v", text, pad) + for _, p := range d.packets { + switch p.ptype { + case packetType1RTT: + tc.t.Logf(" %v pnum=%v", p.ptype, p.num) + default: + tc.t.Logf(" %v pnum=%v ver=%v dst={%x} src={%x}", p.ptype, p.num, p.version, p.dstConnID, p.srcConnID) + } + for _, f := range p.frames { + tc.t.Logf(" %v", f) + } + } +} + // write sends the Conn a datagram. func (tc *testConn) write(d *testDatagram) { tc.t.Helper() var buf []byte + tc.logDatagram("<- conn under test receives", d) for _, p := range d.packets { space := spaceForPacketType(p.ptype) if p.num >= tc.peerNextPacketNum[space] { @@ -374,7 +401,9 @@ func (tc *testConn) readDatagram() *testDatagram { } buf := tc.sentDatagrams[0] tc.sentDatagrams = tc.sentDatagrams[1:] - return tc.parseTestDatagram(buf) + d := tc.parseTestDatagram(buf) + tc.logDatagram("-> conn under test sends", d) + return d } // readPacket reads the next packet sent by the Conn. diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go index 45ed2517e9..1e3d6b6223 100644 --- a/internal/quic/tls_test.go +++ b/internal/quic/tls_test.go @@ -18,6 +18,13 @@ import ( // handshake executes the handshake. func (tc *testConn) handshake() { tc.t.Helper() + if *testVV { + *testVV = false + defer func() { + *testVV = true + tc.t.Logf("performed connection handshake") + }() + } defer func(saved map[byte]bool) { tc.ignoreFrames = saved }(tc.ignoreFrames) From 0b21d06592a511ec037411df9c245e8c15f31b22 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 2 Aug 2023 17:51:42 -0700 Subject: [PATCH 23/46] quic: framework for testing blocking operations For some tests, we want to start a blocking operation and then subsequently control the progress of that operation. For example, we might write to a stream, and then feed the connection MAX_STREAM_DATA frames to permit it to gradually send the written data. This is difficult to coordinate: We can start the write in a goroutine, but we have no way to synchronize with it. Add support for testing this sort of operation, instrumenting locations where operations can block and tracking when operations are in progress and when they are blocked. This is all rather terribly complicated, but it mostly puts the complexity in one place rather than in every test. For golang/go#58547 Change-Id: I09d8f0e359f3c9fd0d444bc0320e9d53391d4877 Reviewed-on: https://go-review.googlesource.com/c/net/+/515340 TryBot-Result: Gopher Robot Reviewed-by: Olif Oftimis Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam --- internal/quic/conn.go | 9 ++ internal/quic/conn_async_test.go | 185 +++++++++++++++++++++++++++++ internal/quic/conn_streams.go | 2 +- internal/quic/conn_streams_test.go | 29 +++++ internal/quic/conn_test.go | 35 +++--- internal/quic/queue.go | 14 ++- 6 files changed, 258 insertions(+), 16 deletions(-) create mode 100644 internal/quic/conn_async_test.go diff --git a/internal/quic/conn.go b/internal/quic/conn.go index 5601b989e3..90e6739630 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -7,6 +7,7 @@ package quic import ( + "context" "crypto/tls" "errors" "fmt" @@ -71,6 +72,7 @@ type connTestHooks interface { nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any) handleTLSEvent(tls.QUICEvent) newConnID(seq int64) ([]byte, error) + waitAndLockGate(ctx context.Context, g *gate) error } func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, config *Config, l connListener, hooks connTestHooks) (*Conn, error) { @@ -299,6 +301,13 @@ func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error { return nil } +func (c *Conn) waitAndLockGate(ctx context.Context, g *gate) error { + if c.testHooks != nil { + return c.testHooks.waitAndLockGate(ctx, g) + } + return g.waitAndLockContext(ctx) +} + // abort terminates a connection with an error. func (c *Conn) abort(now time.Time, err error) { if c.errForPeer == nil { diff --git a/internal/quic/conn_async_test.go b/internal/quic/conn_async_test.go new file mode 100644 index 0000000000..2078325a53 --- /dev/null +++ b/internal/quic/conn_async_test.go @@ -0,0 +1,185 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "runtime" + "sync" +) + +// asyncTestState permits handling asynchronous operations in a synchronous test. +// +// For example, a test may want to write to a stream and observe that +// STREAM frames are sent with the contents of the write in response +// to MAX_STREAM_DATA frames received from the peer. +// The Stream.Write is an asynchronous operation, but the test is simpler +// if we can start the write, observe the first STREAM frame sent, +// send a MAX_STREAM_DATA frame, observe the next STREAM frame sent, etc. +// +// We do this by instrumenting points where operations can block. +// We start async operations like Write in a goroutine, +// and wait for the operation to either finish or hit a blocking point. +// When the connection event loop is idle, we check a list of +// blocked operations to see if any can be woken. +type asyncTestState struct { + mu sync.Mutex + notify chan struct{} + blocked map[*blockedAsync]struct{} +} + +// An asyncOp is an asynchronous operation that results in (T, error). +type asyncOp[T any] struct { + v T + err error + + caller string + state *asyncTestState + donec chan struct{} + cancelFunc context.CancelFunc +} + +// cancel cancels the async operation's context, and waits for +// the operation to complete. +func (a *asyncOp[T]) cancel() { + select { + case <-a.donec: + return // already done + default: + } + a.cancelFunc() + <-a.state.notify + select { + case <-a.donec: + default: + panic(fmt.Errorf("%v: async op failed to finish after being canceled", a.caller)) + } +} + +var errNotDone = errors.New("async op is not done") + +// result returns the result of the async operation. +// It returns errNotDone if the operation is still in progress. +// +// Note that unlike a traditional async/await, this doesn't block +// waiting for the operation to complete. Since tests have full +// control over the progress of operations, an asyncOp can only +// become done in reaction to the test taking some action. +func (a *asyncOp[T]) result() (v T, err error) { + select { + case <-a.donec: + return a.v, a.err + default: + return v, errNotDone + } +} + +// A blockedAsync is a blocked async operation. +// +// Currently, the only type of blocked operation is one waiting on a gate. +type blockedAsync struct { + g *gate + donec chan struct{} // closed when the operation is unblocked +} + +type asyncContextKey struct{} + +// runAsync starts an asynchronous operation. +// +// The function f should call a blocking function such as +// Stream.Write or Conn.AcceptStream and return its result. +// It must use the provided context. +func runAsync[T any](ts *testConn, f func(context.Context) (T, error)) *asyncOp[T] { + as := &ts.asyncTestState + if as.notify == nil { + as.notify = make(chan struct{}) + as.blocked = make(map[*blockedAsync]struct{}) + } + _, file, line, _ := runtime.Caller(1) + ctx := context.WithValue(context.Background(), asyncContextKey{}, true) + ctx, cancel := context.WithCancel(ctx) + a := &asyncOp[T]{ + state: as, + caller: fmt.Sprintf("%v:%v", filepath.Base(file), line), + donec: make(chan struct{}), + cancelFunc: cancel, + } + go func() { + a.v, a.err = f(ctx) + close(a.donec) + as.notify <- struct{}{} + }() + ts.t.Cleanup(func() { + if _, err := a.result(); err == errNotDone { + ts.t.Errorf("%v: async operation is still executing at end of test", a.caller) + a.cancel() + } + }) + // Wait for the operation to either finish or block. + <-as.notify + return a +} + +// waitAndLockGate replaces gate.waitAndLock in tests. +func (as *asyncTestState) waitAndLockGate(ctx context.Context, g *gate) error { + if g.lockIfSet() { + // Gate can be acquired without blocking. + return nil + } + if err := ctx.Err(); err != nil { + // Context has already expired. + return err + } + if ctx.Value(asyncContextKey{}) == nil { + // Context is not one that we've created, and hasn't expired. + // This probably indicates that we've tried to perform a + // blocking operation without using the async test harness here, + // which may have unpredictable results. + panic("blocking async point with unexpected Context") + } + // Record this as a pending blocking operation. + as.mu.Lock() + b := &blockedAsync{ + g: g, + donec: make(chan struct{}), + } + as.blocked[b] = struct{}{} + as.mu.Unlock() + // Notify the creator of the operation that we're blocked, + // and wait to be woken up. + as.notify <- struct{}{} + select { + case <-b.donec: + case <-ctx.Done(): + return ctx.Err() + } + return nil +} + +// wakeAsync tries to wake up a blocked async operation. +// It returns true if one was woken, false otherwise. +func (as *asyncTestState) wakeAsync() bool { + as.mu.Lock() + var woken *blockedAsync + for w := range as.blocked { + if w.g.lockIfSet() { + woken = w + delete(as.blocked, woken) + break + } + } + as.mu.Unlock() + if woken == nil { + return false + } + close(woken.donec) + <-as.notify // must not hold as.mu while blocked here + return true +} diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go index 82e9028609..f626323b5a 100644 --- a/internal/quic/conn_streams.go +++ b/internal/quic/conn_streams.go @@ -36,7 +36,7 @@ func (c *Conn) streamsInit() { // AcceptStream waits for and returns the next stream created by the peer. func (c *Conn) AcceptStream(ctx context.Context) (*Stream, error) { - return c.streams.queue.get(ctx) + return c.streams.queue.getWithHooks(ctx, c.testHooks) } // NewStream creates a stream. diff --git a/internal/quic/conn_streams_test.go b/internal/quic/conn_streams_test.go index 8481a604c5..bcbbe81ce3 100644 --- a/internal/quic/conn_streams_test.go +++ b/internal/quic/conn_streams_test.go @@ -95,6 +95,35 @@ func TestStreamsAccept(t *testing.T) { } } +func TestStreamsBlockingAccept(t *testing.T) { + tc := newTestConn(t, serverSide) + tc.handshake() + + a := runAsync(tc, func(ctx context.Context) (*Stream, error) { + return tc.conn.AcceptStream(ctx) + }) + if _, err := a.result(); err != errNotDone { + tc.t.Fatalf("AcceptStream() = _, %v; want errNotDone", err) + } + + sid := newStreamID(clientSide, bidiStream, 0) + tc.writeFrames(packetType1RTT, + debugFrameStream{ + id: sid, + }) + + s, err := a.result() + if err != nil { + t.Fatalf("conn.AcceptStream() = _, %v, want stream", err) + } + if got, want := s.id, sid; got != want { + t.Fatalf("conn.AcceptStream() = stream %v, want %v", got, want) + } + if got, want := s.IsReadOnly(), false; got != want { + t.Fatalf("s.IsReadOnly() = %v, want %v", got, want) + } +} + func TestStreamsStreamNotCreated(t *testing.T) { // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR // if it receives a STREAM frame for a locally initiated stream that has diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 110b0a9f90..5aad69f4d1 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -144,6 +144,8 @@ type testConn struct { // Frame types to ignore in tests. ignoreFrames map[byte]bool + + asyncTestState } type keyData struct { @@ -700,21 +702,26 @@ func (tc *testConnHooks) handleTLSEvent(e tls.QUICEvent) { // nextMessage is called by the Conn's event loop to request its next event. func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.Time, m any) { tc.timer = timer - if !timer.IsZero() && !timer.After(tc.now) { - if timer.Equal(tc.timerLastFired) { - // If the connection timer fires at time T, the Conn should take some - // action to advance the timer into the future. If the Conn reschedules - // the timer for the same time, it isn't making progress and we have a bug. - tc.t.Errorf("connection timer spinning; now=%v timer=%v", tc.now, timer) - } else { - tc.timerLastFired = timer - return tc.now, timerEvent{} + for { + if !timer.IsZero() && !timer.After(tc.now) { + if timer.Equal(tc.timerLastFired) { + // If the connection timer fires at time T, the Conn should take some + // action to advance the timer into the future. If the Conn reschedules + // the timer for the same time, it isn't making progress and we have a bug. + tc.t.Errorf("connection timer spinning; now=%v timer=%v", tc.now, timer) + } else { + tc.timerLastFired = timer + return tc.now, timerEvent{} + } + } + select { + case m := <-msgc: + return tc.now, m + default: + } + if !tc.wakeAsync() { + break } - } - select { - case m := <-msgc: - return tc.now, m - default: } // If the message queue is empty, then the conn is idle. if tc.idlec != nil { diff --git a/internal/quic/queue.go b/internal/quic/queue.go index 9bb71ca3f4..489721a8af 100644 --- a/internal/quic/queue.go +++ b/internal/quic/queue.go @@ -45,8 +45,20 @@ func (q *queue[T]) put(v T) bool { // get removes the first item from the queue, blocking until ctx is done, an item is available, // or the queue is closed. func (q *queue[T]) get(ctx context.Context) (T, error) { + return q.getWithHooks(ctx, nil) +} + +// getWithHooks is get, but uses testHooks for locking when non-nil. +// This is a bit of an layer violation, but a simplification overall. +func (q *queue[T]) getWithHooks(ctx context.Context, testHooks connTestHooks) (T, error) { var zero T - if err := q.gate.waitAndLockContext(ctx); err != nil { + var err error + if testHooks != nil { + err = testHooks.waitAndLockGate(ctx, &q.gate) + } else { + err = q.gate.waitAndLockContext(ctx) + } + if err != nil { return zero, err } defer q.unlock() From c8c0290b421c479315f66c7b68b617ef6e73c668 Mon Sep 17 00:00:00 2001 From: Gopher Robot Date: Fri, 4 Aug 2023 20:36:23 +0000 Subject: [PATCH 24/46] go.mod: update golang.org/x dependencies Update golang.org/x dependencies to their latest tagged versions. Once this CL is submitted, and post-submit testing succeeds on all first-class ports across all supported Go versions, this repository will be tagged with its next minor version. Change-Id: I0e70dd95a267e08181e5ee4d7c3239a032aebdb3 Reviewed-on: https://go-review.googlesource.com/c/net/+/516036 Run-TryBot: Gopher Robot Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Auto-Submit: Gopher Robot Reviewed-by: Carlos Amedee --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 018af6f4e1..90f428f40d 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module golang.org/x/net go 1.17 require ( - golang.org/x/crypto v0.11.0 - golang.org/x/sys v0.10.0 - golang.org/x/term v0.10.0 - golang.org/x/text v0.11.0 + golang.org/x/crypto v0.12.0 + golang.org/x/sys v0.11.0 + golang.org/x/term v0.11.0 + golang.org/x/text v0.12.0 ) diff --git a/go.sum b/go.sum index a9f84de711..c39d831315 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -20,21 +20,21 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From 1e23797619c957fb2d0a7ed9ae1083fb31f592b8 Mon Sep 17 00:00:00 2001 From: Tiago Peczenyj Date: Mon, 7 Aug 2023 18:39:43 +0000 Subject: [PATCH 25/46] publicsuffix: update table to version 20230804 using version 63cbc63 last update was done in 2022 Change-Id: Ic4634caf5c9dfd97211a5dff966a3ea2ed6a461e GitHub-Last-Rev: 5b94982f4d7ad7032c80df6a20d7ac09f0e3fc96 GitHub-Pull-Request: golang/net#187 Reviewed-on: https://go-review.googlesource.com/c/net/+/515895 Auto-Submit: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Reviewed-by: Michael Knyszek --- publicsuffix/data/children | Bin 2876 -> 2976 bytes publicsuffix/data/nodes | Bin 48280 -> 46610 bytes publicsuffix/data/text | 2 +- publicsuffix/example_test.go | 2 +- publicsuffix/table.go | 14 +- publicsuffix/table_test.go | 1667 ++++++++++------------------------ 6 files changed, 497 insertions(+), 1188 deletions(-) diff --git a/publicsuffix/data/children b/publicsuffix/data/children index 1038c561ade4683e91b37e8298fc5ead7c306779..08261bffd196fd6942b4cebb5ff06e0ffe53808d 100644 GIT binary patch literal 2976 zcmWO8`9qWS9{_Nl=dkqI0;nk~GuU>uM@qcI1JG0(VzeBB> z_6N0UM-8>A@-p>RrAY0X)8av$LqKtrG2viXZot8+JnGl50;&X;1(bx9 zQ%BN&qmERar^@gJsx0ghbv&$+IuT~4D#F^SiYg~{Cd@^h3%eh1F8yi1xvFQ>dHkHJ zPJcmJt9_}U-)3UpLH7T2?FgI zK8yCum`(SGIrM;xx%2?5iuTHwPs^+t+85Rm-|#frKVu#7w{E2aU>h9}zMT%t*r5ru z=8&N9JtQb&Fa3dapJp%|q(2NlM2A2L2?@87kc=7aMum5g(HTy9wDkr(25you8MnyTj63vL>jN?_;~_oH`iP7Ve@st+ zr}RX4LC>_lrUmJTL@E7b25I61 z(kv)Rvu-wN;pUJQVJ>OaMUz%hMXm_*$rW8ZY2y}>HbG6=#01hVXi2*+iF9zu><~6d&y01AGs;)CpX1Ba*Hbu z9UymvL*$Nln7+&XLf_T>O701zn)_TCxvwjy{}E4+$6_^k!qt%{LOppRHjt-66M3p@ zAy36t@=Ulwp6S}jGqHm_=dRMvg}=y4u9LhJZjqO|+vKHqhknJ~C9i~g;#MTEfneQRU9Dt;5gqJD< zQEyui&nZ9Txy&KFo7+$%vkm8c-PR)C%rxX{`-b;(ThIF`zvcZhHy}UTMkIIJ#LJa> zB+oP;xy^`zZMkT$n+1K8c?b=2JBEfS&uWKdoad2nc3q#N4q;f~Th`l9bVywFaML1<@oAlmC8N5;sx$e6tZ{hGa$ zFY!3WmqeEHCE366CABB`Qje2-X=DXontci#jXcdC&Ay0^*8YLYJpSa%A}{gBA}jf0 zwKe?lNRdCDZRg89YWecWdcHin0iEz@L=};3=yar0Lb7iMlKNeNd|zE4KPxkk-<>51 z+?_3v@6M4V)*B@&8}>-j8mc85XS*bs4gX4V=7dW2^_w9%&=@N@mh+S3Y`@Er%6@-K z#3)b6gQg*pmr-XJ_PyH-*F1>voI8y1@|?}cJk?C#k6$GPHs>S-dG1LHn)@R&#B(nb z(p=7r`B7v(^Sr~%><`SWmS9G^cM7wle|FN6=ptrS%K>Ihv>3EzZws@re=D=K?ry4Ps35K4*$sw=fk0a)T=NIf5!$yMs;*a3!6Z*OPQ=-*x7~ycO@D^)@N#<(zb5=4Qt}I;Y7nuoNUO!OWTcjS==x3Wrh;@vi4G( z5_bgeZ7-AW^NEq~Q^jJvDo(C9CSZMs1{+jbxzQ&{p6jz%o@-2z?^i9y`;9B*`#V<2 z^HrrvygYVMXTD-_UIDwfvyfdP zvnZCt7qLt74k(t&irJ;{huEcghZW26erA_-ma-|bBWz0iQ8p#7Op(%gj7^mtXH(*woGwiskVq+2whs*yWw4*%h)giWTu^*%f)`6kp2DE53~Xon0ln$gY-I*|jno zyEgtWc3oa4`%U}}_M5!h?E3gS?E1X>tUmsMLZ6?YF!*W|hJ}d=Lw*u#^i5_Db}d#M z$}dnH>MB$m_O&PuFDz0V&Og8&?kZ+~%|FPNb{%7n_?}_Qe9y9FU6?1ydxU|)!##)=?9RTav(%)3=T2XK#1cqjBtoB($o|@ z%G3p;)!i`0|2j-i-+~FIe_^8IB}{g_g(+qhrZ_!ds<{t@nft*Erw`0DD0OK)my7PzS69t#dOZ27C|60lQ&| zSq~}B;^5SP4XjDo3O$83^qHT!#;vQ zuZe|xk^rXev|zJlTd=usN3gj&9SXa@gCdO%P89wCCp8^VK{}zL+X1HvU2s}+1J01! zaHjhnoGp9^=QNMtJb4;?Uh@JjkbmJq_iOmQ@GV^AX}IW;z#kSC{?zt{OT0T=a`lBu z%X?6z9RSt5H&nX@fz{#%HCh5TJ_u~C;b6CfLap{wsN+XNoogJ_YbQbjKN%WaQ=!o^ z9h$T=p_z|>X4foeu|z?ub{<^elYnJuk&~5~WWeYg9TcO*v16;fgdiYGZ!S99}T0Pv=8sM&_1n#*?rM)c2 zrF0Q3rF(iyQ6eWrMeb6j$3u#H21+6EeW|o3K*}X5r0zXKq6WiqNQGm^Q7L12~wY;r2hjALVp1O literal 2876 zcmWO8`CAib0svrUzRb+`nSk6zK|{VtNTRiJd7yyG8JU16mRliLD^_b=t;%||>na8l zP!yF%fdB@t+Ui#Avu#}!)CeZrS5%ZMplz*Iv9&5~*B{>h;dOCwadCeq;GIS9q`Z^& z4zVS!huE^`0kP%QL!#hTKe0dV5pkd}k|?!C6Nl&oqRgryj?$?_d0`G=rZt2)emhZX z-9en7jfADpL|Ci`i8}faai*}0IAc9YoTX2R&&HotpS7M5e;NNJao+kBaiQ><_=2^8 zxJch1F2>u5ONGtEC2I%qt+kW*&U&Bt!TN}}690_2YJE;zx4sqEGeBIQz$5DSQiP46 z346kOMDyNYqMeyTbTF|*XM&RGx}8MyGpQt*u$=@DVW1RXU~nZTDBVa8s31KJv7}dH zBIym6lHSS`(z|gP>8ng7eGQqUP?<$eHK@s{jhpc_xP=T*Zp8tHe~|%=yGSwoHz`r> z)<_JcSPBnfsj`ez7!GR`jVH+&@Dv%`cn*ia+c-qoh(iobI27K&p-MXrH8hiA?+&y|}@M@D2V1e1j9<8#Y&blbeWd+C1_t-LV zFPDvbjVnn9e-(BZ^RUCF!FO$1e2@DG-!tapd$u+BKKC)cZ(N7__@9t{+^1xpZ3BK_ z+^BiTZN?961>`V)8y?}C@Ca9iM~sK@DE|l^HJ0O1+cErze;hwDR^UgrD*Tvl#*evb z^0Bc7|IF3mpN(JPpKV{`C;ao|6Yc_jV*C$&V*3XF!oP@r;V$8){LA<$_h0g<@jLPv z|9kS8?F#uTca40(uP4WhjpT3q7V>v~7x}x*LB6)#C*N?7@EhZgCI~j&0$~Cx8>VVw!|d%~wxAQtHFbe- z!9%dvKG>A@uAl4OuxMFt@*X#=tTqgl#u|G&m!hma509ElUken0(Qe4A9O41^* zym>KL(aeFg;#82@KJFwSYKQPHo9H~8wqhMdMH`rIA02L+E*-E#6u$9T1*vgX6*vgj8Y?a#< zwkmlmTUAoPR<-;SnBBGkbMki9T(X0$F4@V}xb0$VN_Mj~Ero1t@?N&Kq=>C;*#|7i zMsTvE6r3(O#&d6}m3X+vNIX(vB_0RjBpz+?JkPcSo_8C^Qyoait;Ej8= z@c!=nmEv{&O$k=uMdO=r+-qkyl^6me1$6X8Kq1|gj8iuh`!2qi@qvt zD`oL5pw9FhpuXujMSXw7MqTasiE8$JOSPqkQ9bF4sRu_hsnJQBsh=j5Q_m-z);~|b zNsS%7MUC~gP`~%KQhyx1PmT8uQGfQ1QGchuqUqj0X(q#uM#8D|1fhf$2<3r-j3C<0 z5ll}ME}$otN6_w$DB80;hV~LB(q0Y~?JZnNPjaNtLSZgFIU|qubLeURjP$^7w=>?-ckWN6n~%?+Tm9zH?b#7@ zXLcOjdpwDD_^k?bWakAsj;rarej55OKV9Ho*?$E7b^JBslKn>JQb8~-eI!HV02%2| z$$&qUfeL|)m*d9pDm-MoK2I5)<0Yf}Cd-%{KN(XoRR;a1$zVk!2=9l1)T!@NY+X-;H1`;(b2(Nd->H-+gk zFOzlkFK4<%sZ4k73Z~oq0n^=|ChN&fXL`(;OizCn(<{oB_2%X<3z;Ev5i^{-j~O->Ll;qr{M`oRE(3&|2mo>-j|YhX z3QnwM$KsKCQt%ZKoA40!@ zPoRImdMK*?6sq!e!VGaR%uwgSj6pTb5^G_WdNo{GlMmJ6&2qJRH`I#vKz)q~>IaX& zj|Pvz{2DX-c<>}#J9r+h6JLbu)R*7}@nyJS@Fv`(zAfKW(++pmbjWuOzJZ^M-@-4% zT<`gC~Y|LJQsx>of=8Da~Pc23Nu}8Vfv&>)<(j8lJ}&;Q65|@XF9N z_&+=buWDYxtF^D;b^Hci*Sf%ZmVoucJlMc8u;B!R4Z{=Q4VDjYX$7!}^?^-V3AAaY zuw{5AY#&}A?_d*PhgJzYhL=ExmV=HHYUmi&z`Lv#KG5pmL+xj9h%JCaS_2%>7Qr{{ zVK}aRj7au5;yIy$(s?N;i;seG`Xbsn2|=A7nqm5d_}q!P)U)ktAE zfu`$Dq8XAiXom9~de3qm&D4E^f+Uwwkn=hUw%kA=Ix7m5G@($Z9fj#y(QHXOn(gdD zb1c1RuI?V1CwYM8IR{X2h9V^|P^9xEiqgG83nj17LgzSI zWceE{){`h&N}=cxh+^vaC|2)=V(UCmoZb_~kNBVjsUK2G{ZXPc043_>XsKR-mexh0 zl#wX3LK=h8q$wy(pMlawGEurT8)fQqP?lbgvPbgKO8t7YYUDGd)^9_ax;;oI-G_Ag Z1B%rnr6^x|0)0GUL2F0Oqfew4{}2BS2s{7) diff --git a/publicsuffix/data/nodes b/publicsuffix/data/nodes index 34751cd5b9d1b14c3a0977e4fa4b04db058b241d..1dae6ede8f292889cb4252aa473312fea1bac46a 100644 GIT binary patch literal 46610 zcmaf*b)1#e_W$?kNoIy-2m?_O6)Zwjp7Wf}IcLsvASMv;X%`~Cg1UVHCVpS5;A`+3e$DplxF$`ke~RpAr%S2WEosVT%r}*7N30Q%5BeZw8-;9A%2-sZly|o-Wj9haN+o7c4W@j` z-ufohs(Wn85j+Z0{XXB$VuUk1=XP8Qbe-L~3gT;Gb z8m6wI~KO$n+ege9N{d z6^I~Gb+1FI)NNf##n+%~)I!Qz14<#tsr!tHj%$xJD7h5YX$HOgM*r_tzZxN+265WEQ>?*T$B65Kzy(%kwJB(nkMP;>S zV&3;6NsL?-R5`8FFlszSM82JNrRF+y343LKP%7U5Dx=0L#D~ix8`aR($!Pem^{8RS zv6%O+v#DXN`&nuGtzHc`3`(&XXcX^{Mj+xsUE>>&K{WTX0!I-so}%A#2|W!;f(RU=x5 z;U~=1qvtU1{!-+=$AD??5Yox`TLn@x3oK?n z=*`L9WOrdLr4+d`H+877?Xxgt&gvz$Ye=H`0r@k(_d|XlYo6Vk)cCeJRyD!=)ucMI zOHJq)*{gQ*{2>}MaanMZV1CA`cI%klu68$0u2FkBQL3D-XxIsa^e@y{l< z*$ZLg zThxlbN2Em7gOdHyAlEjjeOu?&t3%p;K`j(;sYBgsO?7D7L+xrt>!Nm0mDj6T{XM7^ zXJDoLHoO^&deq@Y2N4xqEY*%)wa7CEq@G|S?Cdk+?Jbby(1op9@MpEml6?1o;&apQ|2{ zQ47h!kuS@o&TL&`rIZ_1XQfi6I;-;yt6J>cSfdtqePjO`!%SGjEIdMd;k(%fN;f&b?RdG_e2!?#7|-` zXV#Wo3#0lTA%9@h^dsy(#Z*ho4y#(y<~3>i!=^4VexT%;ib&{{2HKuOE_M#ystYKU zj0(c-153%64n+EhvM`7b-@Z=54^s-b4Ysx=<2QFZP@{DzkxY?}BDkHS=PK5&!s=^lT31V^uQ;kUC{SzCu9TzoWK z<50V*H!L;Go_QYC5Pr5>HMFu=s@RiFs?k`7T;S#m)nxqUQO(Ai-PqfLT0tv5%C?cA zvYEtI@LmSwIf5=}@-iXb2&v}RWk2#svJF`*!)B*y32%TK-Vng(bV$KAGz#7Yb8%)9 z!ZKKWr@K^3`>7!C?MGXr0ukQ>LLLRZ@7aJ_5k9$@((`qY_vsnmmjLp8E7flsA^)m{ z{Nhq8T1T16+U;m3{KJ3|eFm<74s!m4l&X6mCwkOM^H8ER?-#jqK^i>OPBzX8Qk7lJ zUsn&@v1mj#fHdiAq3W$wD_cKC&GQW+ftP?0%;@oxv#Qm`9#o6(2TSq& z4&=VWNBJM!YIUmv%SAu<$W=F{{5Jry`e#JU3K+$|bx8>|PX;hj(`@Q;<4jD2pA2CB zI~cz2NJQA(Nv@u-Vdb+Xb$Ndm)o7QXZ|MNgD-bi@p{_`LZIVu=&=sxyl!}f@V`)y& zmsPm{uYLtog=M&`DhsG9ecP$JeeLSXu2*Z+RYr9U+y~L{#$hCGs-q;r-m8c_TZf;EBNSM1tyS|14bkXVJBhB-}_bmi>N8=8pgm zTn*Rv1HBrJW4@Br;_9AlSdPhd>~3pU*LXgl8a~X4-wy>9<4;xekd5ekA+>;vUM2=^ zlG5EEjXn`n*R(!Yr>^zS$EfEtOnFYjTXX}}!c%=z%?@>Kdp4mKulB0zjOjJ%y0!)F z_&vi9$ScLl+Xx$Bf+=4PTw_fw<7K-|U2h!Pu5M@zfU5KXiQ`ZkNDpLz23 ziJYtv#vCub@KL9bjp}B@jyKP>B76&LyV=yMZr&sl+puHVeOm|4&m|(?=gn{#led@) zX)~XQF<6H+kk}=e>Xz1g{Kl)B)ZOhxR&|e|@D}!Vse4*8FdrD-gS}U=SN2tty4P^g zW=ykV1z9yM@{vDcN>np z+q%>X-Dg|Xi>+^Bs)T8vN?9Q;h+l=BAO(L=zg7*$Df-25Dd%Fha{=B*%Ky%ZBV~@p*L^ z{n?Juxtpa#?|VySj4yyuxCJ%$-r{JHm)Nf9Q7`wOgRtjEhkC`mFCyii`PC~O-w}~J z2WUZgaPsX1!?#yG_BP|(xVT=uYIr&rJ|ztpt?Z(EGO=t7$EY`eMx-!Hy}CKCM(s2V zNb4}58N;w=4D+hj+_F){yiNFcu@lP&`_*f08%*`OQCFj0Zt-0&rjr) zI5K{=sW-w!VS3f=fP9D}Pa0fLS_r?F6P9sYFV(d&+Lt-07NXJFjlI<4LG`9Fr$)Wm znul}mYaQzCg5SVgcrZT7S7V@XCRmID3>e3F)Z2X#7^c~)-tl~o0pEdEtlWl1#l}9W zbp%r&_0Ee9Q@!i{5{<}GM9SyzJk#2VOF!%ByKRYf^`4P1)qCB4 zpyvMrknRVu5^6*^qK_$yI>09f|I|97jLYdi{sVFawjjg-!+TL3!Y?qm3J$Reaj5r= zB7Bron(F*Q!2B zO*7R;t+Q&>$C2kv^>J&U27GU$W;FTLCjrSuC8sY{HI%H98PZ-Ox#fhDt|eLp*-M_gv>!Kbuag`nmNsJ-(Z$`U{fw2w10ap> z*+uSGv|BRWo7&;NLNIO~wN&J6k0rD9+#2%o&kjqLN49A9>2S-xPNQq1_=xMH_)x4A z4`q?5I*?v4N7PxejXZ>71zncxwvCwbthN#KS9Fx==&%fFcYq)whedO39az3;u;c`9 za9VP@?xd}}B>=HwD)JE;$q5J>C)g|_Ua;3%Mj9j9EhAe$BIwYtbW;2ZVHJ#=ad}F&vbJ|xEu-825JoP+c;u|?`2oM) zS=jR$`1Kl@mNAKuMCmo1n3{->@M^zh%(mT{DE(Fo<4WLquk57yI^5`40ZV?o%wfsj zT3urq>ps?M8D~BNme>lf5JqA$1<2s;otAN}$5<`ngEGs4c_1y_6OqL6^;9RhEaSIb zi_zjTyJdn=1jw?(n5zl7J}I~ak;I)2%Y^obIQQb$>&>L?NlJ;QH$gTaXGYlR5`Dy$ z*v3Fgyoqq)&5f4bTKzSa-91xG%kG^wc3AdErei*tUTfK-+iSJ#Sv0Ag>L{F>MMR$1 ziMR5L8VK-)ddr@zV-a?H>Zm@0c|VVH%DlH5xw-U~_Z=HizKP4gtqg|phtLhQ`h;Mr z{1Goxl(5lP;Sfyf7T%OqnczU!+bste6ET%6!CQHgmpDFe zK)9|6dk<4h%t9?Z%3(P$xCd>;yAx{p2z-?1YXY?Lr5%<7yY6qdOf@nvHCKgz_PPV| zW+%x$qMq)lMJ{q40fug_w@foGw-ejkUfRyUTjVJ(WJj-MTI(1LL{CFF+181-A6=G% zj8ou-=9!j*T1Ue*J|Ift18KX+2HXSdEeE^hTv8y9!=@~Z@h)5j_JPrUo|W)7c3KWG zEQnMeN|fG3ewL+vGG6*S5mhY5QEZOGa>(XmSpbTUu><#R%$pZr-u+XDT$V5Jv0gnx=NZ3Y&vPf^D*PPwQU+;C88}L0)0TRp z(Q;VNOiU#cL|*bOr1U#<&0idr>BjDWH14P;gv-F^KFe>J-uEb3b4M^5>~2J@vB5IK zeU}xDSBazK^DfJbZ4*t)OwW(#dPg)^W^M}hSY~;WR6`5NRpK%y_Uh_zKAn=!fK+;! z#QuA_0RAXd<5)s9jgsDPha1Bv#r|Ql9NuOSbo_d+Wp?Vf46t0%f$)`t zn!LhK>6A{(?Dj>X7BekJ7$;FOAHtr0DR4_>3FezI5H!#T?v1y&>@Mc91Z|jImLpno z(J1L|!17h7nTNFt>9!ncPPJN&Y###G`*N4%DEG%!%TcXM+AVWlt98)b(=iYj*<~>T z7uKRXl3)^}#K#?Q{rA-)Qs>3g$$%_8#A`8H(-|EpIUyGw11e*BFWLAMuJ6;3kV0ns z30Nt5(_=APhgh+<7^Ee`K~=T@wd5j)#bbU9qa@#E@wEP7TD5qynUF)oJ0SY$%2V@%759ru`)`EB2sFmf=ON-;pCtVC2vE{d>kLa z72V9E4{`24t^-CJZAnRSa%UY@Zh;Z}jZ*S60!)6r(Q-=rg}_MUW1x_WC{>t4Fok37 zkZ}kXjz_rs0H@_tqZg#rcOzF2CW(oa-E@zf^b1x7aQ-T4=BqH={Z!rkabkPYPidON za%$TW7*!D=Mi&gH*K(RiKfC;^2lFHCmeab<2ar!LNBvCz4^MGH4g^N@WYh|0)-prQ zm6UjJQogMS`$vPN`giOVUQlaU;Bk4hg51~wc^~1jNifRhMIpxtIT?+zMQn>@i=}kIHpr5#kQ%8j7m-yYr|fbLq-D3| zK^}-h9*qc*(s~hodKF|tE~I@Jq1?0^uAfJfHcT)XP$gfx_&)NdX z17!KAOvv6aD$;>lksXEP3K<6Q3OB+P-ZV(0S;(c36E;~ERxHY()Q|^}vV=S-xvyi=B^J9Zi;OzB(YHG+i(03e z#32uqad~P>9)_^-nay(M)(orVEceDb=FFxZ%UK<>YhZMAC^amB%fPLavV4BP_Om^cF6j4tHldBN+2Tlj1X zr5;QfMRZEC1mFb=FrW{(#w(cDCq{j^3tdE2AGhMQAWdaBF(8|8>^iF6FLCbM+P`Zt z*F#{i_Xj&(ce3W{Ha!INh!|}1Md^=$Q8FA<17kym)Sx|mi7;q%aQJ8|)iS&-_tlXc zd9t>J$gZ}RaRZSLATSwqzKJqeT|yiXFWaJZmE6f#?c`w8a)d`MCqr2soyxj2Lil5@ zAh9EC%Mn>ePdh%u(b!{1M5Z0$H2Prt)C8l8l#HB#sWBI0U|6(HSsM43L*>nFMysSm zzsAyeDf^O8_3Y6%l>^r-HGX}IgzJt$ zw7r}g8xcN|F=VMFXxlNOJH1|I>#6Hl<5Ai2+HOdAk90MpWnoCEjPsd785d{oN^$0M z_}~-_`srCX8r80HQui=ybMM7o;q78i7JxA$?aEqPXp(Gu&92g_>D@$Wt7S}bX3f?8 z>uIl3c@p$cjZdzCRYZK~D9u-G_^?}d^|||OlN=0{HhGA1--#*r`5vm1sfyp+7Z#Ij z4b5>7a@w1`bIoi2)6{=8;GIqC{wr6*Ra`46rxm|SUO*|i!?Vb%3VXLy$573(T7gR@ zmDU&>=mTNN)wAwUeVH`am&Tr0G3B9uU@rA(RMXoa|DMyoQ5hoV{*`5Y7N0?eDaNr(%*h0KOnZEL$R!ldJhJ8)%O!>v;mhzGYBvx`>MARKt?ti^5S7& zFvLIUrcG|`-PtyTO&{K5y)t4w%G$ObVN+ixIBD~4f#|ztJxa=JG3t+DzHEOOresYo zpq}5U8va^F&d-#R`7pTPr0p^Gz z;b+Z|()<3uAN_X0eZNOds_%5Ez5Ue;Lgzob@g~Q- ztaEsP(ddWIYgE=Nifh&0`ukILb+c0G=h(p~W5bwK2l;c0F-+R@qGfFcsUDrlVubaH zR^CAlWYQ{$7~F8nW`a4@#pOWTyxRZi1u;5!B$iENvUi)~oT(0}l)D1wMI<(NTsQU1~SgEyG9)1PumHiDO%DEkAIp;Y* zDu;6WjUDQsW&cSIk%Rxn&}wR&iP4C4*Pzw|PICH}{?$WjV5Qc*e<2YWvmw^2GSp#u z;&?Y=U|`=GWwMrJ*6bRI@=c8LS3ul)jE{p6{?^36yOfMfD{9*y#`tztvkxc@e};$A z*svOA#W5yoS3x}2!O$lPd6m3tUN%vRd4wT16Ui)7%}562IYrhI_tthw@=nA&50S(~ z+Kk*9$RCtSdTG;_h5C-H;=&dVG5gSFWI)Qe_&sdr!e%wSXjYgY?Tu=B>scm)P>u!z z9RNd(4zFVQlTYyG${p>n$v!nxXTvVfACirkm%hv$kMGOQUD+5sBMqP1%sBm_dskSw zyU}^hyJ~mE_UGfZwdNEUvX{-u+3BUH<)brw_8C}s&)Ep$yiD3g!f^3&?fB8_5{$J9)GY6ih}_p8_8c^V&cr$${hf<-}G%TYeMC&O9CSV~-Kc%QArW4H3C+0CU(<0PH%*q6M$SSrDalTazUDVAeTyYwN!5vvR0}O_duSI>T`{dR%vUOYOj>! z@+@tOkQasQ=z+Z42zf)uyF&iqVpUcFh)qbYRP%ZuyGv;wDaq%u=`$K3vjULWQZ-03D?s43fkH8P7wiXv?~k7n`aANyd3=I9aPe?yq8;CQ7@>b9(YqrvFr(nft0iWwoEW1ttnrjg~ z?ds&dJw2WAl0I3;%MpB>RLu5d!QS|n!&GZ1W!%XC$@p>;e)F-Hbrd0F$$fj)&IF_6 zHZW%&j)4(%ILe>aLUnEyPj{)%>-L z#C*B?&-wws)@i+jaHtr460UO9P`b5G&6g9v{LVFWRQ8dqog)t+iPjuQ_Vr!J@%@_0 zc3^7EzYuYJL^kXTTGUB_KEJXy_Qljmo5yZfC%Y4N@U8We=a(`taAHiI{NhUD$hit# z>jTTxDYhZ#x}q%WV{az0IhW-@uAwbw9l_+R2Vc&!bgk<~G%VIlM7}rK$elv?xq?cc zs&bDCs8hDNV^n2L%q?cDsUOjGI(Gf1AD7vWNUB-uTPVKL?l0$jFN5cHi6a*&U1-b6&QS0ou54+b#x zeFk!Bg*q)E&)8#kW~tK}yhCs~sRKr(lU&J;ohNHkUYvb!^(pk$_!BYe&x3?&B?Yv(s@5~RlUs^w{Xr? zmRZ{&m5dzA1NvVj&3%ze-g&8)n$;2I zyu~|W=b7}IMP4S3u3ByZhjw=7(45ezft2v@JN7RhMFunTkx;kFBh`2?>)FOjN z?EfI_5n<1NfZ-JwK7rwTK^*;`R{SCy5aECp{tu3VA`%jj(0>yNi%3L7BL9DvQ4x-b zaP0pt92emN5&m1k66G)wJO4LBCp#%>g`!sY-!P{{q)0@H{{JwFMYu$SOaA}Dr6OD= z!hgeDUJj#tCya{1|1tzL`bv?j6uGMZQ?6R%szq+Dmis?@`e+e8T7>_GO1qr56YeqP z|C?)FD!1LX@_J^)ba}lpk~^~65j@Ra7LKqK?Cnw3>o(WYHjQbJo>9j`Xc}hDpcN?PKb0%x2ay^VBeA zp32JOX=3ymcFD2wa546Cyx9+;lrA4|<$p;j?FBog9-@?)Mai`qE3VZ{`|Q0Mgn&Rm zTCZ>SAeR^5IYrhDwZ;mOOIfX^AFp${d*B9$Wi#z@Nd5whx|aH>R-x3Fn^2(Ik%q^a~7 zZW}Ih-V_9K%_r9cyB-y|cGz|M!Se^7PoZI+YNh0ms=P+fFRK1_kEE|t2k(*e1JPeZ z7UCmf1)6?_{R`vIn`zcJR=q~?$s z^5RQVXKz28tw_l$mw_ip31-;W?dqJ4%;oA_k9==tDey7EB%O+1<((@(`cuwh$<_E4 z58VI#M)5E2d-Q{ylIu3xoR~VVb3U1LuIg6jTlVu|Z!`yC`$@D}Z`=sEp$YOSRom5! z({XZC8-5uP4NbAnIM0NUwi_KPk1f{b3n^tD03+u>#^lhoo$CC$ljG`w;HwO3*W(Ph z{B4Mgdxz4<6P@VhQq6him`S@KXb?kv2i*XYo^eWM`w1uR(~(WY;~^hq#l zD#B?`)0T4V&MoGB3TA^F)@1r40c}Ke2(Q@&3cQEQi%%Sx%F>XzN`- z(=WaCZRp_pZf#zl6LU^uSPW5_+@LXMAiYG%Yi;YLN4Qv8?qUAcoY<)@zI;6)j4vT2 zmg8|GFO~l?t{h*2MW@O6UM+njuO$C6#^pYCaIoqZGK1WIzXbe?Tx=(5`U+$~L*GTp zfYaAcdM*qa^_^uycyEZzfTfo|;b6s_P1Vs#!UnyWa&bB6Qs2`UUA5dn{rUPr8`a;x z=(#G>s>U}1wY+JhQ(plmF7v3`)<3Z%r^3~MJonoE$CTVvc?%%M8TNr()eUMyudCgq zp=a{nUmL%I6+LU6&!nmPK{X9r=lRczBQ2-Tf9k)!pZ=S9eJiAIw)G|8sCSmD#*IJ9 zXukoZGQ0;~7F&Ah>CrrSHMPpu`BuikK=>n$=my_+#Fldl4AsS?D}SDnU|t@)aiv<` zEnh!bG8tTaeu#*l5ut&Pe4B}FmGxBbWdpl`UTEdYaWDj(PTf8KDMNa#`%AC>xr)jc zBznXu%YjV4)Lo2ceT!Z33Yfk5{sEBm3@Hvg$e@l-wFJ z8ji-8G4+r?uTKg#$lA1uNvAKr-8U0k#R|^Ap&)Z+uzM08;C$Fm7%=~6A-3bmZelHd z^&;(=Le^Z30^`<>U4voFWbeNXR+%_D{K0{qbfKZIz8Nl)y-8~*`le`eP@oemv%Gw|o&|Jy{=yR*X#NiPtf+Ormo zf1kSkKS;!%m-{1~%_0!$B4?@y!sA)NUrKh3k387zS{w+=a>BUl&oE`>lX8(*x zuP^!okiQ50?~B%mZ-_%r$-f~D+(i;?*V+hmnCGuy*_L$gx_T~p2=pZtto$XXGAr~= zbeS{4oY{?GG4O$-TufIiAz@V|2v_|~wc3J4_0!1B9Ys~muh|v;VB~)b+E%rkaVnoc zjoh69?0TAABcr90eNgWChssZhb7beV$j4Aa&SbC-y&gI1+UCE;X8Q)=;6CKa_>}I} zi--I}$-tZR!M+^)v`OE4>$#w_tC!Nj9t{1#?9uD9p0$D7$d&$ag8OJ3mCN0=POSf9 zT?NvF^itj@#@oI;(V;=Pa^-kzqhWwhoE(odb;Q}Ld@T&E$g;I*}U{tq|au9 z`O|80qonNL(p$MlZA1>;o3ySTj)TJbb*nMN3WARzdUhxG#d1o1^-%8}&69m7jwc*TU6f(im!DMa}|-jXdkH zye|P<&(MLHCkJ73Azfz3Z*z@2(IAPt@;4Wx1G{b~Tz*UigZ+2l^GE$oME{szAU3&} zai0Y-_Ax_WqEm9OU`FYamh(JNWp1Et6E#DyKg8+^-0j)i8plv*L*F@(aoOSq;9`@2A0*Kwd#&gJNHOSIdoXB{*16{ z@~}7VQx2}VFZ8NAYOZmpwf2qVIgh`}P__d}XKrDpTKmTD+(Fg!(oxy(lPft-Dw}+U zlXWr(a^(eW!w&Gtr<3bSu7i*=*iFDuthG~q5frmmnCC8mss&Z4QukS3mKAex? z0k!_MTFmovsdcSfx7}$R=2Ulf^3%M#Dg!Qrmw-xNdj736SRE@rtLH2}&}Dzw#Rtdp zz=kUa8$EmUgwS6@4o+CTPz-)`ucx1$Wsbvup{JEr8~AzqpZk{jka4ACU2{H9S6I&R zGE=(ffa-Z@!RQh5QNW_!+k z5w1KD7_z(?r!f~oNzpxttm&^*^dCEwR?(OGkDV$PceAM0GI5Ms5H4E>7DG;bMsp45 zdxMk=ycvcQxsSF=iCbdd%fKyJ!@;ZaA||Y_m98kAwvml*1L!N30la)W;ipE5DTRTm z8!_PaQPLT5Hwxo6NR_)0;U`cle}dafbG1VVv(dbTQ<^Dn9?dO?nC~)>+;cFX|7IxV zZvgJs0805UBl4MIPZqr38G`vHGU@+$}gq4p?N;`=>@Fuwm zyn89+Lk3OYQx@95S7amb&moW>gz+;70>AAbLsmvp(6y4`a~g>VFZE!`hRfo+7)_=7 z`>8tGN$1G~I50K(apYOu#4^&0NOJvZ$kSxkx{S#fSWozc|A|)oYTMQZf{q5+Gq;Hqz#xEik2y zl6*^NB%Aq^>Czx2Zs?W%VZSM_Jxw`c$5MbaC-%TSu7eQvb<+FuVvysQ2Eox(^%p|k z?iwj!ApA=KM=U}GKWBRLpb&+2I5I{3rx5L(Ok%*Igoy; ziNh0gR0Sa<|-Wb!OBnL3o^+IJ-5AovlZJ#-?ain~Bxb}!Yk*QmxP zlF4LoH&)JU2jm*Cn5)RRF`TOLmW_^o$_b2*KpOM{Bfgfw;5`jeGO>XW4nZS%1EVf^ zOBM*^bA;sk#Npm%LcX;^?v&E&kkkhwnFb!_bkPDS-FOA$_CkE@pj7-WIjvkpimQ$v z4LP+GaxQPvs;=fYXH~ZddEgaD*C@!260$aio~hmwd)52B0GUcPeO`^auYGNux?g`m zrytD+4kB`nu%2E+O5~bKpEm}N8v6Y5w>u+!Oi}NnHt_h6s!JS)u+qOJHH(I{2|C&KpA zdpYlpV+f?lryH63cBzLpABdWj6R67c3z-f#F9TuA0@SQqnXo#n>~)P8*gFW>+6`%h z;RsN5K8l>>3QB1UJIPBB#?TH#^zFWS7_Mu)Q$1YrxDUA~R$SAg=$6~iJrd_mk6j3H zIM4OM-Lg?V-1m}CJrerVr5@>--=`i8tb*YPhMCjPf=|tCz`X2lqZ=_`$ksNVQ|_Z( zQdRR2$!H;%jJ8Yw$xn4Hn?0a9A4ggH_|#*Oy+i7;{wauLok&~WqHgtg^p!64_$$-F zm$u!h)*Jg0k+sF4*1yKjeN_H!AQ->6Sv?V1>r_v4oe@$`7H1=(E^ARwZdsa!c>{ZC zXCpV});jf6pzGUFzvATpFt(7j>y;>FK@x||Z&1%h`O*Ee?RUdX>p(c`^g22;Bb`u}cc|xL zw|A@O-q@>2ZIFLoQ6ts`)Q0ZY>(ujxg$%i(eX7-)=z-Bj{xah@8uzpvQ+Ich`)}>4 zwf|PTYAZjzQMGM3*~PO}6a=z)wd;Q#vAlra(GLd+bt>kEmSe^8Il*Mf$FH_KC^_Ch zEmI!#bH65Qx!+M*`=sg4(4 zXK9%*!LPakao`TU11q-Q8Oru+ovLf&hz8YdPN-Min_7BQPvlsPX6#Rf(idVXZvpu1 zX;{v3!N|UZ>}EbvM|RJ|l>K&x>Urtj9@QIsJEVHIPWOT8>?YM$Fs4!UZJG#h`}tkC z{I(u)hM=#aWR>f-{AS`E`4ZKfeTX;jQ@`36kO$fHy}H%L?q~7q_(uS_0ByF}_*KV( zYHk*bmZv41)odeH{9_}KN(N@=h~=y$Us5fUPv#2WMx*eTO_1MVqZR>6+KOs3SjhHS1)0H?P%Q7Gii=YQ=Bd1}VPQ1>p&o4_zBUzXPtZ1EZds{A>iW{}#HvY%G%-sOn?h z&@Og^$7qZEf=J|h)J%qt@;*<3 zTyk;)2)@R7Y>SAzhlcw>xJHO56Tf1r$PPaDu?Tx_O=k?rc49t`T4)Z{a4x#hBVCxf z1V?_^<3nHcG7d7)C}{Fw>iaMn>rsnb4+-aE-p{|DD)V8~Qe|*`|AbN80V6raPj+Rc z37>#pb3+%xS74=(d9Dh6?*^5;?sCh&i-j$M;CQOxhs4LxO^A$y=#KZ?if-gPA!CCi zVj(Vr`52A8=qIc_sHQwr{lCJET`5*r|5R{zH^CeTqr8)n9s;Gmw!oN8wV;W3Jr5vU z_&RO=*NHb#3nRv2sfzDr^$ES$NUpx3ExrWju~ffA0Xm; z2zIwe@JEqzkwB6;wie#v?7%Sdb z1Hj)RB}osZ$la(JZt@a0=#Rv?-2`(MjOY|Ekq_@d_#;2MpNU)XU*ah86l$?Wb=Z>?v;1aE8M1Cf zr$b8Dpb?a#S~L@4HV{$KWOSo%QZ-gkG6E!9AMTPpm%GyKf?(*X3 zbDn41Dcba44L!>N(%6A;XiXDomoLeyR$#PPR$I>oG(z(b~DzRSZ&=Q z2aKmtGxx_)d^b8k2VFno_a{ zd&ar=Ejmgf!+g{M+2{k(x;b|j_n*$kx6T`(f+RzmKTM7)oIiUS()u@y$e zWaPZEBUaxL+%^9EYw#A5za&mx2zvKD$ocnY(pBu;#4>X3CM@4CmKzWW^*HH;7h#01 z#;ETCG$NB?ke^XAzdJjH&qe8?K61L!OR1%c-oLsYIekKToNDNibmH(K z7xrN}u?%R*dkE0|C@vFnMHsjr;D-EW-tPic=?SnMq6;Q;1EU*P{P&FopI3WcUXkVXehS zG>&kh1!&Ru3P=g&OQ+LjoYjHwexOP|gb!ac8s_;J4c`x-k^>;-R{=1`7#4azFE8+d z`S?ySe-1Rgs>HT~)O&#FJ_m`&YF2R|e)aJq8DX7tKR`9mA4ctzJ`&rBfyn;248Pk; z+e?_&Cv*33Cor5K2;GDEVz~?{?*VCQCUVhm190Uqrq}k$S$=?@3!p%052ltmATkGo zhhZSL1p`I%)3AIWy87RKOY}i=1b*G?5s8n3Q8I~=k^sC{#3+Qe*;h?j+vyiMHGZg4jZYi%)YA(W1?8cZwAcgUfdja{j_JtXRER znN3w6Sju1SLF9IpD)*scU@;=T76;t@dlBx%TTm`4JbR;2EyJSpaQY)@QVPpsQ0497 zqrM5uzxs%1Bf9Ygl+0-il|UYGAQ!8}ig5thh)ae_PHG|T z<#65d9ej8gA;dnwO3{HqIb@*`>H|<&KBROqM!gS$#W>3cLz11$Y5=4YA5jxVX;%-> z4nf#B00TuA_A;m)0la<7lW=?!_S|=)Tety+u>oPDfE%0m;yRqmAPhM%>Z`zBq!$r= z#Bq<3(wi_U`iLziyJ_1@`(KGMwU)@2gJodb`7MfNIlKoCqs`b4wZyqWI*L1^fekmF zi@kycIP$LO##>by2HF5rl9f)|=CFhcDt)3&4in(7Ix%$_Bs#GH-F+b?*%&%z+Zs?}-R`gPO zpZQj0Qk|PaM+HB?=)mKMtV5CUXc%YPAT>m3lp&{cRdG!Qj^v~j>c>DdeG{lo#IGUW z`@{~gQ`#Swp@vRMQz6NDQYyt|>@Y~xbA(?Y2hGAa0i+)W;;sTf_T#sp1mT!mOed@0 zMjpm*WFq*go&{gQACRi&X!Cx9H}5x?GA2_kz8aCr1(fvq7ke5x_c%lfd(yzNF+hN` z>KHyV@LMV;(8T=>xZDE`&xersQz2Z+;cnoWvy*E4IX~v*5vAf6C;40IW1OB&sXC4+ zy^Wds05?3Iwpbx;#_qLL55h`f_a@XP^kC%xpoNd38hM@<0OdEgAa@Ro$Zp+~WaPw` z0+=o^AaHI~cm5c4@gh%Lr)j9V~{Zw`$5y`*G+;eOGG%Ti{1M1DpV znUA;7I{X%1=@)MRH(Pk5iAFSvmB^1_ za@vI>J-qcZi7^!f?)%fx^`(LIqyVWq4z)I;R7S-l-% z&a1<+k5X7xbmLe`CF>z`cW*#Mo|B^y5a^xG9MwnGhT}JIBIW}plI(~aY>H*CO^!#S zR4xGrmapR-aDO5PzCSYP^9NYNf)Dv|z6uiG;Us9eiIE#zPB#!w2@HLZE0RN7lutoa zWDW@Qwpe~T!2O#Wh<801_k4(OB%j#yZH3zpS6^9#uR%8%0blZML;?pjqt>5=503yp z7)I$gPEcJ88JMf`1mxcY7JW!4ewvamiLQAUDJhu(alhsTxI7S*JV8XI0l1}d=JLpK zHF+=2gSR6W{sczsY%IIwWkkVHxKWw9#iz7@^vfV)NcK>D*$_w&TZ}gnD)}I4#RoRP z-Iq46oF{{8R=|~qM}46du5;{KR6PN(6l8VcqYy{Yk*wYcW=$dupx%376nqSc7 znGmA$cE_IHLv`or<5tBagyT!n7`u}DxC2vuxeE%*&K031*`a8jeUzUZr?1q>}^78ezoIFb>8*G!tlrz2v@ncK}642h{UjV<=hj(9QcC zdn4lL4pnI6^`blUND$;j05trce(vutgrPqaHsm)Ku{+Qx$m#+{e-mUCSc<+yPOq4u zuTU$pFlbUaQF`!492F$c2(L{eLpDmWLtP+Ez$kS3$e#?P5;=^S=V3Ikg0{d4>;=YP zUat&g<1y;n-ART{YrxU_eUNK05RxaQR4&{?c@l}rm&;YtXseV9m&p1cmR(>8f6zcC zuOv_fG2tgjh-FL$% zKDiewU*R%x1UV@91ebAnDPYPG!hKp3)hU2X)Wh{Pp;1)t2Sy17N}~uTucTAtt%nNT z1o2#shVD?$2c1N8a~;sq(T%*@4WNSpr0#w@rmmuDRAV44-58alvwsS5#dBR4sKHd! zjU(Sx4lKWpQGHL7nnzpF*R(}$LBpsZ?Gc%8(LIKMbe}YYzX_7gxe*vgiAWPjy(_>H z-IH-;PQiR=8wT`-8$YoNwUcR!RDv||2Yv&?F%X&2jiU>&9QTXbewd2f?ZnDj*} z1kg*QKGJy}=|iNf8B^b5G;&S=k%*7HQyBO=>-T$TcF8r9MeKC0wE zuRAI=11piM;p&ULz!*E#j~Gq*i6HW^pLpBQ@cr6=#uqRmN8+Q>gMsn|WJ5m-`BQKc zzXx$N3U1^CLP%C(v~U8tr6cGCk4;2mX$if6TO=!icU4VDwm)Irqa@)+@C~Zoosl z#-%ul)HWd}J5#6*m!WPfhhK3(-hdnaC)LO(L?U|%_bxxBWpqWPYa3SNHNwCi^_Gu_ z-V4GgC5ie;GbGiD&7vm9nb0yA;WFTst57()dI;a8t1 z+)p5)?}Fk#!A;6Tbu2B744sH`{}c6Cc@F0VS(HL{Ou5&0VNWj8t7^JIFsTQSvMNWT zPUK`b>b`WJN-2DAFMWAGfZ}pWtEdHr`OhHL4IAmG$LKsyfrGXU=tky4+>09s^#d

kM-i?FIu zi!BC$zP>B?7~t{G!56u;iD2sRQ6`tI;X-=Q{VH;W)36uLM$P=P4`CT_{+kgo3(znQ zsv}JoBck7kxnJqR)GytT6)R}lmr~Uff(g#XK=|4q&Se}#_VDk@y}1OrkOeDoxy}zB zj!2{hRLOG?so0Om6StzP?=4F2#Z-9|uJ-^mQa=2KmUyx3r!BM;K%o^_34MWD!AwZ$ zbQcEfV2-S4A^WBbe96%b1*1%KH(C zJ&u)v>VERqjTOJbo_;XVGbOMRzxqfQ{FmrH0G9CQ6Bzq2o8s#FPzJ*IM++F`7F@%HEuZKpmZE^g$@!Hz60}-r&mI@a4{WC=VF!^Aq5^STTZPxxE<#&r4V`_+#?MI6eoYrR(7OFT|8y zseKnw3dk0nYT@y#;2cEsnKd+}n_+Q0R*D<}jON*}x6}z1xz8@$w~4T3z=+DNLC{UL zY%GAhFQ67HP6z#5KpI~FEjiLfY;s!Had;m?&O87{?0gKA7Gtz9(FF`SuayQi0fQI% z1MjHf{m2=O7}bve1NYvt?Syke7&yf%1k2tq-U7!XS1wl6Z;&JgH8BQg<@xqE=Ba1)mGMM^=i3!_1zEc+6<QN<{SVi5w4CA5yFKcha^75wo7KQkMwQ%j<-IFHr%b_?dKE{(?)rDi`fWPwVTz z@Z$eh+<8aWRUM7~oZHo9$-NM;1DN(`rnhtZy^^l3bk&;!#>Di11EGZ`S=kmFmZls(*z!j2&!{uHnJb5Z|p3>4|Hn-4?s$eS2V$vZ14_l^OS?S#|#=a4Vm zn{vzhV54#g^wh$lV=#fzN1)fx8KI1R9|>pi6c-=X^cp~$P*B=*-G_rDs3*LJ<`RDv zKb|b^Ldp%~NXjpTet z)9DypDl3ukeelsRT>TN0()ZD+?374P63OPTbv+mMl2^7i>5&fI-l}K1bPJDeB-PMd z=`CMjbda&ZjCd04Ls~eKB^?D+ zb^>N@0{mCsB!?XFf1y-n{4?sUI6#r6Oc4?(YR}TNpgj)$*p$oKj2GLUi;#oU7l!Xh z*tMBVpLnp5)!=rEkdbPf1dj}1*%2iXio3vhH#$Lb2ba5ZK!jBE8&??9ayWdy$@ zRnL;8Ll_oAKNDQlWo`XE{`CxTf;z2zAof*I?OEu|)Uv z!Cjl>rikf4Q$LCF}jpTLdF{_PZox9f`H@zQ0u>1#D)yb7qg%22UZfPDblKGLhY%0Sk*5E+|kZ4}!}**2ukNOh$;@NGwHnJrXW zqn#sN6{2VJ?AGBbOWVtj+l=QEvK0)Wh~0VLwRze7EnVm7eZIR}o^3DcbCe;ysNQzc zw)0hFyV&sZ7M@J)Di_$|l=HLSKEbZ@o(6ezcq24tKLw?uI&R^q=H7TZo$H_2N*(Hqv2AZa_+nRWefc;d!2fsK)x!KzheA2~ZgMzh zMbW8jZE_ytB~_v^8FJ&m*h^4^`JImn7#RbM79gWE?=#?KzuRVfCdWlTPrNGRUy_CIz+;Sea6?eD|LSgDP#8`Z_7Yo8vj5& zraD2}$2xvImP+@M|HA33-%Uq!*zHf%^V@FN?sd~=>#wBVlwQ%6AEYe4ryYfftM|;J z?dQ!KKoPWn;g}{wui0rWq!CA!4Kc7@PL-7}1|>YE)7aT_MX&K~OBV^L8%jX=jH=KR zwEi=(;+Cb9(Rd{}QlIe}lu7qczK4H0*_IOebqJ?_0134oz6GObGWIPXc(#}SvhN4! zA^Ol--GLB@k0+rj&<~K=#@Lro4qnYud0w1vmy=rZH1zzFIsw%*9RDv!Xn409hGYN~ zIWGqHhvF21_hTw8=kTp!EL0j#YzIYVfD+3mAvJnPcL@P`P z_&0k1N01QPNw!Flm(*u)s^P2Ps^P?bBr^Hda2>2Q+ztr!LA3f;9c24x5WFvtqvj(x zsQHK@>nliYTHlSH%PA;$eLFe!4MAQ^Ad1&`m-vF$^2IU0{}yl;0+uf!Z~4+V;66~wAC%We zAR6$Y&e+coCVDqSW8d2l>for>9?PQk#=tc7(5d8_+qT)pZzk*7llBeiF;OQaiRGe%rUkQrrRZE1R z9J-IZfv3VyIcor2jf|48lBZDBJUG3d*Vr;pN@OB0cs2>8ecd3JqrXJF9m;w5gR+?& z)YjpGH(O9%hF--1{?5FLfvHDb6d@|n) zC_qig$;g)c>40eli*Q=WCdlhwhn}uZJPXm(Phq3>)CfeQXuRabRj~FRg$JLB!QWyM zs>DPin|z@B77c~RQd;mDQY{JY95Ub_yc3>7`wa1PQI~1<%6$p`eL!wJISVQ>>0Uk) zb_2aJ(-Dn9uX+IGlpKK^f$M0~wfpj|((N-HJudRb&Y5F6^@@wytr64N6Kq8r*RF=1 zEXV|IrJyR=_7TeSz}ox{Qa|IV`6*Zn&75Pp0&A(6z-0rd?#xcp_4*+&StqyBHAv+J zk3{Hr0+i^zF?4Vw6;N^OD!?5+z@2cVf7^lM(O>*4w2=xROzVEi0~o>ReA-Kz*^&6K-hh1DpVrY72!`%T*ZFRm?b4&ya+( z3gO@gUTaRFjG8dyYvS^BIY<0!=0U1v0HQT(;jd;rRB9yjea**ZfX@K6C%*s)vm>B3 zE}()OwY9l`j&8tx^7YkQ_K|x+Euj>WnWkJ~l8+Bh5qGAVS z)Mpg{a#3CVKI;K;r$zmt0*=fBoCJY-OI|~gBZk82%Mqr&3U=!o1T>SaVFHvJjzaAX z$ISzroDc9*WJ6q1E7zFW4YdRXXgDwLgNl>kv}<2C^@|I+vs5_LAl$*|mwWbTO*rsb<0L+tYHwgHxQ2CvtKDYt!XCd&I zsp&<;Z~8S1eDRdH5c$j9^jt|T1s{~U^I9w$5K$s3Fy8>E_G;`j_zB|mxK!iP+iM=6yCCRBuy>cI}9vSTAsY_@pK+dGQ5z6*zz0b?SU)_`kFYC)ac%T zJyP$6iY8ROfnsa_mb+)Fji;3DSNlmLAZcFvn?BhE>621c*6i6@dzRDw{nuIDQHv2e z`&l0|6Mx3JB|JjwOzYSNa z)1M7tq)g8N2t9jFK_`uMI?%1aj0@-*+ouu^2#HF`;F3|^W z3o$40LKc6On_KzmUyyAUHh$bcjHgqWljh@D&1csmU=#LN{CiBm`C^AzxWNdUBT^s1 z+N7f(uVL@T-JQ+6%IuBgIFNeNB@KH*Qg4Z|FNo0z$x(;EmAMgT&eWt?q~XcDlcA?m zj*jy-nnlJ1KX3ybB$?4I8=dtOa}kO0G21+|TDTuW}f8dIg6@($<+u4U{*2ODUXQ z6G3&SkauMH8lGOp>FeE@>1oIogfZrIz|a$@Q>P)lMbJx6InJQ)q9>3oKM9r5pHZP^ z5TKKazyf$-)V*LsTQc z>7q~Z*Ey87REi9HNnoz2F(#K91PESI>X0YyAVu@B}$QprbK%OMZmtJJ#9MQ_tn*d!%bM!e7Mi#@2N93Cc(1^H(1lrPD zXB>HaOHUt19ei6Vz{NMmFC?49zcRdl^qQOm!7?ODl_|z${uSVoz*B|tjkUPZI6O7%M=+4A~3y zM3ym&d(Ly1Bg-XPqcOS79LcLb0q$v~qBfxHz&KLV3rjya ztWy?p$3I8~c;`@;`H%86=^=P7{Gc7q9(SXS00yUVi8EH>bW+jwNH%%f`>2<)U)32| zGOcN6v)6=mebAim!cLFr2SoxC*?GflaGEaiaS{TLwE^bx&3}%(%G6|r%hkzDy{9!& zK&mm?3EqiqP`-ioVYMQv_1S8=a+#4I zVXb*H2`wq|T33-?a&e zG=6N2{EP;&0f`j@;LQm#E|Do#oeij&2?x3wX}ySVQj@a2flBZlD5tKXjLNsjTUFwr zpcBa!nCC?=^23(4wZ9C)>GLQowKq}*x5I|URSUJwfrBilRK5aAYT*#S4`+#Dp9Qbt zf==4?*es|#5Bb8oB7D2g1p)tRp58*#icbV@I|+%4DJ_`fTglU;YUtgn+hM5ULZ}2j zMg&KQ)Y_NBh(Ep!4sL+d#K{oQz{_Pfkx+5~$nkOg(3?r#{EBvRoJMumiuu$OQorRV z0+O$FQN|JQT=N)2NA-}c=GD32-5}=pDBr3|eK3Aa2lep*^nwdYFzOd6LpL0x?>qhw z#_P7Ysmr|}UwR`?M#~WY8uH87ytW&n`2!3(Zh}!{w@4?Q2q8s4&TFT}Sd;<+@&U}6i%sTV0c zvDA%}%P6gU8QGF8i0>Z)FV@ipa*}L;Eq&m_Hab5@f`rj4Squk_N9${VZ`G|r zuZ^dNSHi}&K3Ee|)_8M)UA$T`Nj1r9Y0S9qVBEieY~fvyvg#ekmLNy?d0yj>y7;E+ z$y-P$dCN({fdX!vgNC}mOWZEtU0y5RAt53`Ga@^KARin;!23u@t#JW1pvk~4s04O- zhFylYlvKBmY`W;JjzWEyy%ayO1Lde&&+vrS63Vb0p5K6!y)LB9KnHgfPf&PO3gohm zK3eU;;Ay~iOTOtaPe)|Y*?s0u%D$mK^seN{N2L12@o3j!9fjIQPUF?~#TlIXXOJ4a zst?&7&&4x@v7t<4>0wni`@W9Q9V>^~D9>#l9v!MSG=7+F<+8V9d7ecWC7Q8|!R|>^ z@_$vhqGW0)Pb8A@hMhy*jrMe-4umyb&r>xiitXy{<-N3nlF7of_c=;*AdNkJf4t1A z_)l%4jH9u&f(NL5hePx%>vRw`_g&XVbIvdU%SE=dFn{Q!C-LyAM2(MWZ-(TZDC^T% zGa$;qVpxRYM4!3eOXXfZ$uPioF16s54#=^YZw~2GoNv>db+D7$2pjobpyVA(g0W-3 z++VlzN7T4Ngl$m?c&<>=mq(C(#tOYcI`Yd!Zz!1Mq}f2NM0G&Ru-+sLpHS=yzo@7#!_YGb*lksf24$NEtDf~|Dg#;j?u;UJ(~9R2oVg`S-DweImAg92 zFP37#zto}448GSYItI>39;%rkBZF(P+ zGw$Qg;nXa)yE|@s9A0F;fy=OUhp??ZnY@85g@@l}L^H?g`L>3_{9J21x!i2i-O zjITyCu=QZkdCuW7GVH)M8ohueLlL@lP0cYK;QzLj!{XcKXvzPFO~rvDbKJNJE74mB z3NNOPir$8yj9x?amytJZ?Yk!og6n8oT2S)1EU5TVldbHDhd^0bt5}ML!iQo=>gL?b z!JF{QF5JzrWE&@oKI)*19tI*s2bW@h=b~bdc%(_Qu=XM+PwKv$vbyBGp44I&1k`D{ z4f(S!L(k)$Awiv%@4G>=<$D-GiRa3_8v%_A_~!ZpMNarIil;dv4v?9}{010O=QdW> zaGmeN#t)7RHJwG_>b@qu)y~sgrXtjbnVpn2{%yV$odV_w-fN z??f+!uf`*dJp|hr@k;@_om){hRMgh&ickNxh>Hx#I3isYKXx*j#;zs#ZH#no%) zpx^Y@SN}c@4+&~HdWVN@Zqe>ZjA4&#=|0re$~XD@RHJc-9J+%}`xGrBZ7RF%KutjN zDyQ~t5xWnyrc;n|6%*gCzglUreoRHb*|KH)BU+0^8{NSlRMNz&`pMw_5KaD?YP2pA z6#2dO-HuAMNNpNAtYz07DF05AzNIUNa~R!D^Ax?_x=_ya`}U>=<&Mbo{nO(X&{65k ziJ4FBr=zU@w2i)d6v!yY^lKS=+{v(ela8{h7fd7`Acu;InoR$SfMb|(4vt{yE?2Z3Q3_`427bvke6 z+kXxwFRQ+KytwFenuc=V-{~?#DrNO&pr{E%WpPqji~b3o?P^r=#@9|dDw)&Li&dtN z#6=e9C~~*istwtfim3NdD*x(WgQ~o{J{k;JLZ!2Xvr(nMTu_`Fz!KX);$6K zV!rGJ`PG#??GzL-INi@x_Iv0BCxsC~rj=sfh2U9cl#)BTpeNRDy^hbZ9Aryg3SMKF zYE-rRXYj4!oK8qdC*08APmaq4$jq2Eo^PeMqoLRobQLi9mJ|n-4u5?uMEzgzt>PgH zG79LJE0@7brToMNt{@>*45{Fq6xp;zs`feL4Ibx%zb66u7ZZ}B0qQr>o!4$VO}iT@ zQvHjvtA{5P8J}oIdIWhy9EM!Mb~Dy1qte{vypDS9X8i8U#k9+PSP7Z%ChGNe#JumIetn{*tV=2kN3S?RV=Z#18`ewJ;na+V76I1nT0K0@ z@peY(l%5EivCwZ3UoYlzz6tk51|Ypp@G2J;aaa&umZQ4dYf-EY?uu`rk8-f@0MI62 z4mEpxDR~FiU!h9cC%_IhdtAQvzn0b)ds6>ydd(e!9#AZrx1uJQXm@{cb={ zsY6T@Qrt>nCM_8cg;RZIvS%JSn8h|LtTsD0sn)QUuH|u+14zqzs_+|s%%xtup0%l{688DlyNv&qnilYK%bGS>m+Jqv) z`v%SC-kVy@mSDWsY*|s+YfiTg0`!dOHGgKwKl_ONP-4zSo#xMas$0!di5qX8+S3*= zPxEgel?i9_v=#3=%o<_L?(#j_);3Cla}N`idi4&6x!GHfQ!6&}*I%c&ODpW7W?C z=Fj8jv>I5VfPPe`_r(4Ge|6sH#IDBo{okWu*W84k0--tlFi;;tp=CgV=c=`HhEQv&2D;#E!TF1kE2Ts zAB%&k$)yF974n)e=^?@|)2TPL)6XQFc;%W)u>0s$m~v%noKzXGq_y#+n}TFtU%70G zQytVE>|rqR!XR^k%XzAmAE)UKUQ-uRPb z*~1d3Cx@;H*T|-yvUQy#OhYzXFM9b`&Ba3$xdPUzQh|wyq@tm6-#O zpTReK#At2jTf#xPl_!f>OJFGU4+w;_06LVcyfzE*?-3LUc2ao~ua#C8Ovd?^kS%G} z8J!Tll5g>qbd@35iItl99A2d_7b;EN6n;A8R$fgf966oT%HPKrmYq%EdK00p?5XPX z1fy!m`0)>~E)(fEQU7-2)iL*+LHgCNp{#jzsGHWVD7J7!2ZhV1O2uj%MTMqht+J{O zCTpLQr%U0bYB^$6Uj?X{(#4qTOa<_ql3|`+Gicu{n7&Q$DZ2c?F?oR53n6eA+aPNH z2>JTjwGg;;4TX<{tKteqD8)A+bg}FoA1P&w?iMf)%46gl^Oz|D{ulIgIo&P$7|l;t zF&@1M*>oB8N6iXhe%vJ8Qe|q-(TA}dtJrDIR=c+q5u6jn7n`Dp-FbEWO@HUC>>#O+ zw+*W)eXL5RFw>_3bXHe}_GGniRXJ*q+b+?A`CTPk2UM?|qB4_Bv&k|K4Yxa?iaJlqxv8RVnAhWTmPq>`ImF zN)ke;Jk+67WgWf#E9&VT(xg;@k)o1}309S~=N5-jWrG7s#s9Tf$P%SoyXurOzi3d& z&dgd$%Mr_KpsoB-+7jo`R#MffZ0^%UpSwZXT8e4&-eoFP(_>e5BdtZL;zFNNZg+>W zC+5183OeXD9`FJ(+N@M~JE+WfvT_(F&|9z}@18xvrhIXFN((+_ea?XCXxs@vUeMvff|nREqPdT1qMS z;lD4aQWBrFf_@KT@vrEpXirtCiE&_Y9)w#3Uc`ch4Pd#wUZriGhz(;LF1T)Yt8^n5 z7{%vM+IvZxQogf$REA--!jgj1CHHly{>Euk12<4D8il^1QBB0}Hygy+Np(U8FfK)D zX&IQkA)m@NhNIMc*;Lt^Z*wVCz06btjp?=U9ow!3Cd$y~DM3|W8<<@^=yOl%MwSWOzy zs5uBgmG9KiTVyKho`_ZD7^i|D_#S~UP9-FjO<1eC9tHkYR70cGQ@ZG>dF@I6n- zcm=!0D{kU)l!MZb@Hvwh0OvR_&z52%2?QHz64m3k%a&$dNJtuS#Y*&*IW~g}?Hfpwj zCH9m}s>pixg3r4bmhk3eHFj6Ho`AXOaHqRrE)jjBlE8Njs@y?H{ALWfZZH{??_eo@ zqE#K3nBYtNdSSb6tW&`AhIs79|UzS~V9y zRdfC75F_AHha?U`!e5D@!tDSr{?J3#UE0W^HJ@tv%oKH~aY(zG;iAND=Gy2boDTul^^#8=zxgJdDSVr%?~`VAy6cy2oH>cA=>$+w z0+8lefHW_{xOvT1$UQOu>xJcWa>H$<3G|v z&h+%JRcE%uq~nqX$enHKY}eCNi_dANn$b!~uB2p~0U-Yz07YB9jODWs;}ZO>sc_@l zg>~v2!vIV8&3eNAJ-y*K(H9=jNMhcbg2ZC@%AO(yl|43A$D=4kciYLFH}TCs!L82k zn&4FzxK^6naywF5;zTY8)m{xamVLYE+qpgkDdti3eyVN}6Gf3jG zT9Vyycxh;umR|={|c^IwLrYwS#iX+1JoEIVzMeC3? z4mZ_BovUiq#pbu|bc9;fd?Tj?%{L)de6mf=-;A|OTDLAzmpXH;>N4XPdV^QtrLzt# z!HMWAm0T;`&hQj367mmvW9i+>nm7>V_>yOrToQ)vB)Wmv*Qt5~(ga&h1kF zFzj^r9_>+A#>!jORb{!2czLFu%(kJvygy@JFdpBWx#U4!7Qrr&kts_~Rad<^0pJlG zgZt`mbzrhu;9{he>&{M$cOx766lQm318q+d7te0mJg;=B1)GaNP%x_=eZ!^WYxulJ zBI})kQtvSiRU67=O1eK!rg~N?mALX|O!s1^L&MGVfwkZ%~qeaHE*TSTdR9Zvh^ zn5r%@A2H87VR@BQ@gJBZ1oi%Z0kZf8KOM5(2cE;4^KO&jIRf=X(*Yhl)sF1KI;>p? zNY793MP|U~`N@Xt*pYp*y9Q7)E(f!BT)nDy52#i3Z3F65gOP5f zBONWqKx~9h0fuu4HhfRFsD{K?dfij%Rb$V>TGeC>K|$H^053ZpYi2)eRBgAbro3uVe?951p^l!%fHT1dybcCk3k?`Sy_i_f$ zcQH094l~u##1vfclSs-~)uNV#c7Uq#8Q~j@AK~Hfx%V!jG?KW4CD>tqH>9!wa;}j2 zF14)nuw=E|dmf6H6fxNzo;dbPqCw5k<}<@i=`J@`uQ z^T7NJV#UKyP*n(U=ZT`~HB((}>k?M9c+h{DzRD&P1t2>N-lq%+c*?87eYQ|{4x+9U}A_eYA zChW(d%9GiFA5Tj4h9tGdC~m<)c@Bv9Y>>((a_wG$ab_!%{e1LA&aZ=9RHxP?R!}Wi zfoyRA1{PcY6<`!+RLx8(Gwb@ni73>_g)0`;1cQ>B%aQu)|`y(@}r^au8gz z0?b=m)#mMFyR|i1t6Gc$T#U^3C@or!4RfxEjWKqRx*)O39mIZKn`+tIU{$ToVK5hY zz#O}hj`BfX)!Na9jj|g+5P6|awHYO~s_oUc&=PsWP3~4NQti=>TGier0V`hEt~#PU zF4bvxQB^VpEoDPxl{%=MF!>k&qtmWBTb@CmnMH=>kH#Z!Im9?W2^X>&X&Z`|zd~$S zI#gHU?poDt?y!P6mo{%j1JEvQqH5xT=SQj)vJ-n6D8*TeyAyk0E?x;f@B5lWh)^Xd zP-tT!y01;f>2F~!lEt!cS3SLwlde;2swX@iiEzxLdRh;lBd^A*ws_>R!2LDGi|(kS zR0WH>$^$vYiwzH@{OjFnOY8Bi`0+~zs=jE)nS>3})5ZW~AQsA4s@9EqIqI1!}Cgt zdNFahRqZsc#E|=pI<@P?ATCtMEQ^1(2rUseyp%N-wjlq^(kt1_v|KRsTMW`}ZZmx5`wnIm1@<+74M{BCptpp@?6{bGXsvrT>Kx)@or z)Ks4)4xr?E9c%tCP4!tK$;wjj2__33N1|%Ahm^k)%+9Nbf%Enb^||qCt@=DM*o8^i zor9}a5|`!ln#Y>t+FD)mC~M0NE$VmUKJ-;S06ybBY?Q1>QonCL8(H(@B#ScMYN2BQ zy;XORNa1OX*!Y#$*YxWU!W1vxiyzLH>n#?evKB07wOcHSwHR`5MxSAAhrDW9tch2> zV3C^>W1X8U-fp)fRbAd_NlM6iUsZro{|7#ht^{-53t}i?T2fy;tr z){@b=fvW#5Jc`SfAC6)?e6x+wzd)08OJ?3IJJop*&%R``WQy04`Qq6~__C02|D=f^ zD0V}3qagpA21~!vsa8wBgx6~6pYV#RS@cF`gLI1r zh|9L^T*+n_pFv#WGg%qpXC(bL5(c5#_#kx$CyIU;v9om-h=cOK8lv;gXF#UEco)~_E-idT(lKl)Jd?1 z!xzd$B9t3o8Z7gmsss0ptR%}2=XywDY%K`J+bu(5-_jeEqlWixH0S^9whT?YPH*)Y z$(CW8FKMBa(*U45lP$Tf*{zn`#CnZmoj8@<%y!0I%hD;S}KQuD%H#;n&Hec3c8C`yMn`MkqWwne+9D;1PgWf8h zk5tr8$y@2itz(mk>8%|YnrO2e&=pKU%Vc2q=e0wwuVZ9bhaiag?yj?p^=xuk#wN0B zLC{WM{oN}*%gQ2#CtmbBU|2X4NFs{2PgFIns*1+2-W%BLxogN%MH0BUuk z<*RynZ4S#pi5z-e6;{hQV;^vheI`rz3G}+<2Cd}zcFVX{Ih^Dl2sHC$6a?l|ihkZ; zIoLImj=+n_mV*-+lxoJ|pmUlXYe`+O%m+cib*PHG=dp};zbjMYsWzNmipkKVR;;zu zS;i+?a3L@QrNJILoB=D=2AGyZjKi#!LlPD=mp=7a_5pFcn|z zr{nNcWcPMj4vlZIu_Ic+QZ~B-@&K9xRW{3^i3plYex{nY5`6LM4$B0;JlVv%>KX2H znjp*UmI<3*hRV;onEsNS@RK$n~!)cN489IS&lM|76xjz$8uE5beH95V?eEC zvVSh3SS|FvU`_^TJw7Wul;k@s%~({7TX`36l3(2C8P&S*rZ0 zwU(+xKNLh|t;=_|(>ugu`Yp3ll>=^IQ5~g1+VV@K?VMz+_4A_OQA+ujVJ&}Q&{7=< zqs4obhw3ehAvdEc{{iWJuo1EzX6ItNrMhLbmGM;_tQPGyMm3vo27Iu@-hn0djswlr z4MeTQ2j*L;nzwQTUL_lS)u?t$jj@o9s*jo=9q<`uo291Z&RUG;(B^%+-ZH}*bEEXF zEtvca^pVFpr39A9MoO+*TUiP|1lcME?E{MEsDR?&x^1ihFGo4jn0_qEo*$H`-R4IHNvBD3e z<8z`>_$$>SM=@kL){4rJEvlBbO|TSg>xO)f(qbfO96Ds7iimq=^@OW7bWmpRc=wghWsD>EQ# z5G#9x{j%(FOqM;H2T8;r9jGeXvJ|qNu$S%H1=&Ze%0B7``S%tf!t$*!|0r$0T+Ilo zYazBcBo*26jOCC47%CrH067x%`(~4nLb@In|gBUo6KCIXD?r zi`__k*lIZ~+EZ&eEwPri(1<3>>8?9nD3CMtaGp(-lgnUzm)84as2_t2Iqr{e5M5)D z37KG3Dy^Fc=@$xe@%DFS<44*G?-@*re#+2BrV|qPX&@~=6X5x)K~+-yBJ0(lfw=Gx z9f5~@0D1*4J+(dLWMwx~=(bccC!c`n_k#t$(BbkD67y_e#9ksU`V~&>7e=AR!yvnE z>m*F0GAY^W;ad@e+<{o{tBN70cRzw5Pd2^&30^>|HgOu?ToFL#zebzB?e>Hi54q#+ zy{Fc<(*>t3Cc4Wzuq%tX@dc{fho+&=AeHrfVsII7-B+SYUt$I1NL+O@EEOG;0vDjq zWhHc#Pi(<+d2laGrZ@gz0KU_E(6W>cL*A#9#jaybJdCAOKS;Ra@D-g5i|KEI<(WFL zbf7AhpiQ5Az4EB#m!qUU@CBMts*hmNT_7!dnb7GyIl2%9o)qF|j3MQFS9D2%KaIXT z|LTF)gRB#zHyLyaNJjLvxi(VO%%iPI`*TD0{z;$QGxTLsU;1c1*N+6pD(~(zi{)^M zuGp|lZJ<;@iW-kW@*W}e47t)XhEOs&CtB+tfJMC$R4V(|b|FL~2^}~iw^lkXRVTSQ$z10kOa7Sz^9KNTK7^{k#jt37{U!BsmZXF4)nrPO>nNSo zt}+rbIqlgQanZ;v3Lazt;jFBle z^nkjnlPC?6anqUe93bO*rIF`1*C><3bO_4`I>c^P@E+oe8wa|O zXsp{GKW(i~R;NH8vyx}sq=T`c-ujQVzWLTC))z?~KmB;)lEZOM1IhZwf}GR#&Pp{n zdV;?qGZwf zpS=Z}Fkba*GI72gyZXb9sJv&7Z9u&ibB|_rSL_V3RXzzm^HV=1d1%$Xd1k<@*B9qp zR5{OQiuWa^_XU`HmzyPwzFu?krghRLv8sLykK(r?rn9Ny+(s6Kw<5GTQXoYvmhqoq zDZ3vmMk|3ZatN6FFa|zQ(*tHnJww(%{rFJW1GE05D%h38F8&@-^RY6hs+kbO#wZkA zL`=gc0n+%amA$7QM1Mcz+$SZ;tN2DdsyG?GYIhO}TI=YLS86&q#lxu17=>G<$5Sdf zCzFHHD_x)pgFat2pbB{nShK!Wjo7SLr01n~G$D3BNry)knZhj~^%Uc!?+B`{G0XB~X!O7%`iMVpNn$hD?-3+VN$U+=GU z_}rlKonk^3K+0NatCYKI=MqTKiFJhOxF$7H&qniUdP^Rs%{UoiPLt|RNcBNPqs)^8 z=FbAySU_9;#5y&yU1H#zM29h$Hs@gC5AZO@`oEbQp5M z9(e`XXgBC}Mil3EDr-XT>`$8@7=nb;Y(-Yz2kV5|GfnkSZlc!I{wv`Orh#2jr>gCC$tm$v^WW zdP5JP%5^;?{xxEe#dW0eEF{Vvrpj#!TI4px$f3hu)=FExiOGLKf<09AmfD-FZFO|a zg5+PB1XFu$j!E~SBTsayXB72h*L7g(&n6Nr6jvS?`@-8 z8yYQA9RN|XsjCxcQ!<&9z)%8135*nh(YuWa3?(p91%?tBDf^^NU?d3)B`}o0Py!=a zU?_p11cpUm=uJr-CLIDp2@EAL{^x3@%$EPBH(4!0fh>o5Ym+tS0If;e8v#YC3UZoYkE^EJoZ1b+L}9AOkM`j=LnZvm+O}_ z^0uXSfj3%&Mb0Hod4XbBTObl4Pk*WIgM_}XRR>mi+??{Rr9i059xMrqP?_EjW^B#M($PERaG$S6xBPEj&5&Y9|O2v!z@JvR%d7LZnY+f z>fu(1Yy?UIAxFR7ji6vWb_+h`*?I3g#CrF58MnT3tPIqlZ!PmF>yppyIG6?*adNks z;45xY6ZU@QCmK8&9Tqt@NQiA+>hL=bXJPCoN0N;3eom1CL(017#>Hx4io8YcKfhZ| zyvEl>b|(4M5&lX@$JqGTc?h^4Q8blU_&in<+2SVYC}I| zWCkw>sXj?3AC;tz7NvjZ6PC$;>(F9qSqsbtAazpfgp-)|&Yr%B(>oVrWiY0=Abc-6 zb+c@I?x#?oN8c;Zy{_*ddcqpzv}qX5nY6iYwNjP6BK}0JI(C=ZN>!E--h@E1VbO03xJ;)s zf-1MVAl*n+-{I-2tBT`^o+kt9n7u`dm38@34t1S!wqVhjz;q90Fr1Nmvu0AP~HoZ(JWK4@3b`XC9)|;PI%g_ ztg~SmnT=R)i1oIJTh<17G?HEIxT_C^CG$p-L7#&3Oi}X4BgA^k4xo*^kXM(tzS2f% z56Lj3qlcV)z^YCdIh1u{G~b1*)KvgaT_wv1pEasM9Rt(}?YOG?JxMk913!Mq1OA{~ zR)O5awW?&vmrMvX4ns+4T%QaYF0=6SQp+-WE%$pNZ#O|cCD_SV0VAiIxn;?7kmHviJ6J}4@JIbo|HcYr2g4^TYw6Y* zxIgsr6jhe}cm@l=-D#@q^;41P|6>mxWl~DLy%~H#rley{7Y-`?$T^-;(il2)dMCZs zUs+dv5@Atzf%o0RUP}k_SY#~^QnK9R2bBZ2a@Ge?HJvuw*UZMW6KEUS%xDb?plZ~q zOJP}#AN}PHASr;2n@bGz(emfLtwYE1Mv9NcABpJtpKSAEZ-CJyP20fb0R>6!%cj&(osh9oJyNPun;`OBk$Sw8B0fl=kZK__h0Jz9 z&TfLp7fe}}G=}z2oX%atxFtVGTkdwUZs@oHbc|?Z**uR?$f#aSaAY6sKbokiv6Awa z!ZJucMe4T%l6DITtYp1%$PI4B#6^(f8J-M-fzK#PQ5Abe@t&#w(Os(2x1p1^rIN+V z$>Q9-sj6~wvZ!)v!l0z(Z>M)p7?9VaW#mJWK6~g$k(>O~19w6`Xa?W&C>?PmNt4z< z9;A0M$m#FoKz@ZUBfST*SW4^2oQ#d2?>}z`38LW39FjqT45TA-Jq~8Rv<(DlQOH2V z`i)!)xv~(_%-wLmmB{vcAclf(Ff{Z!@b%k?f_}e%s{i@)X0-!yz(-`*pyeUR@5`9` z(~uZ?1geH^L-VlDu3>e&7KsrT5Q7nqqblPCv}Ame2l+P=8Q;;7nM5@+)ep%)bLIpv z=dMkGJPk`$8kmRvt6No7edbqH+b$#anTuHFl4}=X@*mw)A7vO7ADyX`FBxCEL0j^* zX{tKjYG=7_&E&)HcjKxi?^70!?5|N^9g@bFf4_|j@LSXI=%0w$7A;XThDA_nU5)yo zD`DyXF0t>Aan=_fSOjENza|F$5hRjcM>gwH&}UtVTUiU?lLOz37Z>Ad*6k^fwM@T} zlbcy6t|IJ76VSo|P@NK)zeJs~dqoj0d`U-6S1QAE9O?(2f)>licrPIb*q` z1sBdE*tTn4#Kv0M?3E0>zO6{Vf`QLC4;YRkftK622+ftGZT1{OGGcfa3)wRPB;KX! z7=x>h>LrkQFgxzWuHz|K9D87KybSu3abd)MB0?FPJ0V}BF)}_)_0Q^YEG5Kz}0W2lk+T*xF?>S}WA- zx4N3~P0nj+{b13Tah6YTHM@}DTz^bhomzEd4=aB(s7~!Z)sNHX3W66Uew%#Q-GRRJ zWHAo<)3RSBm9yWJs%&@JUoKat-ElNSoAZbmn%<(UwJS|^Vo`ICIw{b#NKG$$E2>W3 zd+v7S3_jLRxr*-UQRYjv%ay0-$f)wZG;eF~L4?ytYVSeBU$t|o3M9VkQNhiYpbEWp zctAy}uSr$WSK7K&-i#x`r_YHaQ<;q;&mDmAyNFfJLNYsN5^Xv2NUo7Df^<{~NJm|b z3q#s6h=T!N<`!NB^xs5vU>zNUJ|=28JzI&|3A7D+<0hOA4kN@#uOWNrD#&rXdm1H2 zgHd}}P}G7|DjwU^t@3YsW|1l^s83ZzFKt<*iW7AKb>g0GMr)W{xTK!8T%GjS>U61| z92iEH{K2A~F$(E0x}~G<4E$FX=U-L1q-tta|4YIxO5IxNUsZWT*85*z_(a0@|HAN# zY~cU38x$>l4u<|(4NEorUrbqm4T;4Zuz z+yxc^X}x^`n~3FoZnqY_!c0*kWZC?Tbytfij&I5)YiIW3DXdMOLy~W)hh=CZ_~d-m zU)~gEtwE`!nI%ZcO|<1r?CKl!%rD>@9V8@LmDZf%LcNktsx0yrz*-MccO?;s-=x~e z9*`mrQ#muNOxT}Ug0d#EXsHWfk%wTV56ALQz5n`2p1}3nFK3aIjzz5ZnO&BD{XD22 z_Vz#2w9+fj_WD6gAN>1R^h*n0j(dM^Uq_#WN-^n`=cA-)c&Tp}inz{Perv1IW zpF_*FO`oiOP1$=VXta_W?$pdab`~Q|GiTW5{wIYE zznK~!@7@P&@RNUOoh)-tr!gM#y2~mDX6wEcn7rA?$P_Z5a%8tIYNC{}nvRs+3`O?W zZR*_m<=fSHW^S0aYiS#tLRvTu3@{C(G<=3gTuKt!zGv$lWMISgElP*~TdYlmFY9G` zt&_lFlRZ@5@#v$F?K`T22hou>wg;BU!aSJV=>N8t(#a7BXBsukw;jIU*y;Lpder%O zx6&q8LuxR;*H!%$&Sd1V@{drkSk|D=aSkMdjnukw7cAN3NMxT*?+`a%EDVWqaJ4=p z@e`uoj!5aeL=FR`lT}ZR!5fpnTt-G%jAS8bk(aE>{v{8_ix0%^pm}Umw(l9sl`~l= zZp&*{7wq^MllmpAzUI^V^zo?QbxT-UxEp~m3i3kLdfnPils?2g*TNs%>cS;oEMf$I z>0pw4!}1{SH2;)Cc6pB1AxV-O>jCn?-=FeX61Xohd*3ujmgt%C*Xyi5@kN)xqMzus zguYe#-%59jzG+F^B#`YFFZr^874tz8%yS4+pO{mw;QCDJYlmX_(zEYgUL)=Oz(TH1 zWC~qkd`rpNATf|R(Rfd*Ebz~*I60)A@z^e>zOY7 z4@Dvib=y0aEU|Cr8k4&|2 z@5yOMd}f7D1|`V>p9VhU{TLmxcIthrcjecMs$8D+z2WV<75c9clEu7lRQr-kK2gva zt8(rELBVDg8J#&*Elk~N`65{d*0~e(;a37&kG;mE8}Qob{rXbQCGT^thg8`(qYqov z!IB~F_gxcICG{s(`y9OAl9!Gyd1B5#j7J{#8BP1W>>Zw`F+T8&M(*o;OnbjvTgQ#o z-V{}N{~~I+4dRD>_b!tpcM=2eC%5#(`RlSCm(5FGE?Z+Ju|FA!KiqH3l^@Fu;j-($ zYti}qN80KT>#f82I>z-Lr6+dZ-E}Xt(f{d9+Q7K!UFh#1|6tKOaY}wbMj^!L%l7`a zsjGja@(1Sqn& z!YfnxNKt=5r|(%SP9h_!ekLcYKZLLPQ>r!hBQ|3oRW*Uw_l3Q84f~tZhrq1Au^FgH zV8;{WpZz_Y);HE^cXyGUqd=APhL^J>)#(E7|aq`y*|sVHGYRU<%{?#*)=M6(W{MDb-MAl1bq5e z_If;Aikl*=PO3+lZAm#gBk zp?1dOkpCFh-?tan!Q#0JyRvI2?;_$-IR^y#gW^AqTURG!fht}O^YG(gc6e=Q)<-`E z;2PItdV2?c3|iv3Xwl!?{c#;_UG~1Y-%CB|dY5n(yp>wBZY#PEXnHHs=div+(!oiV z4;|FJG&Cc2trpYC(8s67*3uidGGzLT07=KZmr+`f2dc8)>Che-b6u3=)urA8YHr0D zz3ys5zGIr{mA5kbU56pp%!atHt@Vz1K`m)3-;_It(A&F49N2;ueVsqB6$zO=6GQ5rj`oHX1^O3rf10%;L8?c~_cXoyf(HqmFY;-fzTMZ~t@rwt?1I^0g~-Q2 zYN5%RJ&SJ}bDmkX-z|-Pz@v-kJ)H3v_TZBJ2P#LpgHjQLocnL)WzxMpjLaO4bhZbV z((&&g2hs_|K!0;G;89Xa-{K@qLyP|K#IUzu=&58Btl++B`8Pb?jGW%cRBPR;t`B|A ztkr=XvMt0~{PYra{jOm?bwmEJpt|Ah6Y2rv$L^>`Y>b*pW?KunaTxptb|u6&23{sY zshf!IfE|>q&mxg)Fh=RmHc_nyc-ppfbz@>Rv$p@y@C}k*S>=>M(ppFmNB34Vy9tR* z`J)jAFn^%r#T2+-e)`Z0uL z!0aw{bCTgvH#hW0>41Fzvc&PtZVRbfLNytPh3LrKhevXhROxb^m1C`Aec6q^+QF!j z@AYnZeWuto9O|~%8b4U%sWR;v#v|j*9(CI*0ot->ap~0|%YHl= z^m2Xow^CnqgK@~^9H?_`px(c3F;ocd;?UI6+&*Z$M;Dp&G-TfUwZ zePX%$2yD)LLYy;&;Ea?t%ykfvtddi`kw)tILIFzh{efrI7T(E~Gv*Pw^EJ|IVNMWi z@4vXA@?6is#y5*^dOf#Xi^-?xbv{M>s*Vf8vLl_2%67yySXs2bqBnmOvWC3SieAzI z^Vgj8iw&5oJ|}916+UBX3(PGkl;o3#!bwOJUBZ#BXzJDIOSsYeNfCBuW73dqv9y@X zu9VkAm5%_j{B!(}zrv#og@WpH?UXL2sw1TTYL4-I3s~fSyz+Mu`#lbpXF=+Gmgg;J zE7tTM!g2l#i%agG<&WMd*OBNeOrUSZm=yq@$hDkn9c_9wb1fm8^k1~`K0(QIZzdgw z5m;j)&iMN8fDBv?IZNPHG9@cd$->nI#2{YBjH-5Ht+WJE)tV|UxR4!+UElZ&$OOV^ z$jjoGoW6`sO2#2V{#Zaw3uEcNSTIY(J*iEQ@jg(=dg@#av!{S8)_*dka5t`c7f0xj z)1mJz7MW-W%ri2RP&x<&W-EgdzuAREvB>r?w4R-!^wV0zisH!S9fwUn;3qPbrTl1-RA13PJ(i9TbKVA z8q0{U>FFT|0F`ipESS{vJtR1Q0l&3 zG#?0aVt#5(PfvWf+ zWJ&22$QoI+1?1OafxE02{{a$T!Jx!f;79xhsXhTQEG-y65s+q!h#gK<|AN;0xFqYF zL@iiMc18oBGA?FwSUl|zY5bokmR?K*p=&4=gX9G zONB2L9%U*dMW?dv17%wo`UfTFpcZ&ICqu25oT|{&^5TQHzuy z5xJb+${XR!zk#;+;%o+QH_Xuwt|j#{cR{AEfLu*GjEGQ#Q~`5bxPfxadb z^oJ$6l~J&ro5HmE7|o6|VM(6DoE}my!}A!iA2G+S*1r9-2hAs8Xh3%uiN|5Kufv-C z5j$dONH}&ZR`&+pZ&3Gkf8VL@3+66T_jUY$KHGV0MJj5;%cDimkqUsp_F!pEqOe$27G*Dh~*18mh9Qg6~1M zTEAsLntIrqSFaxKxv^b661u`W72?O>FI>X zaX)i;z1rYCx=wA_n$)Ns^>egWS+~G4^bE+5c>#{jN3^O(ce^{(WBKz^)MKrwjXYrt z^r^=^yhV9DF}g)Pv3WNkwx)OBbSkA3!>u-kXSb=1-S;-BC-vXANP54X(6RSCxn*So z#wTLPvc4G`FQCu*nUDexfDZA3;3gaA@exEQYYR#TTuR3Pkg5S!BawTgiI<1^3FqtL z)(WqB%6)2^dTMWJliK8-2LcB{RQ7xAYE#c)jp}K4qfI@%#q_CXO0Gvib{F~v%Ae~> zI;W0D_zzOhcPp-rtVA~Hy9V{lYtuT_v%&iosb_bLY=>oJmwGOCo=-j3Gao=1Z_}m@ zwBdr>Y4tk*^!<)R>CkKG&A!yGo-aQXLn#}n+IhTDIX{EcbsDRgTMT*nZ(%V%0a@Qx zO5sUMSRFH|7G$MEhAe^{C2dorP527pK}azpSWw9tR&dJ}$o*-oC2#hyjlIlTT=cGx z52zLggp|NiT)qS{i&E(=luGYh3|Sw7Y)FGVCe@9U$}gg~Tz+U=k+mH%2$l-@)o;ah zQhFJ_iq}#h@>p6S&&1V9^j3eq1@cV~>yZ2aw&ov8*vm4vsONjqo7D@!if;A77G5SM z^zR@?9z-;XUnES1t(9f?Ei%lRLJ+;**$KqObOa-`g=Brn-%d5;V9ZM#os=HNWb|PS zc_#TF6N#yBC#7JBQvSj1sCpIk?&c=cPiw=O_fZv_H(zFMaGO?-DCdt{-U2k2|ub%gsoJQdEG) z_S)e4kf8{tQLXk#TRlp>uftMeN3%IUjp}?ahMuM){%=Y(x1%q3o`c?Z+JW1FmVjK4 zgiU(OFSoOOJV~&F*;osnOIvXfHljti8k_5%e3*8&!p; zN_DOe&7U@7{2{air@$95Iw&2FaX0g_Hw8*xS`4!lmKdL!smKYiRA*z-lS8%0Y^3e= zWMKTjgblpgN$6M|RkQ))#d8|4cCa0#d%6L1F$e;WdvWGgX^YTS6d)~%+@wWiJE#Kp z&=J}Jmgp?1)%T#U`WE=g{{gDV0#!vJEoz25 z)CgI>h=4^pKp*#^S+djE*hRGgeFb)yi(hO9%V#Ladx9xhTniTSV>g=RU|k?Hs-mKY zNKQt=Jdmnvf&95!#C+{Q(=)|;6q)FYR3hcp#@J+a-N4D!B7X`JO>i0qQhAYsorjfd1E7DyU|zvF8X5YQ}E~#s@_Bg zN>2ozaUp!+C-9^EIgEQo(&qdX2R%b5xi=@HYKMnXA(1Tls0l+dc+MRt2);r{0uOqz zn-7b#jgUmg6R`X}^t$B>$&wWSuei!V=?dENWFhlhZW7(6(Hz)qr@D=j^CFa1cvEp& z-ou8r1qsuQsIU0J2fl5{>c5F9M-mluH$%$3*vM(zAGNTkH=oE6L?*u^ndfMO1k{r5fa}RQc;+32aP8>;@iI%yG2o zWDWe7ferb?7r|$-8$1rp5f6r385j?&L?W<~j_AkObyk8UREo*q+02Tfn=xc;qt|(e zn`*pgzccP>Fjw5%Ot9aEC3=#mf5uLj?uI$Y)}{*QHX`Ar!}kID%H`QFuoe=Ux`Ws+ zp;Ue^AWP*mTs0OF+C`YEWCR17ZD=`^-snD5g^r-A*QRJ8>I>Y!a80Kpcr9Au$D+P? zlpT<7kwZok#D7u%@*?_z_crd2ee6;sH0H>vF3E^3^uE?fV*UVgNREyEq zP1fR)!U*bbLX{!uUHqM!7~BoWz??4Jk{KWUm{Kqs5_l56NIUof8zJGB92n|IWz1(n z+!5T0H((<;!A_iCZyO&;<5g5+5loNFPm>1Zjo5{=3w$hx0wK%LW>n4(N6!&ikAl4!1?`ADo$^r z^d<^ineAw9MtxMyj`3+-$R3YuaV;!`mte>_HWkd*!xEei+`^|Q`Dgjb*lzUI+=iI@ z7?f5`LL#(*-U<)#^T<&=(gvSSW|OvGC7CO`%qR0EG;L;qP^pZuwtuw^O4 zLEh`m|ok@^7P+yzp5D@9C>bZxRyY|k^BmBZJch~OAbNuVv;^(+1~wq;GMdr1 zbrBM>pPTck7UeNoC6Btw>pUv#aSNXb2H8=*S`7~%B{MWx&Wd*$#Hy#dVL7OMKd;u?UK zko?URvl+3-(iE9i?S!NgYtb)+&qbSgRUCw0!ldA*c29~to zp`glJMZ#4{U~4uZk$*7?j3>OL&$lRb{|XlORL}lk=bdgxeGeq`5gooIKD2xab7>d_ z#Rp?IuK*WfE}o~OU)gYLCrYd3q~Q4#W|!RaN9QM_FQo^~Ly#yQD*DcBhFOj{RR_Qv z-qaRWuFd?Cre!Y=+#$E(G=SJT^(q-9V~fAY(cXF zLp5^7${5Qi7NUvrPw_%SJz^r@Co2n zUQ1i>Edo~lvklMPkn&$>3oLF0^RILmr%?4wq*PJS0bE)CjPu(ehXBYt5<~g(F;wzU zJs}QIEsbHLbSh%{jO2a>F=Gn~^x;lFOoht=0NN{>+mfuTTCo0N+omjWntAfNQ7pQ z7J18M>P~D#Hf(3e-lt@~jSHcjXpXcZTNJdR^q~k)!Wm=Mi*F~?*@TPFyjhJ<^xTHZa(U( z$h#2Zq%HtSW}Cm%Ltc>TBuqMgh3EkdIWSp$2dZ5CQDx3=NBuQ+lHn2Tnr9}1^y^02 zUP6KY9&A+KMs-HAozar#WdE%w&^uj8CctB7`7u5M=HQ5B$QDyA+>|1~z1Wark6ykC zPeeAfh8V>DEhBRvr<=Tk&>Rr;(H~)s%cDX8H?BM*LWUz zAWKWhOYj+sh`9S1FTJ~bsFK@?q8n`}m;+1oS9T(P4*2rVL5u$zY?R3F>4NV!Q~kCL zlkY9b2@CE=?Uy7kn8>@HN4WGHsgA3KfIeej1Pw_#{ z=mtv|kMtg@2Tvcj3T}frD%l=B0OQdO5V`2)&s_jA_yL)19Ezbl*?P-YqN?b(c1SE0 z%r7uq0`q`tj6@=ujYq}2=Hg9jCw|OEvvDY66nYY9=27@jGQmN$!=auiy0VG;=K)yL zpVb;q!B^nHMtB42L+4;Tn;xdA2R6h8V zcLXYshpLL38j*Oyhf?`SJ0woWFG~i&F0pYK>Z|8@Q1BgqoCjgDSRW?z%5{q$eVM2) zzYmkapqFY7F#Nl*Rv`DvHO)jQaBmaj0c^N`K|&8yNbVU*CtxTz6wM(?QC|lv{*!4d z{RJ(B{je6RLpF9ZRp0aQ#bqbaS688bBH_QG0ezpOfWD@Ssc{Ew)$?d8`34E&0P*PT zZpb~{tw$XgFU^CL-Ik7B*`3W)V3@6t&{`G+=fPMDy@g%9kw(u&m41t2?sO9?AHC6; zSPS3nWq5X@IeHvv<~$g?p*Z@&_aj>|98!D$NIgx>_>qZ8x10ox3q17753u!(Txf1P z=oc>{R!)=#S9YM~>aCF5xN8X|*hoV=QJ|0Zdg;?gwUP};lnlYR??WWaNe&cTgl1Ew zY3VH53d~IOB~jJec2t4DOo-UQ^p?MlT|H1WMk;OVFs?7?{Cn{-w35u$Cw+6BovQ4d z{uzk5U)h4v?*sUjfr9d9#X)(-@tgrEigcm$e0oE*Ot8YosphY5q$JNIt`8{{nc}Zp zy%=YE8Px_@yptPHa4H>^OmIq$C*|ycGn&yXXxj zFcjQ@$>3Ykt8XSwh#_`5w=??nfZltmq8=Rlz8M6AyHS5TC37MY?w&3>21d=2daYGP+wI_hkrHB#2!OJkEL-eeBsfU%$M)TqI)}Fk;8HH zq-L^M{$QeK78BNU1$^c^01DMZsxC%L{=-NVji)2fhWgN*_z`nD{(P`5co9RfWw;eu zj7NbLh((@3Ouq;!ksGe;zuK^?;c7ILbmiLFq;wW^vV-+$skI3`GhItp~INhO2-B5=&K@aBe&3DK8RRpLlf5SplZl03wTqA z^G7z2HU&ZOA)ZIRh3H%2s&C<&kpxSHTm0CEf*Hd*QCf+$ScMO}*Egf@JmC6o6p(tf z!nBo5b2GsTlfnE6eBtLHk?*llVPd!DtHs#3n6|tp9B4idKL4YTSZyPKR@4h2H%flR z1>-~jh2Cg`$T_y?cVHMcO8Wgwi99L<#E?~20XeB1@4{F`wi95 zHdyo#F|rUt#baqR=Gw6A_kEqy0l{EZ)v zF2`DEHxj{lpfa9-#9y}4A+Ja(d)iU;0l?$$!x!I;m)ehl-6?>SQ&r>|AE>sYG;lH= z>EnLMI5q-#_ul)SDf+c|o{Ca^jMo7R$a8#fTmUVVXmM8JrLhs>k~qiC23qvYMyg|2 zfg^H#67AQH#9g$N$c2GB7v|8n*yw#@cNba;laLM0?_k`Lfm>k1ZtvTh1k4`U#%7G9 zE&g6A9kM=0SKx=XTd>wcNzx~@5LEi0S^l~gEiPF6n;|8e>cR2=h9VDu&q!`Y%Nm&V z)1qD(3gy#heWe$Y*Rs{Cv8x|d^~VK;%*dT-oBwMZ@{ z^x-Ld1rqvYSW%d^@FfE1^bXpdZzgo<5OWDB@i8y=5pdT<|e_n72}mFZIFBE2tG-2g;GR)p*SO17e5$7HL}|q^5@M%9bTOF zL-b3oV7b7Ug-O4-75cy8&O1t~Dr@xjR(Dl(qG=jvkSG`t6N8E(>a85QsybJ7ZgfUW z7{=^i0y>e=0dp9Y?=y`f>gbFZrfHfE&<1ow5FK^U5i^L2+4tMKu#2gu^VVDIt?&J_ z*E#p>boSZzoOAa*VMQ45Bv#YNj;sM+n_PM&3x<-LizHmMcaKviOj!hzJb5X(UP`_% z(-vm2M4ojclL7nehmZwaHrdEwWD1ad4yoO7srBRO`coDsOD!cWwNO2+3d%rF7PL%Q>DqX$uO%OB{;x4z*}WpD zg346aVa}#c@B=-sFLsNQl-RH4)*c$_6M9IdCf1tpCwYgcu>s2+G#!m2FRh5Im~2shtaw{HWuWOY$;mGUW~_;^aw7tj6N zGuhxG+uOVQ9Nb*HT*+inWKUH6|f6+!B@sxx~CK$Zqji>Ldp98Z>X8nLb((5G2B*f?no&ob(f zVqld+k!JgJy9i3!jx^mBm0J0zswG`uprv*9yiNKheVO!KFk}OSDx~$$7gqNfsyTJf zkGV68C2p^yt}xwn^A+7JFxwwH%O+bRa|l^I_%_$Tv$YDkPp6-j$ zzC;CC>(Ea+54BZVdw7ngG;Kq23MJJ`rMz7`qg0z`NE|)=4HbV$QK+VBXCw2cCR6*R z1Zmdsgom|$eZhTbx2#RAnH_M3q)&<~r>eh?zwA#{RiVmTv1DWmO=dj^=fHNFwqjj? zHy}<6dYWk?Jxbp@n*#6b1p@fGtC>kl>&@U+?tJhha_o`0isx=-zLw0^n_hsT{xu3B zJ$%R4zYIlGcBSan_~43d!|&m(vU46I#IQkUW{dX9hTus(*i zGuogyRP>YCVbukUZx!;bRyLyxF2mth`v{=o3EZdTbezGe$);fK0#aw8T3J7?;u%fd ztU2iH?GMp4pM%J{A4TNvVBaA8^h$2e&0g%X4&<2 zp>D%(22qbOlqO^L-aQ|(x&qzk;qn7X(h!Y-8~;Ou@0LGB%FyK(P=v-@BKx%Xs&yal@iw>(ewI|S*C z&{Yz<@$|yp(~Dg^yM(hR3`pQF-qVogy$Pnl1d2EIppf?k6!OY+rpgN93?bP2Lg*aW z=ENEottWdqt>wz6W&AZzSZBf|a06D0#t@i@=Qo^v4WRTQtx1<&ilR`tCo9*8Ui zuZRX{EaGgXngXFX)Q-sNZulJpD{sCZ85=jCUmwr(YT}X{Y^C=T*G}xCMn6K~&&&c0 zMq2U);%Pmkru715kP7O5ji;!BtHg)g02&0UK~CNdETIxlE!n`5JitR#Kk#%rK)V0n z0W@RyNLcAjzdnyIqh0w;v z7xQj)p`WGSA5*AS^sY8%zDZni5{&mo2eh(TT`vr+ z_)RZLQNZ^?_L$avLG&39*^n)IU9Zh-e3%vpLoo>O`} z`~5201_KsCRzI9=JoR&A-^o^(Hca!O&<;}Wj20Ye6|HY=r9s`Yv^^-lT&l0hCcBwb z`61{WSWC*<1G3(U>5&mT}M4m;q z>Rd|WB?t;htL4|zv7QYIY(C&q>dZnH=axDgU+bNacqe~VAd#Au9=CYH$ z35u9K$m@48`su|l)>F{NuZApe(*kOA2f}sZS=ITFg-bV}Y14XW^FVR!hcwqU0QaTn z+;lC^+z>QOg-;{;%}nT0Tb=@j>For01RYst?I&@HRG?7EE|M; z>v&Uq>u6P<%oHlykx|+H68=1!ROD^g``dD0dKFo{tgv<%s=0G|C|!rIS#R^2)=LC3 zaT4sKp8>4A7`8fY5TG8zTAi4rY#4Y685RS#kxi_CU+r$H;F(1&gEe?lq#{6ez8wvQ zg=x3jK-OykYnJ8VDtCma`5G))e3)c6QW}4v9q@8Dc(=Bq#)?d;^$vnc?}tH9E@^6P zNBhKIeN^xS*z2`>B{w$X=r>}V#t>@g20HI|sG+yvg__W+em5wd5k1(d8rqs_Xyb>Y znp^(3N8W@?udCDlQPv-k7T%0%!T2%?UR!`6V)#|##vtlu!>_oo4XqlnefYO1*_04X zKSAfFeh2!!AnGn56&-`1^6D%=trIX?vU4ddpOXu}^=RO_6ycs8)CkCVRo+HKdV6Tb zutd~dv$9cg6{%1iT-+aEt11b_dcSJNJ71@ztMYh%tA@OT+8~ zpZsJ9jiEGl1+2WYn(>2!5UEq*)KMNft=;DXRi^eOFJg%FDZ4~=^t&hZb@y|u^fOW! zq^9R-p9RU2Oa>;~$}bHiIoTFn($r85^oon*%zWU}GVP;Kw}4xRxzSl-0Lkxw5jL_D z=3)EeyWlJ#R=KHouZ)3Ny`Dm$FI{rqqC{fcRc_;r4)%Wi!*f4|aE|NV3-oJIPyf-A zrHwPNxUC02826{e#+#kXXw~eG`;E75dj`+Rnnu=fKXP+iw8!ATd5IUa|pnVG`W*cueXYv}1$iM{*+2o!3 zDactna@ef&!SlvH^Cm51)77jfT5&|1ZRri5=sD5)mFk#4J@nX#)Pc`K9&JIc7eRJ; zyY+DAs z=yQA7P^RZF?_3P+aczK@fQJJZM`r8B95*$;i>!GJD7o_eJL*;Jl0TJac@xPd4;Ze> z0$Gj_Hl!K#+_${`T##p?!I(EOY>@|kdSWBNNzhzVNDd>`aq?qK5cH`Z; z+n~*?q?Q?1Ib#kfc&IR9(UVJYmImfGCeqC`1eqZ zJOvxr)@B?3DxHOw*e7A8;b)>+!N-&iDPhZ}9xS{6kzq_C$H|(PcN+g%zc|-;FBIN@ zZ~RDs;{>=EhhmcaPH6QslKkN)o+sPB^h{aj1=$#O6wjP@sLc75rPkXzY{abNv&_mxf=K%?@EZ90>AM8)xU82WL6U%otzY53-zrXv*3YU49{_Y$ngw6O<1c z1kbi?9iH)K6~hU-UGbREWzR z#OxC0GxRDJy){ZBXeKf~PWDLBeK{^d7h@=?WUptq$Z8x;`#Z0MS zRG=cQSJv=JvZEd7@RAWejooPHI}pH8ty0mqk!z&pQp=&wVqHUGEskAuUxQBTrWx`3 zuGdLv+loMeZ`4>*u zxSr;5oeaOo8@RH!tr^-XvUM4x!k1x^fCXOS0{FRq?SsKQbn&>`7=+uPjjqNn@h=?= za=(>=+3P`;-|p25(B4z*gY0t{R4gP_sXrLEI#I2X*Tr)TBIAcHKympZ@a3U><5hHw zQ8_53Y8y>0Og9Bbi47{z&x#}5YJ@>@;PXJ?mG42%C^%PsNvUTX>Uze(+1-en zxo>STc6J0inN|6Ej9tYiw^LdGO6}$Ap{)z=ZL*Mjnuq=_6vd*8g}=DYrYmsE@#gLm zpMz`@GNMQNF!2omY*5n&mjlr+$R=f@+Q1sFS3DR(J}$~005lwoAlECTyt0O`Y!2DT z-vNzJ@XR+J?S13hU~owmW;)vsa`s@dUk5PBE(dZSLz+8|Bl%=vT|0C>Rtr$zzY4j2 z`3DsK9F*Wis9XC~JAP1(R{k4l6#s27^~*cx-@4w|_1v%Up+F~Ucus(Q;A}hK6qp7Y z{D2n8-U3xCT!M%9ckLXwFgsHU*4cz4v_%FwP@lG_TxeDa^ zJf79xolU9CJH7IQF}TzV-f!0fZl_W7I*`~ItyuR)K)eC1>L+)j)h-{xE5Hj>^f24( z!A0w4hf#c$gUQ{86ja>{Mbm9Jd7kSh7K~w*-yduS~`ek#j?U z%Mck^A*^nof{}aufcw@XXjeC>$9Pt=9yR>4z>Du6K-~+;)=h3UKCfCyY0LxDXmKwU z97{n-26&AxQsBRqY*RgkXxd3>BtW*zPHFH8o&{cN$M&bRP!I)_Ue-qGD9Fmfw0^}& zP*g1JgUdY>MBl{uqwfmXu@LZK9&#TACH7|lPjms+gaA)306e<^uvW5NJ?L@)$kyw~ z4TL}r&h18_TtLm4aIS6TS>m%rRBJrQ{)=G{m<|Z-4B$hDW4b^Wsj@YGxO@wjguH&P zHIVrq%oO0j8t3GKcQ_)0E5UQ0nhou!Zl1Z@AzRH`5sy(z*9o9l6046Lj?NJY?KV9^ z*8e%F@>OWi_&5y83|h;-8M%SwWJC9L!_Px1x*FA@_ff&(qagGCD+-q)2V7PI8aIIA z+O{5Z(uEsddBH-zqQ}(Pbx8)o>%T^|+D{P_ls~dAS>sjtvfcQ+qqc;`YiI*B1p%j_ zU&)o-#uvUTY@{B8y?z$x0XK;+o<~WUDa4}Pa5+B*yJYp@G#{c>br9nOFQXtjH4DSK z0U_DHP<0ih9&r`x=m>S2hBSYx=-da2zdC4q(XnM7S^2#Z7k8|mv4HHCXli}bZhYw$ zv%9xp?D{7ut-N9hmA*8HpdW(9mmS0CT3mCRjXin~R(YV?*wax2zp?`{n0F-#S?`FT z`R!QkUD2b2LMAb*cSorN}g^vzP4gitMcJ43~?;9Rdc99)yu-= zMbwSI1YW2b4eEQ~5?stP_fj1w9k?VcC~9XQBkF3!E(w5-Il2+}zz&5( zw&S~`pOk2Y^|lXlRzg<#UN5OqvMvdHxaUJwlkLO?V|i93qf6~t7&MNj1*3oU<6N&o z7P*dR(Pv;9-R_l_iGo-!6qNytUG~>L-j8>vk?T1WFDVj`pcg1P70xARN_JZoVCQ-? zcsd7)VIkui%cL^prD))K64SZF!`v4F%BLXG6Gxh_A%w_#ppD+c06B9q$R!^l(mxhf z{;{BxhlFf81+^cMbzP07o|Vm%R(4Y$Tepg91h2E*_-1niR&k~VM*L1EH6PLk%0y6n z2L+99%R-{0gi1;u4#H&;Dfj3Fc*Mg#GxZ{ zu<=X=Yil`uY}r%rs~;4mzJEdk?_|tS$u96G^Q+l^476j6s02mjW(`0j`9$QbK=~&G4(diJrCcE}{5i9^ffrv{i4w zs_KIXU>CGipOcC=5$lUzL^gg8Oyd$6i$B;8=#bLq@&K=pN(@GBV(fCj1Obzu155!g zQA0x|Zh=AKc0$yNw}t?A45bBa=v?i9LAA`+s`H?zE_?w{1cT}c=vO@##j6)d_6~$s z-}^k^L8MjxO&Dxi3fNi-c!N+~wVdQ${Vp=9cP;_!7FK%%}JgjC8a%--pMm5a>+R&x8meSg5;ZobY z2+%Ii7RLZLNQ%7_Kh)mAu}ZZM!L;@X;%2oHFs*%$1DomwEdh)e44AMCP$j8q$m$wF zuDcvF)!mDrI@!8a_Yh51w|X()X~Fv&4OO>M7`!2++Xeg!&UGJ1>Bn%X&&dM}=2`vF zT)=+N)=z|U{Xt6slW@ZN<6Z{1_0Zbwk#o*1u71~Qrsp`huo0>C+&02fffri)?FbeW(m zMT4g6p9kCkS<|buOw*f4YuZ5rxNqRKS76Cnob=Q%x+ULdyogPU5VQ}n6lk{iXx_`f z5rBP4pQTg_D&@m2A&RF;Qz_+~%BE6!2&K}do@pv2NK?vLRnt%I)OUO;lA4Nq8V6%U z+ud|Q&O7PD^0)WW(;P-vqeDKg^-|p8ADVfV{QRe<(cgQn6W1|DT}-y@86TyoZ+7|e zckf3Bko+mItbgwp7=tN*soZ8t2U;@DbQxW;RIgt#}%ww9C&^LmD~fbl!!bPvZi{LkQvIPXWHtTj$jy?BIPXO??F7D^sNjr7(s`*QS4 z#u$$PMQ$xKbjgi#B4nv+C5D`YQ)h8GkKyQffz+N5LCWs|BNpM=BU+ci<)R2cW04QQ zCGWd7RFmI!gN7CZMq!OX;{;3;a1csrXj1PY(a@xxL7cxCopXQHjn$SyHtaS)foz1< zTd|XI#|J6RsAhOd#U1l!wu2%u#sUU#BUffiMH%1~`k~F~S@=^Faf%q{Xpr~0g=!=< zmpcHA!l9ztA-QID)e=}4^0%+B2#+Y7Dyd^zIbF!DM&5%rG zDa?`RJmezUt!S<^UU?X4+mMm=S0?=k`T_PSHGX!TmP9mDeOPK`VRDt34-L|~Qcp9& zRa)4&jy`b*AR<+@rey#mAeoA@CZnNqJq%voQ!W3Ulo|(-^}e4$c4#lN(kr~oullK0 zV1n{G-al+*aAlS%oA9&@l( zrs^a3`A4F2UIbmrDxYx><{CcS6zEaHGwr5je(oq&KHE`cmi%H>~v@Dxz2 z5Uj#C0wTxeFb>KBi})WXXc|Oz(BmjH=*1q?_%9!`py}A&V@InoQ{n8J4q5CDCj;p? zm|Cmt)VUuDn;j#@olw|j`mxJ5>yh@I&`M;-w+*ZLu7;w{3#&}=KV21(5Mwu92u0$n zR`|&_-@sU6_d)Z$nYKJxDgM53;wiNfq&I$WBt06V--HhrNDs zyXEIgnf#k5yF~CVqaeDZg}PnYiyvfBs&^SwJSG$);F8Egm+a#xb;);B>au}x1q=8w z@<|R=5Ou?+plL|Hn4{$mL@uyn=NM*+#lQ<)(+~SEBWOAqO@m+b!1)~181Wsc8V6iz z#%{#y6Djp)Bh8!LiXU7A=f-c)$~}yfF2Ib*7@)`I*Ni|~`9ahpaWAQ1`S6QexE_jA z5m_l;ZIvT==Gj7{G_3bidI_LhzJSV~f-F=6@QO1Oe$mA<(aL(0XYNUG8F>}74P8Ef zyav^m_fh%?J(A}_Y;TRF)6&D8>sNqWUWCrk4v=HrQt56eN*>Pu zyyWp_DB|E*PB)$?>)TzEVOqA7XEnD9`wJj*wUBj97d$zgSI^`PB|>DI;F9<=OkJ;m z=bIxGPO`z%0G3Iq_i2Fp5ojxZok78k6ojoVgvY%I+K+7c9uF3}jMC`iWGn2=07o~a z7k2LL3TzuGuxVdh;YQlP#49!bGtagoQhMsN9?p^;F4d8RYGIZ7%aiOyTjwLKO61O7 zM?L-@RlGipYo^KXzm=VOff*f5$X|N8Uqk$`g?Z-4{pya@Sp~6#| zR`i2x9SRsbhVCY<=U3$9|D*Bi=yQg9LB4%H?5Ci(s!{trGLpK8HS~~HUE`lxr$1c& zx~T?f>km?cgEIX&;fj)GA^ioWKj2do+KuSoHGC_!(!X=6pQ5C?@O^&P=`bdtzogki z*ucEC3)^p`@$|K7xLE8m8|NSVV=GhLv#_(aDooA&8vo2olAM62TtH?S%FnollzAuY zmFd_7?~x%MG47T&+~;OI){wST{cI~e=BTQ#10YL76_)Up=@e&d6d8sLe0C2`sPmWH zFDcExE{kV18O){L>o!Ntmv!MqLjZ>Kwt5B3;1f`{;Kljo=mIy|7p$bA#;_3F$dDr&ZVlbiptY1b&%>OBExCY&y>NcElN5pJ=n3PF9jzSl; zsajGUB%~vb&bdv zS#M$SUAMOi^3S?5N+Ty7^CKL|l8qtSsu{AU!LkdoV=Z7Dz_)829rsX7?AtY;XL1{b zl_jMzc^dVDw7HL_urlY;OG)6eG{KNx`Z)!b!LzEZl5&`P2UjbDRNY(5ODx-3fEdSe zy37gDn~>pse?3`gX=alhlQN`QdZ}EpZUc5M zI*A5NHkr6oQl$L;Hbw)~l3JScY!;;ht(q~-qsZm~rJWS)KOAJ;k}zyqC$z&d01in> z$E|O&-taa)=-+GIylY4edlC21(s4y>f4PNU0lE=}`@pf4GZTyFZKq$;i;>KuXb-Kz z_)P&^&jxbFtqw#?%fX8~Y0JXwW?cCWgs0Uqy@E2)OhZZTX%LI+O-O;05T52pdXvCD zuJ<*Uo~$U~z0~WP1`VQ(hVDnT(nk$(%rx5MwiX?Lq$xZr`j965kY@?~Rm#qzwxNbL zN?EiW7pTBs+Qa31L(L0XvHg&ZP>jD2YiQq8@a_ie661{@F8?s>xu6XHJv~Fg>7=yh zvdzPOj&CW=`J^4INq=RZSOT~P4>RSzf%aFKkAhcpS2y3L(rM+LLEVPQJZ$)(sB8QP zUalPQHE1gu*=|s> zTAd{M7px_3nUL4Y*n;V- z{b8WPiONtfNe(~*Z)0-k6@N(%+oIn-!{^xfaFH4Xq+7S->nZ8k1CF%MbS71|v@gDC zFwhGYleDLh)h;qzhD|Hqq#3Q<830jy?3_iUM5MkII#V6<9@$^CqBwVrX46`fUGy~)hulk5-RCD#Os!L$XHl@1-LC6=kCNSo)VNNtabWzuXw7FWIC;L5b zqhEd?r`*xCimbjB8M3v^JArY~6N4;r7Jd-vaxz5a0vdKjNL2vhi_tzL2dukVgjNDQ z(a*g!z!tJGxjrE(^K3m&UvG)g^;eNIdf+@$x&ny>c^`pGSi(T1UTUchQZrgpJc}=F zVE~t|H0d-F0V*4e8akm$$cm4KI{_{S1=bEI8iN=oSiAuln>*n=5ry=|kg^|yX$dH? zdr2AjEuSDz6!0pZ2)UE4O(|SwRAra#FDS|B3C|9f=vKz+N7Tjyx3`z z@}#=56H}4t`poI-428B#T(7jnoS`n2*G%=tmvY6$_|y3FGyXgNTdKufpyP7#ApP9^wcD|VBFE?x*($^J9R(snKcTo>CHK+GDg64@a z*>DZQiA?xJpm%K^HsPp|UQi=W4>ZfA6vhGBL3Hgxqe2C9`P zZj%j6wi~F?(B5KFpG3`*Ivj5EWR|3vCwCm=Hczp}begAhoIKC8bn!5sGN#p0;Wk~| z)MUCkk93=E%d&|Np*Xj+neNRp<8eRcF+J9|*pH~Q>FKy=9u+L4;m4G=nO@h6ou;>= zu+#KeSvJ$xQSLVV7RO-uJLCgl)a6uPPe^1Bj2XZk%|O?I^UR>-hhOfNjb^aJVKYM= zhuh3B)-}T&r*&ecRlR00Hw~G^9kp&V!sS6`r1x!`8KvhkqaC@OW{h{$jIBM!ZI)OW zoo30}(~zcHLqjjDGfO?6xy{nGhtD(1+?Lxc>v+*-mNTZ9;X zW=-f^K^-=Z_xx_E@SnA> z-x}JB;(p#k=kYoeiM1^}lW{P`H2SKrlDW6%?+gAZl( zD8*h4w$BtyW+*z#)f)Yq;)uM0Q8y)bpINThK>v8^Rl@ocNon=S1=Ze=`FKi@{*g$| zX=HektZmWuHu^+AKxuU{_LJ2psAXtNF{m#mlwBd`q*mO_2$t9&DOs6RAzwC?TL4v? z7&Yn+%@ROXZw%IZf>Rd*>jX`20Dn-{-8we*i&noaj< zJTamnK<5XMEz)_^c+Aro8~3_EaytEtb8pBoMVtK9GgfG4WB>;IK>uEAXiZx`>8AsY robrQ|owa>aoRhN_X(Mf!48lrA?l?kCzkiF4Q8iUAs#=ol_WAz{i*386 diff --git a/publicsuffix/data/text b/publicsuffix/data/text index 124dcd61f4..7e516413f6 100644 --- a/publicsuffix/data/text +++ b/publicsuffix/data/text @@ -1 +1 @@ -billustrationionjukudoyamakeupowiathletajimageandsoundandvision-riopretobishimagentositecnologiabiocelotenkawabipanasonicatfoodnetworkinggroupperbirdartcenterprisecloudaccesscamdvrcampaniabirkenesoddtangenovarahkkeravjuegoshikikiraraholtalenishikatakazakindependent-revieweirbirthplaceu-1bitbucketrzynishikatsuragirlyuzawabitternidiscoverybjarkoybjerkreimdbaltimore-og-romsdalp1bjugnishikawazukamishihoronobeautydalwaysdatabaseballangenkainanaejrietisalatinabenogatabitorderblackfridaybloombergbauernishimerabloxcms3-website-us-west-2blushakotanishinomiyashironocparachutingjovikarateu-2bmoattachmentsalangenishinoomotegovtattoolforgerockartuzybmsalon-1bmwellbeingzoneu-3bnrwesteuropenairbusantiquesaltdalomzaporizhzhedmarkaratsuginamikatagamilanotairesistanceu-4bondigitaloceanspacesaludishangrilanciabonnishinoshimatsusakahoginankokubunjindianapolis-a-bloggerbookonlinewjerseyboomlahppiacenzachpomorskienishiokoppegardiskussionsbereichattanooganordkapparaglidinglassassinationalheritageu-north-1boschaefflerdalondonetskarelianceu-south-1bostik-serveronagasukevje-og-hornnesalvadordalibabalatinord-aurdalipaywhirlondrinaplesknsalzburgleezextraspace-to-rentalstomakomaibarabostonakijinsekikogentappssejnyaarparalleluxembourglitcheltenham-radio-opensocialorenskogliwicebotanicalgardeno-staginglobodoes-itcouldbeworldisrechtranakamurataiwanairforcechireadthedocsxeroxfinitybotanicgardenishitosashimizunaminamiawajikindianmarketinglogowestfalenishiwakindielddanuorrindigenamsskoganeindustriabotanyanagawallonieruchomoscienceandindustrynissandiegoddabouncemerckmsdnipropetrovskjervoyageorgeorgiabounty-fullensakerrypropertiesamegawaboutiquebecommerce-shopselectaxihuanissayokkaichintaifun-dnsaliasamnangerboutireservditchyouriparasiteboyfriendoftheinternetflixjavaldaostathellevangerbozen-sudtirolottokorozawabozen-suedtirolouvreisenissedalovepoparisor-fronisshingucciprianiigataipeidsvollovesickariyakumodumeloyalistoragebplaceducatorprojectcmembersampalermomahaccapooguybrandywinevalleybrasiliadboxosascoli-picenorddalpusercontentcp4bresciaokinawashirosatobamagazineuesamsclubartowestus2brindisibenikitagataikikuchikumagayagawalmartgorybristoloseyouriparliamentjeldsundivtasvuodnakaniikawatanagurabritishcolumbialowiezaganiyodogawabroadcastlebtimnetzlgloomy-routerbroadwaybroke-itvedestrandivttasvuotnakanojohanamakindlefrakkestadiybrokerbrothermesaverdeatnulmemergencyachtsamsungloppennebrowsersafetymarketsandnessjoenl-ams-1brumunddalublindesnesandoybrunelastxn--0trq7p7nnbrusselsandvikcoromantovalle-daostavangerbruxellesanfranciscofreakunekobayashikaoirmemorialucaniabryanskodjedugit-pagespeedmobilizeroticagliaricoharuovatlassian-dev-builderscbglugsjcbnpparibashkiriabrynewmexicoacharterbuzzwfarmerseinebwhalingmbhartiffany-2bzhitomirbzzcodyn-vpndnsantacruzsantafedjeffersoncoffeedbackdropocznordlandrudupontariobranconavstackasaokamikoaniikappudownloadurbanamexhibitioncogretakamatsukawacollectioncolognewyorkshirebungoonordre-landurhamburgrimstadynamisches-dnsantamariakecolonialwilliamsburgripeeweeklylotterycoloradoplateaudnedalncolumbusheycommunexus-3community-prochowicecomobaravendbambleborkapsicilyonagoyauthgear-stagingivestbyglandroverhallair-traffic-controlleyombomloabaths-heilbronnoysunddnslivegarsheiheijibigawaustraliaustinnfshostrolekamisatokaizukameyamatotakadaustevollivornowtv-infolldalolipopmcdircompanychipstmncomparemarkerryhotelsantoandrepbodynaliasnesoddenmarkhangelskjakdnepropetrovskiervaapsteigenflfannefrankfurtjxn--12cfi8ixb8lutskashibatakashimarshallstatebankashiharacomsecaaskimitsubatamibuildingriwatarailwaycondoshichinohealth-carereformemsettlersanukindustriesteamfamberlevagangaviikanonjinfinitigotembaixadaconferenceconstructionconsuladogadollsaobernardomniweatherchanneluxuryconsultanthropologyconsultingroks-thisayamanobeokakegawacontactkmaxxn--12co0c3b4evalled-aostamayukinsuregruhostingrondarcontagematsubaravennaharimalborkashiwaracontemporaryarteducationalchikugodonnakaiwamizawashtenawsmppl-wawdev-myqnapcloudcontrolledogawarabikomaezakirunoopschlesischesaogoncartoonartdecologiacontractorskenconventureshinodearthickashiwazakiyosatokamachilloutsystemscloudsitecookingchannelsdvrdnsdojogaszkolancashirecifedexetercoolblogdnsfor-better-thanawassamukawatarikuzentakatairavpagecooperativano-frankivskygearapparochernigovernmentksatxn--1ck2e1bananarepublic-inquiryggeebinatsukigatajimidsundevelopmentatarantours3-external-1copenhagencyclopedichiropracticatholicaxiashorokanaiecoproductionsaotomeinforumzcorporationcorsicahcesuoloanswatch-and-clockercorvettenrissagaeroclubmedecincinnativeamericanantiquest-le-patron-k3sapporomuracosenzamamidorittoeigersundynathomebuiltwithdarkasserverrankoshigayaltakasugaintelligencecosidnshome-webservercellikescandypoppdaluzerncostumedicallynxn--1ctwolominamatargets-itlon-2couchpotatofriesardegnarutomobegetmyiparsardiniacouncilvivanovoldacouponsarlcozoracq-acranbrookuwanalyticsarpsborgrongausdalcrankyowariasahikawatchandclockasukabeauxartsandcraftsarufutsunomiyawakasaikaitabashijonawatecrdyndns-at-homedepotaruinterhostsolutionsasayamatta-varjjatmpartinternationalfirearmsaseboknowsitallcreditcardyndns-at-workshoppingrossetouchigasakitahiroshimansionsaskatchewancreditunioncremonashgabadaddjaguarqcxn--1lqs03ncrewhmessinarashinomutashinaintuitoyosatoyokawacricketnedalcrimeast-kazakhstanangercrotonecrownipartsassarinuyamashinazawacrsaudacruisesauheradyndns-blogsitextilegnicapetownnews-stagingroundhandlingroznycuisinellancasterculturalcentertainmentoyotapartysvardocuneocupcakecuritibabymilk3curvallee-d-aosteinkjerusalempresashibetsurugashimaringatlantajirinvestmentsavannahgacutegirlfriendyndns-freeboxoslocalzonecymrulvikasumigaurawa-mazowszexnetlifyinzairtrafficplexus-1cyonabarumesswithdnsaveincloudyndns-homednsaves-the-whalessandria-trani-barletta-andriatranibarlettaandriacyouthruherecipescaracaltanissettaishinomakilovecollegefantasyleaguernseyfembetsukumiyamazonawsglobalacceleratorahimeshimabaridagawatchesciencecentersciencehistoryfermockasuyamegurownproviderferraraferraris-a-catererferrerotikagoshimalopolskanlandyndns-picsaxofetsundyndns-remotewdyndns-ipasadenaroyfgujoinvilleitungsenfhvalerfidontexistmein-iservschulegallocalhostrodawarafieldyndns-serverdalfigueresindevicenzaolkuszczytnoipirangalsaceofilateliafilegear-augustowhoswholdingsmall-webthingscientistordalfilegear-debianfilegear-gbizfilegear-iefilegear-jpmorganfilegear-sg-1filminamiechizenfinalfinancefineartscrapper-sitefinlandyndns-weblikes-piedmonticellocus-4finnoyfirebaseappaviancarrdyndns-wikinkobearalvahkijoetsuldalvdalaskanittedallasalleasecuritytacticschoenbrunnfirenetoystre-slidrettozawafirenzefirestonefirewebpaascrappingulenfirmdaleikangerfishingoldpoint2thisamitsukefitjarvodkafjordyndns-workangerfitnessettlementozsdellogliastradingunmanxn--1qqw23afjalerfldrvalleeaosteflekkefjordyndns1flesberguovdageaidnunjargaflickragerogerscrysecretrosnubar0flierneflirfloginlinefloppythonanywhereggio-calabriafloraflorencefloridatsunangojomedicinakamagayahabackplaneapplinzis-a-celticsfanfloripadoval-daostavalleyfloristanohatakahamalselvendrellflorokunohealthcareerscwienflowerservehalflifeinsurancefltrani-andria-barletta-trani-andriaflynnhosting-clusterfnchiryukyuragifuchungbukharanzanfndynnschokokekschokoladenfnwkaszubytemarkatowicefoolfor-ourfor-somedio-campidano-mediocampidanomediofor-theaterforexrothachijolsterforgotdnservehttpbin-butterforli-cesena-forlicesenaforlillesandefjordynservebbscholarshipschoolbusinessebyforsaleirfjordynuniversityforsandasuolodingenfortalfortefortmissoulangevagrigentomologyeonggiehtavuoatnagahamaroygardencowayfortworthachinoheavyfosneservehumourfotraniandriabarlettatraniandriafoxfordecampobassociatest-iserveblogsytemp-dnserveirchitachinakagawashingtondchernivtsiciliafozfr-par-1fr-par-2franamizuhobby-sitefrancaiseharafranziskanerimalvikatsushikabedzin-addrammenuorochesterfredrikstadtvserveminecraftranoyfreeddnsfreebox-oservemp3freedesktopfizerfreemasonryfreemyiphosteurovisionfreesitefreetlservep2pgfoggiafreiburgushikamifuranorfolkebibleksvikatsuyamarugame-hostyhostingxn--2m4a15efrenchkisshikirkeneservepicservequakefreseniuscultureggio-emilia-romagnakasatsunairguardiannakadomarinebraskaunicommbankaufentigerfribourgfriuli-v-giuliafriuli-ve-giuliafriuli-vegiuliafriuli-venezia-giuliafriuli-veneziagiuliafriuli-vgiuliafriuliv-giuliafriulive-giuliafriulivegiuliafriulivenezia-giuliafriuliveneziagiuliafriulivgiuliafrlfroganservesarcasmatartanddesignfrognfrolandynv6from-akrehamnfrom-alfrom-arfrom-azurewebsiteshikagamiishibukawakepnoorfrom-capitalonewportransipharmacienservicesevastopolefrom-coalfrom-ctranslatedynvpnpluscountryestateofdelawareclaimschoolsztynsettsupportoyotomiyazakis-a-candidatefrom-dchitosetodayfrom-dediboxafrom-flandersevenassisienarvikautokeinoticeablewismillerfrom-gaulardalfrom-hichisochikuzenfrom-iafrom-idyroyrvikingruenoharafrom-ilfrom-in-berlindasewiiheyaizuwakamatsubushikusakadogawafrom-ksharpharmacyshawaiijimarcheapartmentshellaspeziafrom-kyfrom-lanshimokawafrom-mamurogawatsonfrom-mdfrom-medizinhistorischeshimokitayamattelekommunikationfrom-mifunefrom-mnfrom-modalenfrom-mshimonitayanagit-reposts-and-telecommunicationshimonosekikawafrom-mtnfrom-nchofunatoriginstantcloudfrontdoorfrom-ndfrom-nefrom-nhktistoryfrom-njshimosuwalkis-a-chefarsundyndns-mailfrom-nminamifuranofrom-nvalleedaostefrom-nynysagamiharafrom-ohdattorelayfrom-oketogolffanshimotsukefrom-orfrom-padualstackazoologicalfrom-pratogurafrom-ris-a-conservativegashimotsumayfirstockholmestrandfrom-schmidtre-gauldalfrom-sdscloudfrom-tnfrom-txn--2scrj9chonanbunkyonanaoshimakanegasakikugawaltervistailscaleforcefrom-utsiracusaikirovogradoyfrom-vald-aostarostwodzislawildlifestylefrom-vtransportefrom-wafrom-wiardwebview-assetshinichinanfrom-wvanylvenneslaskerrylogisticshinjournalismartlabelingfrom-wyfrosinonefrostalowa-wolawafroyal-commissionfruskydivingfujiiderafujikawaguchikonefujiminokamoenairkitapps-auction-rancherkasydneyfujinomiyadattowebhoptogakushimotoganefujiokayamandalfujisatoshonairlinedre-eikerfujisawafujishiroishidakabiratoridedyn-berlincolnfujitsuruokazakiryuohkurafujiyoshidavvenjargap-east-1fukayabeardubaiduckdnsncfdfukuchiyamadavvesiidappnodebalancertmgrazimutheworkpccwilliamhillfukudomigawafukuis-a-cpalacefukumitsubishigakisarazure-mobileirvikazteleportlligatransurlfukuokakamigaharafukuroishikarikaturindalfukusakishiwadazaifudaigokaseljordfukuyamagatakaharunusualpersonfunabashiriuchinadafunagatakahashimamakisofukushimangonnakatombetsumy-gatewayfunahashikamiamakusatsumasendaisenergyfundaciofunkfeuerfuoiskujukuriyamangyshlakasamatsudoomdnstracefuosskoczowinbar1furubirafurudonostiaafurukawajimaniwakuratefusodegaurafussaintlouis-a-anarchistoireggiocalabriafutabayamaguchinomihachimanagementrapaniizafutboldlygoingnowhere-for-morenakatsugawafuttsurutaharafuturecmshinjukumamotoyamashikefuturehostingfuturemailingfvghamurakamigoris-a-designerhandcraftedhandsonyhangglidinghangoutwentehannanmokuizumodenaklodzkochikuseihidorahannorthwesternmutualhanyuzenhapmircloudletshintokushimahappounzenharvestcelebrationhasamap-northeast-3hasaminami-alpshintomikasaharahashbangryhasudahasura-apphiladelphiaareadmyblogspotrdhasvikfh-muensterhatogayahoooshikamaishimofusartshinyoshitomiokamisunagawahatoyamazakitakatakanabeatshiojirishirifujiedahatsukaichikaiseiyoichimkentrendhostinghattfjelldalhayashimamotobusellfylkesbiblackbaudcdn-edgestackhero-networkisboringhazuminobushistoryhelplfinancialhelsinkitakyushuaiahembygdsforbundhemneshioyanaizuerichardlimanowarudahemsedalhepforgeblockshirahamatonbetsurgeonshalloffameiwamasoyheroyhetemlbfanhgtvaohigashiagatsumagoianiahigashichichibuskerudhigashihiroshimanehigashiizumozakitamigrationhigashikagawahigashikagurasoedahigashikawakitaaikitamotosunndalhigashikurumeeresinstaginghigashimatsushimarburghigashimatsuyamakitaakitadaitoigawahigashimurayamamotorcycleshirakokonoehigashinarusells-for-lesshiranukamitondabayashiogamagoriziahigashinehigashiomitamanortonsberghigashiosakasayamanakakogawahigashishirakawamatakanezawahigashisumiyoshikawaminamiaikitanakagusukumodernhigashitsunosegawahigashiurausukitashiobarahigashiyamatokoriyamanashifteditorxn--30rr7yhigashiyodogawahigashiyoshinogaris-a-doctorhippyhiraizumisatohnoshoohirakatashinagawahiranairportland-4-salernogiessennanjobojis-a-financialadvisor-aurdalhirarahiratsukaerusrcfastlylbanzaicloudappspotagerhirayaitakaokalmykiahistorichouseshiraois-a-geekhakassiahitachiomiyagildeskaliszhitachiotagonohejis-a-greenhitraeumtgeradegreehjartdalhjelmelandholeckodairaholidayholyhomegoodshiraokamitsuehomeiphilatelyhomelinkyard-cloudjiffyresdalhomelinuxn--32vp30hachiojiyahikobierzycehomeofficehomesecuritymacaparecidahomesecuritypchoseikarugamvikarlsoyhomesenseeringhomesklepphilipsynology-diskstationhomeunixn--3bst00minamiiserniahondahongooglecodebergentinghonjyoitakarazukaluganskharkivaporcloudhornindalhorsells-for-ustkanmakiwielunnerhortendofinternet-dnshiratakahagitapphoenixn--3ds443ghospitalhoteleshishikuis-a-guruhotelwithflightshisognehotmailhoyangerhoylandetakasagophonefosshisuifuettertdasnetzhumanitieshitaramahungryhurdalhurumajis-a-hard-workershizukuishimogosenhyllestadhyogoris-a-hunterhyugawarahyundaiwafuneis-into-carsiiitesilkharkovaresearchaeologicalvinklein-the-bandairtelebitbridgestoneenebakkeshibechambagricultureadymadealstahaugesunderseaportsinfolionetworkdalaheadjudygarlandis-into-cartoonsimple-urlis-into-gamesserlillyis-leetrentin-suedtirolis-lostre-toteneis-a-lawyeris-not-certifiedis-savedis-slickhersonis-uberleetrentino-a-adigeis-very-badajozis-a-liberalis-very-evillageis-very-goodyearis-very-niceis-very-sweetpepperugiais-with-thebandovre-eikerisleofmanaustdaljellybeanjenv-arubahccavuotnagaragusabaerobaticketsirdaljeonnamerikawauejetztrentino-aadigejevnakershusdecorativeartslupskhmelnytskyivarggatrentino-alto-adigejewelryjewishartgalleryjfkhplaystation-cloudyclusterjgorajlljls-sto1jls-sto2jls-sto3jmphotographysiojnjaworznospamproxyjoyentrentino-altoadigejoyokaichibajddarchitecturealtorlandjpnjprslzjurkotohiradomainstitutekotourakouhokutamamurakounosupabasembokukizunokunimilitarykouyamarylhurstjordalshalsenkouzushimasfjordenkozagawakozakis-a-llamarnardalkozowindowskrakowinnersnoasakatakkokamiminersokndalkpnkppspbarcelonagawakkanaibetsubamericanfamilyds3-fips-us-gov-west-1krasnikahokutokashikis-a-musiciankrasnodarkredstonekrelliankristiansandcatsolarssonkristiansundkrodsheradkrokstadelvalle-aostatic-accessolognekryminamiizukaminokawanishiaizubangekumanotteroykumatorinovecoregontrailroadkumejimashikis-a-nascarfankumenantokonamegatakatoris-a-nursells-itrentin-sud-tirolkunisakis-a-painteractivelvetrentin-sudtirolkunitachiaraindropilotsolundbecknx-serversellsyourhomeftphxn--3e0b707ekunitomigusukuleuvenetokigawakunneppuboliviajessheimpertrixcdn77-secureggioemiliaromagnamsosnowiechristiansburgminakamichiharakunstsammlungkunstunddesignkuokgroupimientaketomisatoolsomakurehabmerkurgankurobeeldengeluidkurogimimatakatsukis-a-patsfankuroisoftwarezzoologykuromatsunais-a-personaltrainerkuronkurotakikawasakis-a-photographerokussldkushirogawakustanais-a-playershiftcryptonomichigangwonkusupersalezajskomakiyosemitekutchanelkutnowruzhgorodeokuzumakis-a-republicanonoichinomiyakekvafjordkvalsundkvamscompute-1kvanangenkvinesdalkvinnheradkviteseidatingkvitsoykwpspdnsomnatalkzmisakis-a-soxfanmisasaguris-a-studentalmisawamisconfusedmishimasudamissilemisugitokuyamatsumaebashikshacknetrentino-sued-tirolmitakeharamitourismilemitoyoakemiuramiyazurecontainerdpolicemiyotamatsukuris-a-teacherkassyno-dshowamjondalenmonstermontrealestatefarmequipmentrentino-suedtirolmonza-brianzapposor-odalmonza-e-della-brianzaptokyotangotpantheonsitemonzabrianzaramonzaebrianzamonzaedellabrianzamoonscalebookinghostedpictetrentinoa-adigemordoviamoriyamatsumotofukemoriyoshiminamiashigaramormonmouthachirogatakamoriokakudamatsuemoroyamatsunomortgagemoscowiosor-varangermoseushimodatemosjoenmoskenesorfoldmossorocabalena-devicesorreisahayakawakamiichikawamisatottoris-a-techietis-a-landscaperspectakasakitchenmosvikomatsushimarylandmoteginowaniihamatamakinoharamoviemovimientolgamozilla-iotrentinoaadigemtranbytomaritimekeepingmuginozawaonsensiositemuikaminoyamaxunispacemukoebenhavnmulhouseoullensvanguardmunakatanemuncienciamuosattemupinbarclaycards3-sa-east-1murmanskomforbar2murotorcraftrentinoalto-adigemusashinoharamuseetrentinoaltoadigemuseumverenigingmusicargodaddyn-o-saurlandesortlandmutsuzawamy-wanggoupilemyactivedirectorymyamazeplaymyasustor-elvdalmycdmycloudnsoruminamimakis-a-rockstarachowicemydattolocalcertificationmyddnsgeekgalaxymydissentrentinos-tirolmydobissmarterthanyoumydrobofageologymydsoundcastronomy-vigorlicemyeffectrentinostirolmyfastly-terrariuminamiminowamyfirewalledreplittlestargardmyforuminamioguni5myfritzmyftpaccessouthcarolinaturalhistorymuseumcentermyhome-servermyjinomykolaivencloud66mymailermymediapchristmasakillucernemyokohamamatsudamypepinkommunalforbundmypetsouthwest1-uslivinghistorymyphotoshibalashovhadanorth-kazakhstanmypicturestaurantrentinosud-tirolmypsxn--3pxu8kommunemysecuritycamerakermyshopblocksowamyshopifymyspreadshopwarendalenugmythic-beastspectruminamisanrikubetsuppliesoomytis-a-bookkeepermaritimodspeedpartnermytuleap-partnersphinxn--41amyvnchromediatechnologymywirepaircraftingvollohmusashimurayamashikokuchuoplantationplantspjelkavikomorotsukagawaplatformsharis-a-therapistoiaplatter-appinokofuefukihaboromskogplatterpioneerplazaplcube-serversicherungplumbingoplurinacionalpodhalepodlasiellaktyubinskiptveterinairealmpmnpodzonepohlpoivronpokerpokrovskomvuxn--3hcrj9choyodobashichikashukujitawaraumalatvuopmicrosoftbankarmoypoliticarrierpolitiendapolkowicepoltavalle-d-aostaticspydebergpomorzeszowitdkongsbergponpesaro-urbino-pesarourbinopesaromasvuotnarusawapordenonepornporsangerporsangugeporsgrunnanyokoshibahikariwanumatakinouepoznanpraxis-a-bruinsfanprdpreservationpresidioprgmrprimetelemarkongsvingerprincipeprivatizehealthinsuranceprofesionalprogressivestfoldpromombetsupplypropertyprotectionprotonetrentinosued-tirolprudentialpruszkowithgoogleapiszprvcyberprzeworskogpulawypunyufuelveruminamiuonumassa-carrara-massacarraramassabuyshousesopotrentino-sud-tirolpupugliapussycateringebuzentsujiiepvhadselfiphdfcbankazunoticiashinkamigototalpvtrentinosuedtirolpwchungnamdalseidsbergmodellingmxn--11b4c3dray-dnsupdaterpzqhaebaruericssongdalenviknakayamaoris-a-cubicle-slavellinodeobjectshinshinotsurfashionstorebaselburguidefinimamateramochizukimobetsumidatlantichirurgiens-dentistes-en-franceqldqotoyohashimotoshimatsuzakis-an-accountantshowtimelbourneqponiatowadaqslgbtrentinsud-tirolqualifioappippueblockbusternopilawaquickconnectrentinsudtirolquicksytesrhtrentinsued-tirolquipelementsrltunestuff-4-saletunkonsulatrobeebyteappigboatsmolaquilanxessmushcdn77-sslingturystykaniepcetuscanytushuissier-justicetuvalleaostaverntuxfamilytwmailvestvagoyvevelstadvibo-valentiavibovalentiavideovillastufftoread-booksnestorfjordvinnicasadelamonedagestangevinnytsiavipsinaappiwatevirginiavirtual-uservecounterstrikevirtualcloudvirtualservervirtualuserveexchangevirtuelvisakuhokksundviterbolognagasakikonaikawagoevivianvivolkenkundenvixn--42c2d9avlaanderennesoyvladikavkazimierz-dolnyvladimirvlogintoyonezawavminanovologdanskonyveloftrentino-stirolvolvolkswagentstuttgartrentinsuedtirolvolyngdalvoorlopervossevangenvotevotingvotoyonovps-hostrowiecircustomer-ocimmobilienwixsitewloclawekoobindalwmcloudwmflabsurnadalwoodsidelmenhorstabackyardsurreyworse-thandawowithyoutuberspacekitagawawpdevcloudwpenginepoweredwphostedmailwpmucdnpixolinodeusercontentrentinosudtirolwpmudevcdnaccessokanagawawritesthisblogoipizzawroclawiwatsukiyonoshiroomgwtcirclerkstagewtfastvps-serverisignwuozuwzmiuwajimaxn--4gbriminingxn--4it168dxn--4it797kooris-a-libertarianxn--4pvxs4allxn--54b7fta0ccivilaviationredumbrellajollamericanexpressexyxn--55qw42gxn--55qx5dxn--5dbhl8dxn--5js045dxn--5rtp49civilisationrenderxn--5rtq34koperviklabudhabikinokawachinaganoharamcocottempurlxn--5su34j936bgsgxn--5tzm5gxn--6btw5axn--6frz82gxn--6orx2rxn--6qq986b3xlxn--7t0a264civilizationthewifiatmallorcafederation-webspacexn--80aaa0cvacationsusonoxn--80adxhksuzakananiimiharuxn--80ao21axn--80aqecdr1axn--80asehdbarclays3-us-east-2xn--80aswgxn--80aukraanghkembuchikujobservableusercontentrevisohughestripperxn--8dbq2axn--8ltr62koryokamikawanehonbetsuwanouchijiwadeliveryxn--8pvr4uxn--8y0a063axn--90a1affinitylotterybnikeisenbahnxn--90a3academiamicable-modemoneyxn--90aeroportalabamagasakishimabaraffleentry-snowplowiczeladzxn--90aishobarakawaharaoxn--90amckinseyxn--90azhytomyrxn--9dbhblg6dietritonxn--9dbq2axn--9et52uxn--9krt00axn--andy-iraxn--aroport-byandexcloudxn--asky-iraxn--aurskog-hland-jnbarefootballooningjerstadgcapebretonamicrolightingjesdalombardiadembroideryonagunicloudiherokuappanamasteiermarkaracoldwarszawauthgearappspacehosted-by-previderxn--avery-yuasakuragawaxn--b-5gaxn--b4w605ferdxn--balsan-sdtirol-nsbsuzukanazawaxn--bck1b9a5dre4civilwarmiasadoesntexisteingeekarpaczest-a-la-maisondre-landrayddns5yxn--bdddj-mrabdxn--bearalvhki-y4axn--berlevg-jxaxn--bhcavuotna-s4axn--bhccavuotna-k7axn--bidr-5nachikatsuuraxn--bievt-0qa2xn--bjarky-fyaotsurgeryxn--bjddar-ptargithubpreviewsaitohmannore-og-uvdalxn--blt-elabourxn--bmlo-graingerxn--bod-2naturalsciencesnaturellesuzukis-an-actorxn--bozen-sdtirol-2obanazawaxn--brnny-wuacademy-firewall-gatewayxn--brnnysund-m8accident-investigation-acornxn--brum-voagatroandinosaureportrentoyonakagyokutoyakomaganexn--btsfjord-9zaxn--bulsan-sdtirol-nsbaremetalpha-myqnapcloud9guacuiababia-goracleaningitpagexlimoldell-ogliastraderxn--c1avgxn--c2br7gxn--c3s14mincomcastreserve-onlinexn--cck2b3bargainstances3-us-gov-west-1xn--cckwcxetdxn--cesena-forl-mcbremangerxn--cesenaforl-i8axn--cg4bkis-an-actresshwindmillxn--ciqpnxn--clchc0ea0b2g2a9gcdxn--comunicaes-v6a2oxn--correios-e-telecomunicaes-ghc29axn--czr694barreaudiblebesbydgoszczecinemagnethnologyoriikaragandauthordalandroiddnss3-ap-southeast-2ix4432-balsan-suedtirolimiteddnskinggfakefurniturecreationavuotnaritakoelnayorovigotsukisosakitahatakahatakaishimoichinosekigaharaurskog-holandingitlaborxn--czrs0trogstadxn--czru2dxn--czrw28barrel-of-knowledgeappgafanquanpachicappacificurussiautomotivelandds3-ca-central-16-balsan-sudtirollagdenesnaaseinet-freaks3-ap-southeast-123websiteleaf-south-123webseiteckidsmynasushiobarackmazerbaijan-mayen-rootaribeiraogakibichuobiramusementdllpages3-ap-south-123sitewebhareidfjordvagsoyerhcloudd-dnsiskinkyolasiteastcoastaldefenceastus2038xn--d1acj3barrell-of-knowledgecomputerhistoryofscience-fictionfabricafjs3-us-west-1xn--d1alfaromeoxn--d1atromsakegawaxn--d5qv7z876clanbibaidarmeniaxn--davvenjrga-y4axn--djrs72d6uyxn--djty4kosaigawaxn--dnna-grajewolterskluwerxn--drbak-wuaxn--dyry-iraxn--e1a4cldmailukowhitesnow-dnsangohtawaramotoineppubtlsanjotelulubin-brbambinagisobetsuitagajoburgjerdrumcprequalifymein-vigorgebetsukuibmdeveloperauniteroizumizakinderoyomitanobninskanzakiyokawaraustrheimatunduhrennebulsan-suedtirololitapunk123kotisivultrobjectselinogradimo-siemenscaledekaascolipiceno-ipifony-1337xn--eckvdtc9dxn--efvn9svalbardunloppaderbornxn--efvy88hagakhanamigawaxn--ehqz56nxn--elqq16hagebostadxn--eveni-0qa01gaxn--f6qx53axn--fct429kosakaerodromegallupaasdaburxn--fhbeiarnxn--finny-yuaxn--fiq228c5hsvchurchaseljeepsondriodejaneirockyotobetsuliguriaxn--fiq64barsycenterprisesakievennodesadistcgrouplidlugolekagaminord-frontierxn--fiqs8sveioxn--fiqz9svelvikoninjambylxn--fjord-lraxn--fjq720axn--fl-ziaxn--flor-jraxn--flw351exn--forl-cesena-fcbssvizzeraxn--forlcesena-c8axn--fpcrj9c3dxn--frde-grandrapidsvn-repostorjcloud-ver-jpchowderxn--frna-woaraisaijosoyroroswedenxn--frya-hraxn--fzc2c9e2cleverappsannanxn--fzys8d69uvgmailxn--g2xx48clicketcloudcontrolapparmatsuuraxn--gckr3f0fauskedsmokorsetagayaseralingenoamishirasatogliattipschulserverxn--gecrj9clickrisinglesannohekinannestadraydnsanokaruizawaxn--ggaviika-8ya47haibarakitakamiizumisanofidelitysfjordxn--gildeskl-g0axn--givuotna-8yasakaiminatoyookaneyamazoexn--gjvik-wuaxn--gk3at1exn--gls-elacaixaxn--gmq050is-an-anarchistoricalsocietysnesigdalxn--gmqw5axn--gnstigbestellen-zvbrplsbxn--45br5cylxn--gnstigliefern-wobihirosakikamijimatsushigexn--h-2failxn--h1aeghair-surveillancexn--h1ahnxn--h1alizxn--h2breg3eveneswidnicasacampinagrandebungotakadaemongolianxn--h2brj9c8clinichippubetsuikilatironporterxn--h3cuzk1digickoseis-a-linux-usershoujis-a-knightpointtohoboleslawieconomiastalbanshizuokamogawaxn--hbmer-xqaxn--hcesuolo-7ya35barsyonlinewhampshirealtychyattorneyagawakuyabukihokumakogeniwaizumiotsurugimbalsfjordeportexaskoyabeagleboardetroitskypecorivneatonoshoes3-eu-west-3utilitiesquare7xn--hebda8basicserversaillesjabbottateshinanomachildrensgardenhlfanhsbc66xn--hery-iraxn--hgebostad-g3axn--hkkinen-5waxn--hmmrfeasta-s4accident-prevention-aptibleangaviikadenaamesjevuemielnoboribetsuckswidnikkolobrzegersundxn--hnefoss-q1axn--hobl-iraxn--holtlen-hxaxn--hpmir-xqaxn--hxt814exn--hyanger-q1axn--hylandet-54axn--i1b6b1a6a2exn--imr513nxn--indery-fyasugithubusercontentromsojamisonxn--io0a7is-an-artistgstagexn--j1adpkomonotogawaxn--j1aefbsbxn--1lqs71dyndns-office-on-the-webhostingrpassagensavonarviikamiokameokamakurazakiwakunigamihamadaxn--j1ael8basilicataniautoscanadaeguambulancentralus-2xn--j1amhakatanorthflankddiamondshinshiroxn--j6w193gxn--jlq480n2rgxn--jlq61u9w7basketballfinanzgorzeleccodespotenzakopanewspaperxn--jlster-byasuokannamihokkaidopaaskvollxn--jrpeland-54axn--jvr189miniserversusakis-a-socialistg-builderxn--k7yn95exn--karmy-yuaxn--kbrq7oxn--kcrx77d1x4axn--kfjord-iuaxn--klbu-woaxn--klt787dxn--kltp7dxn--kltx9axn--klty5xn--45brj9cistrondheimperiaxn--koluokta-7ya57hakodatexn--kprw13dxn--kpry57dxn--kput3is-an-engineeringxn--krager-gyatominamibosogndalxn--kranghke-b0axn--krdsherad-m8axn--krehamn-dxaxn--krjohka-hwab49jdevcloudfunctionsimplesitexn--ksnes-uuaxn--kvfjord-nxaxn--kvitsy-fyatsukanoyakagexn--kvnangen-k0axn--l-1fairwindswiebodzin-dslattuminamiyamashirokawanabeepilepsykkylvenicexn--l1accentureklamborghinikolaeventswinoujscienceandhistoryxn--laheadju-7yatsushiroxn--langevg-jxaxn--lcvr32dxn--ldingen-q1axn--leagaviika-52batochigifts3-us-west-2xn--lesund-huaxn--lgbbat1ad8jdfaststackschulplattformetacentrumeteorappassenger-associationxn--lgrd-poacctrusteexn--lhppi-xqaxn--linds-pramericanartrvestnestudioxn--lns-qlavagiskexn--loabt-0qaxn--lrdal-sraxn--lrenskog-54axn--lt-liacliniquedapliexn--lten-granexn--lury-iraxn--m3ch0j3axn--mely-iraxn--merker-kuaxn--mgb2ddeswisstpetersburgxn--mgb9awbfbx-ostrowwlkpmguitarschwarzgwangjuifminamidaitomanchesterxn--mgba3a3ejtrycloudflarevistaplestudynamic-dnsrvaroyxn--mgba3a4f16axn--mgba3a4fra1-deloittevaksdalxn--mgba7c0bbn0axn--mgbaakc7dvfstdlibestadxn--mgbaam7a8hakonexn--mgbab2bdxn--mgbah1a3hjkrdxn--mgbai9a5eva00batsfjordiscordsays3-website-ap-northeast-1xn--mgbai9azgqp6jejuniperxn--mgbayh7gpalmaseratis-an-entertainerxn--mgbbh1a71exn--mgbc0a9azcgxn--mgbca7dzdoxn--mgbcpq6gpa1axn--mgberp4a5d4a87gxn--mgberp4a5d4arxn--mgbgu82axn--mgbi4ecexposedxn--mgbpl2fhskosherbrookegawaxn--mgbqly7c0a67fbclintonkotsukubankarumaifarmsteadrobaknoluoktachikawakayamadridvallee-aosteroyxn--mgbqly7cvafr-1xn--mgbt3dhdxn--mgbtf8flapymntrysiljanxn--mgbtx2bauhauspostman-echocolatemasekd1xn--mgbx4cd0abbvieeexn--mix082fbxoschweizxn--mix891fedorainfraclouderaxn--mjndalen-64axn--mk0axin-vpnclothingdustdatadetectjmaxxxn--12c1fe0bradescotlandrrxn--mk1bu44cn-northwest-1xn--mkru45is-bykleclerchoshibuyachiyodancexn--mlatvuopmi-s4axn--mli-tlavangenxn--mlselv-iuaxn--moreke-juaxn--mori-qsakurais-certifiedxn--mosjen-eyawaraxn--mot-tlazioxn--mre-og-romsdal-qqbuseranishiaritakurashikis-foundationxn--msy-ula0hakubaghdadultravelchannelxn--mtta-vrjjat-k7aflakstadaokagakicks-assnasaarlandxn--muost-0qaxn--mxtq1minisitexn--ngbc5azdxn--ngbe9e0axn--ngbrxn--45q11citadelhicampinashikiminohostfoldnavyxn--nit225koshimizumakiyosunnydayxn--nmesjevuemie-tcbalestrandabergamoarekeymachineustarnbergxn--nnx388axn--nodessakyotanabelaudiopsysynology-dstreamlitappittsburghofficialxn--nqv7fs00emaxn--nry-yla5gxn--ntso0iqx3axn--ntsq17gxn--nttery-byaeserveftplanetariuminamitanexn--nvuotna-hwaxn--nyqy26axn--o1achernihivgubsxn--o3cw4hakuis-a-democratravelersinsurancexn--o3cyx2axn--od0algxn--od0aq3belementorayoshiokanumazuryukuhashimojibxos3-website-ap-southeast-1xn--ogbpf8flatangerxn--oppegrd-ixaxn--ostery-fyawatahamaxn--osyro-wuaxn--otu796dxn--p1acfedorapeoplegoismailillehammerfeste-ipatriaxn--p1ais-gonexn--pgbs0dhlx3xn--porsgu-sta26fedoraprojectoyotsukaidoxn--pssu33lxn--pssy2uxn--q7ce6axn--q9jyb4cngreaterxn--qcka1pmcpenzaporizhzhiaxn--qqqt11minnesotaketakayamassivegridxn--qxa6axn--qxamsterdamnserverbaniaxn--rady-iraxn--rdal-poaxn--rde-ulaxn--rdy-0nabaris-into-animeetrentin-sued-tirolxn--rennesy-v1axn--rhkkervju-01afeiraquarelleasingujaratoyouraxn--rholt-mragowoltlab-democraciaxn--rhqv96gxn--rht27zxn--rht3dxn--rht61exn--risa-5naturbruksgymnxn--risr-iraxn--rland-uuaxn--rlingen-mxaxn--rmskog-byaxn--rny31hakusanagochihayaakasakawaiishopitsitexn--rovu88bellevuelosangeles3-website-ap-southeast-2xn--rros-granvindafjordxn--rskog-uuaxn--rst-0naturhistorischesxn--rsta-framercanvasxn--rvc1e0am3exn--ryken-vuaxn--ryrvik-byaxn--s-1faithaldenxn--s9brj9cnpyatigorskolecznagatorodoyxn--sandnessjen-ogbellunord-odalombardyn53xn--sandy-yuaxn--sdtirol-n2axn--seral-lraxn--ses554gxn--sgne-graphoxn--4dbgdty6citichernovtsyncloudrangedaluccarbonia-iglesias-carboniaiglesiascarboniaxn--skierv-utazasxn--skjervy-v1axn--skjk-soaxn--sknit-yqaxn--sknland-fxaxn--slat-5natuurwetenschappenginexn--slt-elabcieszynh-servebeero-stageiseiroumuenchencoreapigeelvinckoshunantankmpspawnextdirectrentino-s-tirolxn--smla-hraxn--smna-gratangentlentapisa-geekosugexn--snase-nraxn--sndre-land-0cbeneventochiokinoshimaintenancebinordreisa-hockeynutazurestaticappspaceusercontentateyamaveroykenglandeltaitogitsumitakagiizeasypanelblagrarchaeologyeongbuk0emmafann-arboretumbriamallamaceiobbcg123homepagefrontappchizip61123minsidaarborteaches-yogasawaracingroks-theatree123hjemmesidealerimo-i-rana4u2-localhistorybolzano-altoadigeometre-experts-comptables3-ap-northeast-123miwebcambridgehirn4t3l3p0rtarumizusawabogadobeaemcloud-fr123paginaweberkeleyokosukanrabruzzombieidskoguchikushinonsenasakuchinotsuchiurakawafaicloudineat-url-o-g-i-naval-d-aosta-valleyokote164-b-datacentermezproxyzgoraetnabudejjudaicadaquest-mon-blogueurodirumaceratabuseating-organicbcn-north-123saitamakawabartheshopencraftrainingdyniajuedischesapeakebayernavigationavoi234lima-cityeats3-ap-northeast-20001wwwedeployokozeastasiamunemurorangecloudplatform0xn--snes-poaxn--snsa-roaxn--sr-aurdal-l8axn--sr-fron-q1axn--sr-odal-q1axn--sr-varanger-ggbentleyurihonjournalistjohnikonanporovnobserverxn--srfold-byaxn--srreisa-q1axn--srum-gratis-a-bulls-fanxn--stfold-9xaxn--stjrdal-s1axn--stjrdalshalsen-sqbeppublishproxyusuharavocatanzarowegroweiboltashkentatamotorsitestingivingjemnes3-eu-central-1kappleadpages-12hpalmspringsakerxn--stre-toten-zcbeskidyn-ip24xn--t60b56axn--tckweddingxn--tiq49xqyjelasticbeanstalkhmelnitskiyamarumorimachidaxn--tjme-hraxn--tn0agrocerydxn--tnsberg-q1axn--tor131oxn--trany-yuaxn--trentin-sd-tirol-rzbestbuyshoparenagareyamaizurugbyenvironmentalconservationflashdrivefsnillfjordiscordsezjampaleoceanographics3-website-eu-west-1xn--trentin-sdtirol-7vbetainaboxfuseekloges3-website-sa-east-1xn--trentino-sd-tirol-c3bhzcasertainaioirasebastopologyeongnamegawafflecellclstagemologicaliforniavoues3-eu-west-1xn--trentino-sdtirol-szbielawalbrzycharitypedreamhostersvp4xn--trentinosd-tirol-rzbiellaakesvuemieleccebizenakanotoddeninoheguriitatebayashiibahcavuotnagaivuotnagaokakyotambabybluebitelevisioncilla-speziaxarnetbank8s3-eu-west-2xn--trentinosdtirol-7vbieszczadygeyachimataijiiyamanouchikuhokuryugasakitaurayasudaxn--trentinsd-tirol-6vbievat-band-campaignieznombrendlyngengerdalces3-website-us-east-1xn--trentinsdtirol-nsbifukagawalesundiscountypeformelhusgardeninomiyakonojorpelandiscourses3-website-us-west-1xn--trgstad-r1axn--trna-woaxn--troms-zuaxn--tysvr-vraxn--uc0atvestre-slidrexn--uc0ay4axn--uist22halsakakinokiaxn--uisz3gxn--unjrga-rtarnobrzegyptianxn--unup4yxn--uuwu58axn--vads-jraxn--valle-aoste-ebbtularvikonskowolayangroupiemontexn--valle-d-aoste-ehboehringerikexn--valleaoste-e7axn--valledaoste-ebbvadsoccerxn--vard-jraxn--vegrshei-c0axn--vermgensberater-ctb-hostingxn--vermgensberatung-pwbigvalledaostaobaomoriguchiharag-cloud-championshiphoplixboxenirasakincheonishiazaindependent-commissionishigouvicasinordeste-idclkarasjohkamikitayamatsurindependent-inquest-a-la-masionishiharaxn--vestvgy-ixa6oxn--vg-yiabkhaziaxn--vgan-qoaxn--vgsy-qoa0jelenia-goraxn--vgu402cnsantabarbaraxn--vhquvestre-totennishiawakuraxn--vler-qoaxn--vre-eiker-k8axn--vrggt-xqadxn--vry-yla5gxn--vuq861biharstadotsubetsugaruhrxn--w4r85el8fhu5dnraxn--w4rs40lxn--wcvs22dxn--wgbh1cntjomeldaluroyxn--wgbl6axn--xhq521bihorologyusuisservegame-serverxn--xkc2al3hye2axn--xkc2dl3a5ee0hammarfeastafricaravantaaxn--y9a3aquariumintereitrentino-sudtirolxn--yer-znaumburgxn--yfro4i67oxn--ygarden-p1axn--ygbi2ammxn--4dbrk0cexn--ystre-slidre-ujbikedaejeonbukarasjokarasuyamarriottatsunoceanographiquehimejindependent-inquiryuufcfanishiizunazukindependent-panelomoliseminemrxn--zbx025dxn--zf0ao64axn--zf0avxlxn--zfr164bilbaogashimadachicagoboavistanbulsan-sudtirolbia-tempio-olbiatempioolbialystokkeliwebredirectme-south-1xnbayxz \ No newline at end of file +birkenesoddtangentinglogoweirbitbucketrzynishikatakayamatta-varjjatjomembersaltdalovepopartysfjordiskussionsbereichatinhlfanishikatsuragitappassenger-associationishikawazukamiokameokamakurazakitaurayasudabitternidisrechtrainingloomy-routerbjarkoybjerkreimdbalsan-suedtirololitapunkapsienamsskoganeibmdeveloperauniteroirmemorialombardiadempresashibetsukumiyamagasakinderoyonagunicloudevelopmentaxiijimarriottayninhaccanthobby-siteval-d-aosta-valleyoriikaracolognebinatsukigataiwanumatajimidsundgcahcesuolocustomer-ocimperiautoscanalytics-gatewayonagoyaveroykenflfanpachihayaakasakawaiishopitsitemasekd1kappenginedre-eikerimo-siemenscaledekaascolipicenoboribetsucks3-eu-west-3utilities-16-balestrandabergentappsseekloges3-eu-west-123paginawebcamauction-acornfshostrodawaraktyubinskaunicommbank123kotisivultrobjectselinogradimo-i-rana4u2-localhostrolekanieruchomoscientistordal-o-g-i-nikolaevents3-ap-northeast-2-ddnsking123homepagefrontappchizip61123saitamakawababia-goracleaningheannakadomarineat-urlimanowarudakuneustarostwodzislawdev-myqnapcloudcontrolledgesuite-stagingdyniamusementdllclstagehirnikonantomobelementorayokosukanoyakumoliserniaurland-4-salernord-aurdalipaywhirlimiteddnslivelanddnss3-ap-south-123siteweberlevagangaviikanonji234lima-cityeats3-ap-southeast-123webseiteambulancechireadmyblogspotaribeiraogakicks-assurfakefurniturealmpmninoheguribigawaurskog-holandinggfarsundds3-ap-southeast-20001wwwedeployokote123hjemmesidealerdalaheadjuegoshikibichuobiraustevollimombetsupplyokoze164-balena-devices3-ca-central-123websiteleaf-south-12hparliamentatsunobninsk8s3-eu-central-1337bjugnishimerablackfridaynightjxn--11b4c3ditchyouripatriabloombergretaijindustriesteinkjerbloxcmsaludivtasvuodnakaiwanairlinekobayashimodatecnologiablushakotanishinomiyashironomniwebview-assetsalvadorbmoattachmentsamegawabmsamnangerbmwellbeingzonebnrweatherchannelsdvrdnsamparalleluxenishinoomotegotsukishiwadavvenjargamvikarpaczest-a-la-maisondre-landivttasvuotnakamai-stagingloppennebomlocalzonebonavstackartuzybondigitaloceanspacesamsclubartowest1-usamsunglugsmall-webspacebookonlineboomlaakesvuemielecceboschristmasakilatiron-riopretoeidsvollovesickaruizawabostik-serverrankoshigayachtsandvikcoromantovalle-d-aostakinouebostonakijinsekikogentlentapisa-geekarumaifmemsetkmaxxn--12c1fe0bradescotksatmpaviancapitalonebouncemerckmsdscloudiybounty-fullensakerrypropertiesangovtoyosatoyokawaboutiquebecologialaichaugiangmbhartiengiangminakamichiharaboutireservdrangedalpusercontentoyotapfizerboyfriendoftheinternetflixn--12cfi8ixb8lublindesnesanjosoyrovnoticiasannanishinoshimattelemarkasaokamikitayamatsurinfinitigopocznore-og-uvdalucaniabozen-sudtiroluccanva-appstmnishiokoppegardray-dnsupdaterbozen-suedtirolukowesteuropencraftoyotomiyazakinsurealtypeformesswithdnsannohekinanporovigonohejinternationaluroybplacedogawarabikomaezakirunordkappgfoggiabrandrayddns5ybrasiliadboxoslockerbresciaogashimadachicappadovaapstemp-dnswatchest-mon-blogueurodirumagazinebrindisiciliabroadwaybroke-itvedestrandraydnsanokashibatakashimashikiyosatokigawabrokerbrothermesserlifestylebtimnetzpisdnpharmaciensantamariakebrowsersafetymarketingmodumetacentrumeteorappharmacymruovatlassian-dev-builderschaefflerbrumunddalutskashiharabrusselsantoandreclaimsanukintlon-2bryanskiptveterinaireadthedocsaobernardovre-eikerbrynebwestus2bzhitomirbzzwhitesnowflakecommunity-prochowicecomodalenissandoycompanyaarphdfcbankasumigaurawa-mazowszexn--1ck2e1bambinagisobetsuldalpha-myqnapcloudaccess3-us-east-2ixboxeroxfinityolasiteastus2comparemarkerryhotelsaves-the-whalessandria-trani-barletta-andriatranibarlettaandriacomsecaasnesoddeno-stagingrondarcondoshifteditorxn--1ctwolominamatarnobrzegrongrossetouchijiwadedyn-berlincolnissayokoshibahikariyaltakazakinzais-a-bookkeepermarshallstatebankasuyalibabahccavuotnagaraholtaleniwaizumiotsurugashimaintenanceomutazasavonarviikaminoyamaxunispaceconferenceconstructionflashdrivefsncf-ipfsaxoconsuladobeio-static-accesscamdvrcampaniaconsultantranoyconsultingroundhandlingroznysaitohnoshookuwanakayamangyshlakdnepropetrovskanlandyndns-freeboxostrowwlkpmgrphilipsyno-dschokokekscholarshipschoolbusinessebycontactivetrailcontagematsubaravendbambleborkdalvdalcest-le-patron-rancherkasydneyukuhashimokawavoues3-sa-east-1contractorskenissedalcookingruecoolblogdnsfor-better-thanhhoarairforcentralus-1cooperativano-frankivskodjeephonefosschoolsztynsetransiphotographysiocoproductionschulplattforminamiechizenisshingucciprianiigatairaumalatvuopmicrolightinguidefinimaringatlancastercorsicafjschulservercosenzakopanecosidnshome-webservercellikescandypopensocialcouchpotatofrieschwarzgwangjuh-ohtawaramotoineppueblockbusternopilawacouncilcouponscrapper-sitecozoravennaharimalborkaszubytemarketscrappinguitarscrysecretrosnubananarepublic-inquiryurihonjoyenthickaragandaxarnetbankanzakiwielunnerepairbusanagochigasakishimabarakawaharaolbia-tempio-olbiatempioolbialowiezachpomorskiengiangjesdalolipopmcdirepbodyn53cqcxn--1lqs03niyodogawacrankyotobetsumidaknongujaratmallcrdyndns-homednscwhminamifuranocreditcardyndns-iphutholdingservehttpbincheonl-ams-1creditunionionjukujitawaravpagecremonashorokanaiecrewhoswholidaycricketnedalcrimeast-kazakhstanangercrotonecrowniphuyencrsvp4cruiseservehumourcuisinellair-traffic-controllagdenesnaaseinet-freakserveircasertainaircraftingvolloansnasaarlanduponthewifidelitypedreamhostersaotomeldaluxurycuneocupcakecuritibacgiangiangryggeecurvalled-aostargets-itranslatedyndns-mailcutegirlfriendyndns-office-on-the-webhoptogurafedoraprojectransurlfeirafembetsukuis-a-bruinsfanfermodenakasatsunairportrapaniizaferraraferraris-a-bulls-fanferrerotikagoshimalopolskanittedalfetsundyndns-wikimobetsumitakagildeskaliszkolamericanfamilydservemp3fgunmaniwamannorth-kazakhstanfhvalerfilegear-augustowiiheyakagefilegear-deatnuniversitysvardofilegear-gbizfilegear-iefilegear-jpmorgangwonporterfilegear-sg-1filminamiizukamiminefinalchikugokasellfyis-a-candidatefinancefinnoyfirebaseappiemontefirenetlifylkesbiblackbaudcdn-edgestackhero-networkinggroupowiathletajimabaria-vungtaudiopsysharpigboatshawilliamhillfirenzefirestonefireweblikes-piedmontravelersinsurancefirmdalegalleryfishingoldpoint2thisamitsukefitjarfitnessettsurugiminamimakis-a-catererfjalerfkatsushikabeebyteappilottonsberguovdageaidnunjargausdalflekkefjordyndns-workservep2phxn--1lqs71dyndns-remotewdyndns-picserveminecraftransporteflesbergushikamifuranorthflankatsuyamashikokuchuoflickragerokunohealthcareershellflierneflirfloginlinefloppythonanywherealtorfloraflorencefloripalmasfjordenfloristanohatajiris-a-celticsfanfloromskogxn--2m4a15eflowershimokitayamafltravinhlonganflynnhosting-clusterfncashgabadaddjabbottoyourafndyndns1fnwkzfolldalfoolfor-ourfor-somegurownproviderfor-theaterfordebianforexrotheworkpccwinbar0emmafann-arborlandd-dnsiskinkyowariasahikawarszawashtenawsmppl-wawsglobalacceleratorahimeshimakanegasakievennodebalancern4t3l3p0rtatarantours3-ap-northeast-123minsidaarborteaches-yogano-ipifony-123miwebaccelastx4432-b-datacenterprisesakijobservableusercontentateshinanomachintaifun-dnsdojournalistoloseyouriparisor-fronavuotnarashinoharaetnabudejjunipereggio-emilia-romagnaroyboltateyamajureggiocalabriakrehamnayoro0o0forgotdnshimonitayanagithubpreviewsaikisarazure-mobileirfjordynnservepicservequakeforli-cesena-forlicesenaforlillehammerfeste-ipimientaketomisatoolshimonosekikawaforsalegoismailillesandefjordynservebbservesarcasmileforsandasuolodingenfortalfortefosneshimosuwalkis-a-chefashionstorebaseljordyndns-serverisignfotrdynulvikatowicefoxn--2scrj9casinordlandurbanamexnetgamersapporomurafozfr-1fr-par-1fr-par-2franamizuhoboleslawiecommerce-shoppingyeongnamdinhachijohanamakisofukushimaoris-a-conservativegarsheiheijis-a-cparachutingfredrikstadynv6freedesktopazimuthaibinhphuocelotenkawakayamagnetcieszynh-servebeero-stageiseiroumugifuchungbukharag-cloud-championshiphoplixn--30rr7yfreemyiphosteurovisionredumbrellangevagrigentobishimadridvagsoygardenebakkeshibechambagricoharugbydgoszczecin-berlindasdaburfreesitefreetlshimotsukefreisennankokubunjis-a-cubicle-slavellinodeobjectshimotsumafrenchkisshikindleikangerfreseniushinichinanfriuli-v-giuliafriuli-ve-giuliafriuli-vegiuliafriuli-venezia-giuliafriuli-veneziagiuliafriuli-vgiuliafriuliv-giuliafriulive-giuliafriulivegiuliafriulivenezia-giuliafriuliveneziagiuliafriulivgiuliafrlfroganshinjotelulubin-vpncateringebunkyonanaoshimamateramockashiwarafrognfrolandynvpnpluservicesevastopolitiendafrom-akamaized-stagingfrom-alfrom-arfrom-azurewebsiteshikagamiishibuyabukihokuizumobaragusabaerobaticketshinjukuleuvenicefrom-campobassociatest-iserveblogsytenrissadistdlibestadultrentin-sudtirolfrom-coachaseljeducationcillahppiacenzaganfrom-ctrentin-sued-tirolfrom-dcatfooddagestangefrom-decagliarikuzentakataikillfrom-flapymntrentin-suedtirolfrom-gap-east-1from-higashiagatsumagoianiafrom-iafrom-idyroyrvikingulenfrom-ilfrom-in-the-bandairtelebitbridgestonemurorangecloudplatform0from-kshinkamigototalfrom-kyfrom-langsonyantakahamalselveruminamiminowafrom-malvikaufentigerfrom-mdfrom-mein-vigorlicefrom-mifunefrom-mnfrom-modshinshinotsurgeryfrom-mshinshirofrom-mtnfrom-ncatholicurus-4from-ndfrom-nefrom-nhs-heilbronnoysundfrom-njshintokushimafrom-nminamioguni5from-nvalledaostargithubusercontentrentino-a-adigefrom-nycaxiaskvollpagesardegnarutolgaulardalvivanovoldafrom-ohdancefrom-okegawassamukawataris-a-democratrentino-aadigefrom-orfrom-panasonichernovtsykkylvenneslaskerrylogisticsardiniafrom-pratohmamurogawatsonrenderfrom-ris-a-designerimarugame-hostyhostingfrom-schmidtre-gauldalfrom-sdfrom-tnfrom-txn--32vp30hachinoheavyfrom-utsiracusagaeroclubmedecin-addrammenuorodoyerfrom-val-daostavalleyfrom-vtrentino-alto-adigefrom-wafrom-wiardwebthingsjcbnpparibashkiriafrom-wvallee-aosteroyfrom-wyfrosinonefrostabackplaneapplebesbyengerdalp1froyal-commissionfruskydivingfujiiderafujikawaguchikonefujiminokamoenairtrafficplexus-2fujinomiyadapliefujiokazakinkobearalvahkikonaibetsubame-south-1fujisatoshoeshintomikasaharafujisawafujishiroishidakabiratoridediboxn--3bst00minamisanrikubetsupportrentino-altoadigefujitsuruokakamigaharafujiyoshidappnodearthainguyenfukayabeardubaikawagoefukuchiyamadatsunanjoburgfukudomigawafukuis-a-doctorfukumitsubishigakirkeneshinyoshitomiokamisatokamachippubetsuikitchenfukuokakegawafukuroishikariwakunigamigrationfukusakirovogradoyfukuyamagatakaharunusualpersonfunabashiriuchinadattorelayfunagatakahashimamakiryuohkurafunahashikamiamakusatsumasendaisenergyeongginowaniihamatamakinoharafundfunkfeuerfuoiskujukuriyamandalfuosskoczowindowskrakowinefurubirafurudonordreisa-hockeynutwentertainmentrentino-s-tirolfurukawajimangolffanshiojirishirifujiedafusoctrangfussagamiharafutabayamaguchinomihachimanagementrentino-stirolfutboldlygoingnowhere-for-more-og-romsdalfuttsurutashinais-a-financialadvisor-aurdalfuturecmshioyamelhushirahamatonbetsurnadalfuturehostingfuturemailingfvghakuis-a-gurunzenhakusandnessjoenhaldenhalfmoonscalebookinghostedpictetrentino-sud-tirolhalsakakinokiaham-radio-opinbar1hamburghammarfeastasiahamurakamigoris-a-hard-workershiraokamisunagawahanamigawahanawahandavvesiidanangodaddyn-o-saurealestatefarmerseinehandcrafteducatorprojectrentino-sudtirolhangglidinghangoutrentino-sued-tirolhannannestadhannosegawahanoipinkazohanyuzenhappouzshiratakahagianghasamap-northeast-3hasaminami-alpshishikuis-a-hunterhashbanghasudazaifudaigodogadobeioruntimedio-campidano-mediocampidanomediohasura-appinokokamikoaniikappudopaashisogndalhasvikazteleportrentino-suedtirolhatogayahoooshikamagayaitakamoriokakudamatsuehatoyamazakitahiroshimarcheapartmentshisuifuettertdasnetzhatsukaichikaiseiyoichipshitaramahattfjelldalhayashimamotobusells-for-lesshizukuishimoichilloutsystemscloudsitehazuminobushibukawahelplfinancialhelsinkitakamiizumisanofidonnakamurataitogliattinnhemneshizuokamitondabayashiogamagoriziahemsedalhepforgeblockshoujis-a-knightpointtokaizukamaishikshacknetrentinoa-adigehetemlbfanhigashichichibuzentsujiiehigashihiroshimanehigashiizumozakitakatakanabeautychyattorneyagawakkanaioirasebastopoleangaviikadenagahamaroyhigashikagawahigashikagurasoedahigashikawakitaaikitakyushunantankazunovecorebungoonow-dnshowahigashikurumeinforumzhigashimatsushimarnardalhigashimatsuyamakitaakitadaitoigawahigashimurayamamotorcycleshowtimeloyhigashinarusells-for-uhigashinehigashiomitamanoshiroomghigashiosakasayamanakakogawahigashishirakawamatakanezawahigashisumiyoshikawaminamiaikitamihamadahigashitsunospamproxyhigashiurausukitamotosunnydayhigashiyamatokoriyamanashiibaclieu-1higashiyodogawahigashiyoshinogaris-a-landscaperspectakasakitanakagusukumoldeliveryhippyhiraizumisatohokkaidontexistmein-iservschulecznakaniikawatanagurahirakatashinagawahiranais-a-lawyerhirarahiratsukaeruhirayaizuwakamatsubushikusakadogawahitachiomiyaginozawaonsensiositehitachiotaketakaokalmykiahitraeumtgeradegreehjartdalhjelmelandholyhomegoodshwinnersiiitesilkddiamondsimple-urlhomeipioneerhomelinkyard-cloudjiffyresdalhomelinuxn--3ds443ghomeofficehomesecuritymacaparecidahomesecuritypchiryukyuragiizehomesenseeringhomeskleppippugliahomeunixn--3e0b707ehondahonjyoitakarazukaluganskfh-muensterhornindalhorsells-itrentinoaadigehortendofinternet-dnsimplesitehospitalhotelwithflightsirdalhotmailhoyangerhoylandetakasagooglecodespotrentinoalto-adigehungyenhurdalhurumajis-a-liberalhyllestadhyogoris-a-libertarianhyugawarahyundaiwafuneis-very-evillasalleitungsenis-very-goodyearis-very-niceis-very-sweetpepperugiais-with-thebandoomdnstraceisk01isk02jenv-arubacninhbinhdinhktistoryjeonnamegawajetztrentinostiroljevnakerjewelryjgorajlljls-sto1jls-sto2jls-sto3jmpixolinodeusercontentrentinosud-tiroljnjcloud-ver-jpchitosetogitsuliguriajoyokaichibahcavuotnagaivuotnagaokakyotambabymilk3jozis-a-musicianjpnjprsolarvikhersonlanxessolundbeckhmelnitskiyamasoykosaigawakosakaerodromegalloabatobamaceratachikawafaicloudineencoreapigeekoseis-a-painterhostsolutionslupskhakassiakosheroykoshimizumakis-a-patsfankoshughesomakosugekotohiradomainstitutekotourakouhokumakogenkounosupersalevangerkouyamasudakouzushimatrixn--3pxu8khplaystation-cloudyclusterkozagawakozakis-a-personaltrainerkozowiosomnarviklabudhabikinokawachinaganoharamcocottekpnkppspbarcelonagawakepnord-odalwaysdatabaseballangenkainanaejrietisalatinabenogiehtavuoatnaamesjevuemielnombrendlyngen-rootaruibxos3-us-gov-west-1krasnikahokutokonamegatakatoris-a-photographerokussldkrasnodarkredstonekrelliankristiansandcatsoowitdkmpspawnextdirectrentinosudtirolkristiansundkrodsheradkrokstadelvaldaostavangerkropyvnytskyis-a-playershiftcryptonomichinomiyakekryminamiyamashirokawanabelaudnedalnkumamotoyamatsumaebashimofusakatakatsukis-a-republicanonoichinosekigaharakumanowtvaokumatorinokumejimatsumotofukekumenanyokkaichirurgiens-dentistes-en-francekundenkunisakis-a-rockstarachowicekunitachiaraisaijolsterkunitomigusukukis-a-socialistgstagekunneppubtlsopotrentinosued-tirolkuokgroupizzakurgankurobegetmyipirangalluplidlugolekagaminorddalkurogimimozaokinawashirosatochiokinoshimagentositempurlkuroisodegaurakuromatsunais-a-soxfankuronkurotakikawasakis-a-studentalkushirogawakustanais-a-teacherkassyncloudkusuppliesor-odalkutchanelkutnokuzumakis-a-techietipslzkvafjordkvalsundkvamsterdamnserverbaniakvanangenkvinesdalkvinnheradkviteseidatingkvitsoykwpspdnsor-varangermishimatsusakahogirlymisugitokorozawamitakeharamitourismartlabelingmitoyoakemiuramiyazurecontainerdpoliticaobangmiyotamatsukuris-an-actormjondalenmonzabrianzaramonzaebrianzamonzaedellabrianzamordoviamorenapolicemoriyamatsuuramoriyoshiminamiashigaramormonstermoroyamatsuzakis-an-actressmushcdn77-sslingmortgagemoscowithgoogleapiszmoseushimogosenmosjoenmoskenesorreisahayakawakamiichikawamisatottoris-an-anarchistjordalshalsenmossortlandmosviknx-serversusakiyosupabaseminemotegit-reposoruminanomoviemovimientokyotangotembaixadattowebhareidsbergmozilla-iotrentinosuedtirolmtranbytomaridagawalmartrentinsud-tirolmuikaminokawanishiaizubangemukoelnmunakatanemuosattemupkomatsushimassa-carrara-massacarraramassabuzzmurmanskomforbar2murotorcraftranakatombetsumy-gatewaymusashinodesakegawamuseumincomcastoripressorfoldmusicapetownnews-stagingmutsuzawamy-vigormy-wanggoupilemyactivedirectorymyamazeplaymyasustor-elvdalmycdmycloudnsoundcastorjdevcloudfunctionsokndalmydattolocalcertificationmyddnsgeekgalaxymydissentrentinsudtirolmydobissmarterthanyoumydrobofageometre-experts-comptablesowamydspectruminisitemyeffectrentinsued-tirolmyfastly-edgekey-stagingmyfirewalledreplittlestargardmyforuminterecifedextraspace-to-rentalstomakomaibaramyfritzmyftpaccesspeedpartnermyhome-servermyjinomykolaivencloud66mymailermymediapchoseikarugalsacemyokohamamatsudamypeplatformsharis-an-artistockholmestrandmypetsphinxn--41amyphotoshibajddarvodkafjordvaporcloudmypictureshinomypsxn--42c2d9amysecuritycamerakermyshopblockspjelkavikommunalforbundmyshopifymyspreadshopselectrentinsuedtirolmytabitordermythic-beastspydebergmytis-a-anarchistg-buildermytuleap-partnersquaresindevicenzamyvnchoshichikashukudoyamakeuppermywirecipescaracallypoivronpokerpokrovskommunepolkowicepoltavalle-aostavernpomorzeszowithyoutuberspacekitagawaponpesaro-urbino-pesarourbinopesaromasvuotnaritakurashikis-bykleclerchitachinakagawaltervistaipeigersundynamic-dnsarlpordenonepornporsangerporsangugeporsgrunnanpoznanpraxihuanprdprgmrprimetelprincipeprivatelinkomonowruzhgorodeoprivatizehealthinsuranceprofesionalprogressivegasrlpromonza-e-della-brianzaptokuyamatsushigepropertysnesrvarggatrevisogneprotectionprotonetroandindependent-inquest-a-la-masionprudentialpruszkowiwatsukiyonotaireserve-onlineprvcyonabarumbriaprzeworskogpunyufuelpupulawypussycatanzarowixsitepvhachirogatakahatakaishimojis-a-geekautokeinotteroypvtrogstadpwchowderpzqhadanorthwesternmutualqldqotoyohashimotoshimaqponiatowadaqslgbtroitskomorotsukagawaqualifioapplatter-applatterplcube-serverquangngais-certifiedugit-pagespeedmobilizeroticaltanissettailscaleforcequangninhthuanquangtritonoshonais-foundationquickconnectromsakuragawaquicksytestreamlitapplumbingouvaresearchitectesrhtrentoyonakagyokutoyakomakizunokunimimatakasugais-an-engineeringquipelementstrippertuscanytushungrytuvalle-daostamayukis-into-animeiwamizawatuxfamilytuyenquangbinhthuantwmailvestnesuzukis-gonevestre-slidreggio-calabriavestre-totennishiawakuravestvagoyvevelstadvibo-valentiaavibovalentiavideovinhphuchromedicinagatorogerssarufutsunomiyawakasaikaitakokonoevinnicarbonia-iglesias-carboniaiglesiascarboniavinnytsiavipsinaapplurinacionalvirginanmokurennebuvirtual-userveexchangevirtualservervirtualuserveftpodhalevisakurais-into-carsnoasakuholeckodairaviterboliviajessheimmobilienvivianvivoryvixn--45br5cylvlaanderennesoyvladikavkazimierz-dolnyvladimirvlogintoyonezawavmintsorocabalashovhachiojiyahikobierzycevologdanskoninjambylvolvolkswagencyouvolyngdalvoorlopervossevangenvotevotingvotoyonovps-hostrowiechungnamdalseidfjordynathomebuiltwithdarkhangelskypecorittogojomeetoystre-slidrettozawawmemergencyahabackdropalermochizukikirarahkkeravjuwmflabsvalbardunloppadualstackomvuxn--3hcrj9chonanbuskerudynamisches-dnsarpsborgripeeweeklylotterywoodsidellogliastradingworse-thanhphohochiminhadselbuyshouseshirakolobrzegersundongthapmircloudletshiranukamishihorowowloclawekonskowolawawpdevcloudwpenginepoweredwphostedmailwpmucdnipropetrovskygearappodlasiellaknoluoktagajobojis-an-entertainerwpmudevcdnaccessojamparaglidingwritesthisblogoipodzonewroclawmcloudwsseoullensvanguardianwtcp4wtfastlylbanzaicloudappspotagereporthruherecreationinomiyakonojorpelandigickarasjohkameyamatotakadawuozuerichardlillywzmiuwajimaxn--4it797konsulatrobeepsondriobranconagareyamaizuruhrxn--4pvxs4allxn--54b7fta0ccistrondheimpertrixcdn77-secureadymadealstahaugesunderxn--55qw42gxn--55qx5dxn--5dbhl8dxn--5js045dxn--5rtp49citadelhichisochimkentozsdell-ogliastraderxn--5rtq34kontuminamiuonumatsunoxn--5su34j936bgsgxn--5tzm5gxn--6btw5axn--6frz82gxn--6orx2rxn--6qq986b3xlxn--7t0a264citicarrdrobakamaiorigin-stagingmxn--12co0c3b4evalleaostaobaomoriguchiharaffleentrycloudflare-ipfstcgroupaaskimitsubatamibulsan-suedtirolkuszczytnoopscbgrimstadrrxn--80aaa0cvacationsvchoyodobashichinohealth-carereforminamidaitomanaustdalxn--80adxhksveioxn--80ao21axn--80aqecdr1axn--80asehdbarclaycards3-us-west-1xn--80aswgxn--80aukraanghkeliwebpaaskoyabeagleboardxn--8dbq2axn--8ltr62konyvelohmusashimurayamassivegridxn--8pvr4uxn--8y0a063axn--90a1affinitylotterybnikeisencowayxn--90a3academiamicable-modemoneyxn--90aeroportsinfolionetworkangerxn--90aishobaraxn--90amckinseyxn--90azhytomyrxn--9dbq2axn--9et52uxn--9krt00axn--andy-iraxn--aroport-byanagawaxn--asky-iraxn--aurskog-hland-jnbarclays3-us-west-2xn--avery-yuasakurastoragexn--b-5gaxn--b4w605ferdxn--balsan-sdtirol-nsbsvelvikongsbergxn--bck1b9a5dre4civilaviationfabricafederation-webredirectmediatechnologyeongbukashiwazakiyosembokutamamuraxn--bdddj-mrabdxn--bearalvhki-y4axn--berlevg-jxaxn--bhcavuotna-s4axn--bhccavuotna-k7axn--bidr-5nachikatsuuraxn--bievt-0qa2xn--bjarky-fyanaizuxn--bjddar-ptarumizusawaxn--blt-elabcienciamallamaceiobbcn-north-1xn--bmlo-graingerxn--bod-2natalxn--bozen-sdtirol-2obanazawaxn--brnny-wuacademy-firewall-gatewayxn--brnnysund-m8accident-investigation-aptibleadpagesquare7xn--brum-voagatrustkanazawaxn--btsfjord-9zaxn--bulsan-sdtirol-nsbarefootballooningjovikarasjoketokashikiyokawaraxn--c1avgxn--c2br7gxn--c3s14misakis-a-therapistoiaxn--cck2b3baremetalombardyn-vpndns3-website-ap-northeast-1xn--cckwcxetdxn--cesena-forl-mcbremangerxn--cesenaforl-i8axn--cg4bkis-into-cartoonsokamitsuexn--ciqpnxn--clchc0ea0b2g2a9gcdxn--czr694bargainstantcloudfrontdoorestauranthuathienhuebinordre-landiherokuapparochernigovernmentjeldsundiscordsays3-website-ap-southeast-1xn--czrs0trvaroyxn--czru2dxn--czrw28barrel-of-knowledgeapplinziitatebayashijonawatebizenakanojoetsumomodellinglassnillfjordiscordsezgoraxn--d1acj3barrell-of-knowledgecomputermezproxyzgorzeleccoffeedbackanagawarmiastalowa-wolayangroupars3-website-ap-southeast-2xn--d1alfaststacksevenassigdalxn--d1atrysiljanxn--d5qv7z876clanbibaiduckdnsaseboknowsitallxn--davvenjrga-y4axn--djrs72d6uyxn--djty4koobindalxn--dnna-grajewolterskluwerxn--drbak-wuaxn--dyry-iraxn--e1a4cldmail-boxaxn--eckvdtc9dxn--efvn9svn-repostuff-4-salexn--efvy88haebaruericssongdalenviknaklodzkochikushinonsenasakuchinotsuchiurakawaxn--ehqz56nxn--elqq16hagakhanhhoabinhduongxn--eveni-0qa01gaxn--f6qx53axn--fct429kooris-a-nascarfanxn--fhbeiarnxn--finny-yuaxn--fiq228c5hsbcleverappsassarinuyamashinazawaxn--fiq64barsycenterprisecloudcontrolappgafanquangnamasteigenoamishirasatochigifts3-website-eu-west-1xn--fiqs8swidnicaravanylvenetogakushimotoganexn--fiqz9swidnikitagatakkomaganexn--fjord-lraxn--fjq720axn--fl-ziaxn--flor-jraxn--flw351exn--forl-cesena-fcbsswiebodzindependent-commissionxn--forlcesena-c8axn--fpcrj9c3dxn--frde-granexn--frna-woaxn--frya-hraxn--fzc2c9e2clickrisinglesjaguarxn--fzys8d69uvgmailxn--g2xx48clinicasacampinagrandebungotakadaemongolianishitosashimizunaminamiawajikintuitoyotsukaidownloadrudtvsaogoncapooguyxn--gckr3f0fastvps-serveronakanotoddenxn--gecrj9cliniquedaklakasamatsudoesntexisteingeekasserversicherungroks-theatrentin-sud-tirolxn--ggaviika-8ya47hagebostadxn--gildeskl-g0axn--givuotna-8yandexcloudxn--gjvik-wuaxn--gk3at1exn--gls-elacaixaxn--gmq050is-into-gamessinamsosnowieconomiasadojin-dslattuminamitanexn--gmqw5axn--gnstigbestellen-zvbrplsbxn--45brj9churcharterxn--gnstigliefern-wobihirosakikamijimayfirstorfjordxn--h-2failxn--h1ahnxn--h1alizxn--h2breg3eveneswinoujsciencexn--h2brj9c8clothingdustdatadetectrani-andria-barletta-trani-andriaxn--h3cuzk1dienbienxn--hbmer-xqaxn--hcesuolo-7ya35barsyonlinehimejiiyamanouchikujoinvilleirvikarasuyamashikemrevistathellequipmentjmaxxxjavald-aostatics3-website-sa-east-1xn--hebda8basicserversejny-2xn--hery-iraxn--hgebostad-g3axn--hkkinen-5waxn--hmmrfeasta-s4accident-prevention-k3swisstufftoread-booksnestudioxn--hnefoss-q1axn--hobl-iraxn--holtlen-hxaxn--hpmir-xqaxn--hxt814exn--hyanger-q1axn--hylandet-54axn--i1b6b1a6a2exn--imr513nxn--indery-fyaotsusonoxn--io0a7is-leetrentinoaltoadigexn--j1adpohlxn--j1aefauskedsmokorsetagayaseralingenovaraxn--j1ael8basilicataniaxn--j1amhaibarakisosakitahatakamatsukawaxn--j6w193gxn--jlq480n2rgxn--jlster-byasakaiminatoyookananiimiharuxn--jrpeland-54axn--jvr189misasaguris-an-accountantsmolaquilaocais-a-linux-useranishiaritabashikaoizumizakitashiobaraxn--k7yn95exn--karmy-yuaxn--kbrq7oxn--kcrx77d1x4axn--kfjord-iuaxn--klbu-woaxn--klt787dxn--kltp7dxn--kltx9axn--klty5xn--45q11circlerkstagentsasayamaxn--koluokta-7ya57haiduongxn--kprw13dxn--kpry57dxn--kput3is-lostre-toteneis-a-llamarumorimachidaxn--krager-gyasugitlabbvieeexn--kranghke-b0axn--krdsherad-m8axn--krehamn-dxaxn--krjohka-hwab49jdfastly-terrariuminamiiseharaxn--ksnes-uuaxn--kvfjord-nxaxn--kvitsy-fyasuokanmakiwakuratexn--kvnangen-k0axn--l-1fairwindsynology-diskstationxn--l1accentureklamborghinikkofuefukihabororosynology-dsuzakadnsaliastudynaliastrynxn--laheadju-7yatominamibosoftwarendalenugxn--langevg-jxaxn--lcvr32dxn--ldingen-q1axn--leagaviika-52basketballfinanzjaworznoticeableksvikaratsuginamikatagamilanotogawaxn--lesund-huaxn--lgbbat1ad8jejuxn--lgrd-poacctulaspeziaxn--lhppi-xqaxn--linds-pramericanexpresservegame-serverxn--loabt-0qaxn--lrdal-sraxn--lrenskog-54axn--lt-liacn-northwest-1xn--lten-granvindafjordxn--lury-iraxn--m3ch0j3axn--mely-iraxn--merker-kuaxn--mgb2ddesxn--mgb9awbfbsbxn--1qqw23axn--mgba3a3ejtunesuzukamogawaxn--mgba3a4f16axn--mgba3a4fra1-deloittexn--mgba7c0bbn0axn--mgbaakc7dvfsxn--mgbaam7a8haiphongonnakatsugawaxn--mgbab2bdxn--mgbah1a3hjkrdxn--mgbai9a5eva00batsfjordiscountry-snowplowiczeladzlgleezeu-2xn--mgbai9azgqp6jelasticbeanstalkharkovalleeaostexn--mgbayh7gparasitexn--mgbbh1a71exn--mgbc0a9azcgxn--mgbca7dzdoxn--mgbcpq6gpa1axn--mgberp4a5d4a87gxn--mgberp4a5d4arxn--mgbgu82axn--mgbi4ecexposedxn--mgbpl2fhskopervikhmelnytskyivalleedaostexn--mgbqly7c0a67fbcngroks-thisayamanobeatsaudaxn--mgbqly7cvafricargoboavistanbulsan-sudtirolxn--mgbt3dhdxn--mgbtf8flatangerxn--mgbtx2bauhauspostman-echofunatoriginstances3-website-us-east-1xn--mgbx4cd0abkhaziaxn--mix082fbx-osewienxn--mix891fbxosexyxn--mjndalen-64axn--mk0axindependent-inquiryxn--mk1bu44cnpyatigorskjervoyagexn--mkru45is-not-certifiedxn--mlatvuopmi-s4axn--mli-tlavagiskexn--mlselv-iuaxn--moreke-juaxn--mori-qsakuratanxn--mosjen-eyatsukannamihokksundxn--mot-tlavangenxn--mre-og-romsdal-qqbuservecounterstrikexn--msy-ula0hair-surveillancexn--mtta-vrjjat-k7aflakstadaokayamazonaws-cloud9guacuiababybluebiteckidsmynasushiobaracingrok-freeddnsfreebox-osascoli-picenogatabuseating-organicbcgjerdrumcprequalifymelbourneasypanelblagrarq-authgear-stagingjerstadeltaishinomakilovecollegefantasyleaguenoharauthgearappspacehosted-by-previderehabmereitattoolforgerockyombolzano-altoadigeorgeorgiauthordalandroideporteatonamidorivnebetsukubankanumazuryomitanocparmautocodebergamoarekembuchikumagayagawafflecelloisirs3-external-180reggioemiliaromagnarusawaustrheimbalsan-sudtirolivingitpagexlivornobserveregruhostingivestbyglandroverhalladeskjakamaiedge-stagingivingjemnes3-eu-west-2038xn--muost-0qaxn--mxtq1misawaxn--ngbc5azdxn--ngbe9e0axn--ngbrxn--4dbgdty6ciscofreakamaihd-stagingriwataraindroppdalxn--nit225koryokamikawanehonbetsuwanouchikuhokuryugasakis-a-nursellsyourhomeftpiwatexn--nmesjevuemie-tcbalatinord-frontierxn--nnx388axn--nodessakurawebsozais-savedxn--nqv7fs00emaxn--nry-yla5gxn--ntso0iqx3axn--ntsq17gxn--nttery-byaeservehalflifeinsurancexn--nvuotna-hwaxn--nyqy26axn--o1achernivtsicilynxn--4dbrk0cexn--o3cw4hakatanortonkotsunndalxn--o3cyx2axn--od0algardxn--od0aq3beneventodayusuharaxn--ogbpf8fldrvelvetromsohuissier-justicexn--oppegrd-ixaxn--ostery-fyatsushiroxn--osyro-wuaxn--otu796dxn--p1acfedjeezxn--p1ais-slickharkivallee-d-aostexn--pgbs0dhlx3xn--porsgu-sta26fedorainfraclouderaxn--pssu33lxn--pssy2uxn--q7ce6axn--q9jyb4cnsauheradyndns-at-homedepotenzamamicrosoftbankasukabedzin-brbalsfjordietgoryoshiokanravocats3-fips-us-gov-west-1xn--qcka1pmcpenzapposxn--qqqt11misconfusedxn--qxa6axn--qxamunexus-3xn--rady-iraxn--rdal-poaxn--rde-ulazioxn--rdy-0nabaris-uberleetrentinos-tirolxn--rennesy-v1axn--rhkkervju-01afedorapeoplefrakkestadyndns-webhostingujogaszxn--rholt-mragowoltlab-democraciaxn--rhqv96gxn--rht27zxn--rht3dxn--rht61exn--risa-5naturalxn--risr-iraxn--rland-uuaxn--rlingen-mxaxn--rmskog-byawaraxn--rny31hakodatexn--rovu88bentleyusuitatamotorsitestinglitchernihivgubs3-website-us-west-1xn--rros-graphicsxn--rskog-uuaxn--rst-0naturbruksgymnxn--rsta-framercanvasxn--rvc1e0am3exn--ryken-vuaxn--ryrvik-byawatahamaxn--s-1faitheshopwarezzoxn--s9brj9cntraniandriabarlettatraniandriaxn--sandnessjen-ogbentrendhostingliwiceu-3xn--sandy-yuaxn--sdtirol-n2axn--seral-lraxn--ses554gxn--sgne-graphoxn--4gbriminiserverxn--skierv-utazurestaticappspaceusercontentunkongsvingerxn--skjervy-v1axn--skjk-soaxn--sknit-yqaxn--sknland-fxaxn--slat-5navigationxn--slt-elabogadobeaemcloud-fr1xn--smla-hraxn--smna-gratangenxn--snase-nraxn--sndre-land-0cbeppublishproxyuufcfanirasakindependent-panelomonza-brianzaporizhzhedmarkarelianceu-4xn--snes-poaxn--snsa-roaxn--sr-aurdal-l8axn--sr-fron-q1axn--sr-odal-q1axn--sr-varanger-ggbeskidyn-ip24xn--srfold-byaxn--srreisa-q1axn--srum-gratis-a-bloggerxn--stfold-9xaxn--stjrdal-s1axn--stjrdalshalsen-sqbestbuyshoparenagasakikuchikuseihicampinashikiminohostfoldnavyuzawaxn--stre-toten-zcbetainaboxfuselfipartindependent-reviewegroweibolognagasukeu-north-1xn--t60b56axn--tckweddingxn--tiq49xqyjelenia-goraxn--tjme-hraxn--tn0agrocerydxn--tnsberg-q1axn--tor131oxn--trany-yuaxn--trentin-sd-tirol-rzbhzc66xn--trentin-sdtirol-7vbialystokkeymachineu-south-1xn--trentino-sd-tirol-c3bielawakuyachimataharanzanishiazaindielddanuorrindigenamerikawauevje-og-hornnes3-website-us-west-2xn--trentino-sdtirol-szbiella-speziaxn--trentinosd-tirol-rzbieszczadygeyachiyodaeguamfamscompute-1xn--trentinosdtirol-7vbievat-band-campaignieznoorstaplesakyotanabellunordeste-idclkarlsoyxn--trentinsd-tirol-6vbifukagawalbrzycharitydalomzaporizhzhiaxn--trentinsdtirol-nsbigv-infolkebiblegnicalvinklein-butterhcloudiscoursesalangenishigotpantheonsitexn--trgstad-r1axn--trna-woaxn--troms-zuaxn--tysvr-vraxn--uc0atventuresinstagingxn--uc0ay4axn--uist22hakonexn--uisz3gxn--unjrga-rtashkenturindalxn--unup4yxn--uuwu58axn--vads-jraxn--valle-aoste-ebbturystykaneyamazoexn--valle-d-aoste-ehboehringerikexn--valleaoste-e7axn--valledaoste-ebbvadsoccertmgreaterxn--vard-jraxn--vegrshei-c0axn--vermgensberater-ctb-hostingxn--vermgensberatung-pwbiharstadotsubetsugarulezajskiervaksdalondonetskarmoyxn--vestvgy-ixa6oxn--vg-yiabruzzombieidskogasawarackmazerbaijan-mayenbaidarmeniaxn--vgan-qoaxn--vgsy-qoa0jellybeanxn--vgu402coguchikuzenishiwakinvestmentsaveincloudyndns-at-workisboringsakershusrcfdyndns-blogsitexn--vhquvestfoldxn--vler-qoaxn--vre-eiker-k8axn--vrggt-xqadxn--vry-yla5gxn--vuq861bihoronobeokagakikugawalesundiscoverdalondrinaplesknsalon-1xn--w4r85el8fhu5dnraxn--w4rs40lxn--wcvs22dxn--wgbh1communexn--wgbl6axn--xhq521bikedaejeonbuk0xn--xkc2al3hye2axn--xkc2dl3a5ee0hakubackyardshiraois-a-greenxn--y9a3aquarelleasingxn--yer-znavois-very-badxn--yfro4i67oxn--ygarden-p1axn--ygbi2ammxn--4it168dxn--ystre-slidre-ujbiofficialorenskoglobodoes-itcouldbeworldishangrilamdongnairkitapps-audibleasecuritytacticsxn--0trq7p7nnishiharaxn--zbx025dxn--zf0ao64axn--zf0avxlxn--zfr164bipartsaloonishiizunazukindustriaxnbayernxz \ No newline at end of file diff --git a/publicsuffix/example_test.go b/publicsuffix/example_test.go index 3f44dcfe75..c051dac8e0 100644 --- a/publicsuffix/example_test.go +++ b/publicsuffix/example_test.go @@ -77,7 +77,7 @@ func ExamplePublicSuffix_manager() { // > golang.dev dev is ICANN Managed // > golang.net net is ICANN Managed // > play.golang.org org is ICANN Managed - // > gophers.in.space.museum space.museum is ICANN Managed + // > gophers.in.space.museum museum is ICANN Managed // > // > 0emm.com com is ICANN Managed // > a.0emm.com a.0emm.com is Privately Managed diff --git a/publicsuffix/table.go b/publicsuffix/table.go index 6bdadcc448..78d400fa65 100644 --- a/publicsuffix/table.go +++ b/publicsuffix/table.go @@ -4,7 +4,7 @@ package publicsuffix import _ "embed" -const version = "publicsuffix.org's public_suffix_list.dat, git revision e248cbc92a527a166454afe9914c4c1b4253893f (2022-11-15T18:02:38Z)" +const version = "publicsuffix.org's public_suffix_list.dat, git revision 63cbc63d470d7b52c35266aa96c4c98c96ec499c (2023-08-03T10:01:25Z)" const ( nodesBits = 40 @@ -26,7 +26,7 @@ const ( ) // numTLD is the number of top level domains. -const numTLD = 1494 +const numTLD = 1474 // text is the combined text of all labels. // @@ -63,8 +63,8 @@ var nodes uint40String //go:embed data/children var children uint32String -// max children 718 (capacity 1023) -// max text offset 32976 (capacity 65535) -// max text length 36 (capacity 63) -// max hi 9656 (capacity 16383) -// max lo 9651 (capacity 16383) +// max children 743 (capacity 1023) +// max text offset 30876 (capacity 65535) +// max text length 31 (capacity 63) +// max hi 9322 (capacity 16383) +// max lo 9317 (capacity 16383) diff --git a/publicsuffix/table_test.go b/publicsuffix/table_test.go index 99698271a9..a297b3b0dd 100644 --- a/publicsuffix/table_test.go +++ b/publicsuffix/table_test.go @@ -2,7 +2,7 @@ package publicsuffix -const numICANNRules = 7367 +const numICANNRules = 6893 var rules = [...]string{ "ac", @@ -302,9 +302,26 @@ var rules = [...]string{ "org.bi", "biz", "bj", - "asso.bj", - "barreau.bj", - "gouv.bj", + "africa.bj", + "agro.bj", + "architectes.bj", + "assur.bj", + "avocats.bj", + "co.bj", + "com.bj", + "eco.bj", + "econo.bj", + "edu.bj", + "info.bj", + "loisirs.bj", + "money.bj", + "net.bj", + "org.bj", + "ote.bj", + "resto.bj", + "restaurant.bj", + "tourism.bj", + "univ.bj", "bm", "com.bm", "edu.bm", @@ -3596,552 +3613,6 @@ var rules = [...]string{ "co.mu", "or.mu", "museum", - "academy.museum", - "agriculture.museum", - "air.museum", - "airguard.museum", - "alabama.museum", - "alaska.museum", - "amber.museum", - "ambulance.museum", - "american.museum", - "americana.museum", - "americanantiques.museum", - "americanart.museum", - "amsterdam.museum", - "and.museum", - "annefrank.museum", - "anthro.museum", - "anthropology.museum", - "antiques.museum", - "aquarium.museum", - "arboretum.museum", - "archaeological.museum", - "archaeology.museum", - "architecture.museum", - "art.museum", - "artanddesign.museum", - "artcenter.museum", - "artdeco.museum", - "arteducation.museum", - "artgallery.museum", - "arts.museum", - "artsandcrafts.museum", - "asmatart.museum", - "assassination.museum", - "assisi.museum", - "association.museum", - "astronomy.museum", - "atlanta.museum", - "austin.museum", - "australia.museum", - "automotive.museum", - "aviation.museum", - "axis.museum", - "badajoz.museum", - "baghdad.museum", - "bahn.museum", - "bale.museum", - "baltimore.museum", - "barcelona.museum", - "baseball.museum", - "basel.museum", - "baths.museum", - "bauern.museum", - "beauxarts.museum", - "beeldengeluid.museum", - "bellevue.museum", - "bergbau.museum", - "berkeley.museum", - "berlin.museum", - "bern.museum", - "bible.museum", - "bilbao.museum", - "bill.museum", - "birdart.museum", - "birthplace.museum", - "bonn.museum", - "boston.museum", - "botanical.museum", - "botanicalgarden.museum", - "botanicgarden.museum", - "botany.museum", - "brandywinevalley.museum", - "brasil.museum", - "bristol.museum", - "british.museum", - "britishcolumbia.museum", - "broadcast.museum", - "brunel.museum", - "brussel.museum", - "brussels.museum", - "bruxelles.museum", - "building.museum", - "burghof.museum", - "bus.museum", - "bushey.museum", - "cadaques.museum", - "california.museum", - "cambridge.museum", - "can.museum", - "canada.museum", - "capebreton.museum", - "carrier.museum", - "cartoonart.museum", - "casadelamoneda.museum", - "castle.museum", - "castres.museum", - "celtic.museum", - "center.museum", - "chattanooga.museum", - "cheltenham.museum", - "chesapeakebay.museum", - "chicago.museum", - "children.museum", - "childrens.museum", - "childrensgarden.museum", - "chiropractic.museum", - "chocolate.museum", - "christiansburg.museum", - "cincinnati.museum", - "cinema.museum", - "circus.museum", - "civilisation.museum", - "civilization.museum", - "civilwar.museum", - "clinton.museum", - "clock.museum", - "coal.museum", - "coastaldefence.museum", - "cody.museum", - "coldwar.museum", - "collection.museum", - "colonialwilliamsburg.museum", - "coloradoplateau.museum", - "columbia.museum", - "columbus.museum", - "communication.museum", - "communications.museum", - "community.museum", - "computer.museum", - "computerhistory.museum", - "xn--comunicaes-v6a2o.museum", - "contemporary.museum", - "contemporaryart.museum", - "convent.museum", - "copenhagen.museum", - "corporation.museum", - "xn--correios-e-telecomunicaes-ghc29a.museum", - "corvette.museum", - "costume.museum", - "countryestate.museum", - "county.museum", - "crafts.museum", - "cranbrook.museum", - "creation.museum", - "cultural.museum", - "culturalcenter.museum", - "culture.museum", - "cyber.museum", - "cymru.museum", - "dali.museum", - "dallas.museum", - "database.museum", - "ddr.museum", - "decorativearts.museum", - "delaware.museum", - "delmenhorst.museum", - "denmark.museum", - "depot.museum", - "design.museum", - "detroit.museum", - "dinosaur.museum", - "discovery.museum", - "dolls.museum", - "donostia.museum", - "durham.museum", - "eastafrica.museum", - "eastcoast.museum", - "education.museum", - "educational.museum", - "egyptian.museum", - "eisenbahn.museum", - "elburg.museum", - "elvendrell.museum", - "embroidery.museum", - "encyclopedic.museum", - "england.museum", - "entomology.museum", - "environment.museum", - "environmentalconservation.museum", - "epilepsy.museum", - "essex.museum", - "estate.museum", - "ethnology.museum", - "exeter.museum", - "exhibition.museum", - "family.museum", - "farm.museum", - "farmequipment.museum", - "farmers.museum", - "farmstead.museum", - "field.museum", - "figueres.museum", - "filatelia.museum", - "film.museum", - "fineart.museum", - "finearts.museum", - "finland.museum", - "flanders.museum", - "florida.museum", - "force.museum", - "fortmissoula.museum", - "fortworth.museum", - "foundation.museum", - "francaise.museum", - "frankfurt.museum", - "franziskaner.museum", - "freemasonry.museum", - "freiburg.museum", - "fribourg.museum", - "frog.museum", - "fundacio.museum", - "furniture.museum", - "gallery.museum", - "garden.museum", - "gateway.museum", - "geelvinck.museum", - "gemological.museum", - "geology.museum", - "georgia.museum", - "giessen.museum", - "glas.museum", - "glass.museum", - "gorge.museum", - "grandrapids.museum", - "graz.museum", - "guernsey.museum", - "halloffame.museum", - "hamburg.museum", - "handson.museum", - "harvestcelebration.museum", - "hawaii.museum", - "health.museum", - "heimatunduhren.museum", - "hellas.museum", - "helsinki.museum", - "hembygdsforbund.museum", - "heritage.museum", - "histoire.museum", - "historical.museum", - "historicalsociety.museum", - "historichouses.museum", - "historisch.museum", - "historisches.museum", - "history.museum", - "historyofscience.museum", - "horology.museum", - "house.museum", - "humanities.museum", - "illustration.museum", - "imageandsound.museum", - "indian.museum", - "indiana.museum", - "indianapolis.museum", - "indianmarket.museum", - "intelligence.museum", - "interactive.museum", - "iraq.museum", - "iron.museum", - "isleofman.museum", - "jamison.museum", - "jefferson.museum", - "jerusalem.museum", - "jewelry.museum", - "jewish.museum", - "jewishart.museum", - "jfk.museum", - "journalism.museum", - "judaica.museum", - "judygarland.museum", - "juedisches.museum", - "juif.museum", - "karate.museum", - "karikatur.museum", - "kids.museum", - "koebenhavn.museum", - "koeln.museum", - "kunst.museum", - "kunstsammlung.museum", - "kunstunddesign.museum", - "labor.museum", - "labour.museum", - "lajolla.museum", - "lancashire.museum", - "landes.museum", - "lans.museum", - "xn--lns-qla.museum", - "larsson.museum", - "lewismiller.museum", - "lincoln.museum", - "linz.museum", - "living.museum", - "livinghistory.museum", - "localhistory.museum", - "london.museum", - "losangeles.museum", - "louvre.museum", - "loyalist.museum", - "lucerne.museum", - "luxembourg.museum", - "luzern.museum", - "mad.museum", - "madrid.museum", - "mallorca.museum", - "manchester.museum", - "mansion.museum", - "mansions.museum", - "manx.museum", - "marburg.museum", - "maritime.museum", - "maritimo.museum", - "maryland.museum", - "marylhurst.museum", - "media.museum", - "medical.museum", - "medizinhistorisches.museum", - "meeres.museum", - "memorial.museum", - "mesaverde.museum", - "michigan.museum", - "midatlantic.museum", - "military.museum", - "mill.museum", - "miners.museum", - "mining.museum", - "minnesota.museum", - "missile.museum", - "missoula.museum", - "modern.museum", - "moma.museum", - "money.museum", - "monmouth.museum", - "monticello.museum", - "montreal.museum", - "moscow.museum", - "motorcycle.museum", - "muenchen.museum", - "muenster.museum", - "mulhouse.museum", - "muncie.museum", - "museet.museum", - "museumcenter.museum", - "museumvereniging.museum", - "music.museum", - "national.museum", - "nationalfirearms.museum", - "nationalheritage.museum", - "nativeamerican.museum", - "naturalhistory.museum", - "naturalhistorymuseum.museum", - "naturalsciences.museum", - "nature.museum", - "naturhistorisches.museum", - "natuurwetenschappen.museum", - "naumburg.museum", - "naval.museum", - "nebraska.museum", - "neues.museum", - "newhampshire.museum", - "newjersey.museum", - "newmexico.museum", - "newport.museum", - "newspaper.museum", - "newyork.museum", - "niepce.museum", - "norfolk.museum", - "north.museum", - "nrw.museum", - "nyc.museum", - "nyny.museum", - "oceanographic.museum", - "oceanographique.museum", - "omaha.museum", - "online.museum", - "ontario.museum", - "openair.museum", - "oregon.museum", - "oregontrail.museum", - "otago.museum", - "oxford.museum", - "pacific.museum", - "paderborn.museum", - "palace.museum", - "paleo.museum", - "palmsprings.museum", - "panama.museum", - "paris.museum", - "pasadena.museum", - "pharmacy.museum", - "philadelphia.museum", - "philadelphiaarea.museum", - "philately.museum", - "phoenix.museum", - "photography.museum", - "pilots.museum", - "pittsburgh.museum", - "planetarium.museum", - "plantation.museum", - "plants.museum", - "plaza.museum", - "portal.museum", - "portland.museum", - "portlligat.museum", - "posts-and-telecommunications.museum", - "preservation.museum", - "presidio.museum", - "press.museum", - "project.museum", - "public.museum", - "pubol.museum", - "quebec.museum", - "railroad.museum", - "railway.museum", - "research.museum", - "resistance.museum", - "riodejaneiro.museum", - "rochester.museum", - "rockart.museum", - "roma.museum", - "russia.museum", - "saintlouis.museum", - "salem.museum", - "salvadordali.museum", - "salzburg.museum", - "sandiego.museum", - "sanfrancisco.museum", - "santabarbara.museum", - "santacruz.museum", - "santafe.museum", - "saskatchewan.museum", - "satx.museum", - "savannahga.museum", - "schlesisches.museum", - "schoenbrunn.museum", - "schokoladen.museum", - "school.museum", - "schweiz.museum", - "science.museum", - "scienceandhistory.museum", - "scienceandindustry.museum", - "sciencecenter.museum", - "sciencecenters.museum", - "science-fiction.museum", - "sciencehistory.museum", - "sciences.museum", - "sciencesnaturelles.museum", - "scotland.museum", - "seaport.museum", - "settlement.museum", - "settlers.museum", - "shell.museum", - "sherbrooke.museum", - "sibenik.museum", - "silk.museum", - "ski.museum", - "skole.museum", - "society.museum", - "sologne.museum", - "soundandvision.museum", - "southcarolina.museum", - "southwest.museum", - "space.museum", - "spy.museum", - "square.museum", - "stadt.museum", - "stalbans.museum", - "starnberg.museum", - "state.museum", - "stateofdelaware.museum", - "station.museum", - "steam.museum", - "steiermark.museum", - "stjohn.museum", - "stockholm.museum", - "stpetersburg.museum", - "stuttgart.museum", - "suisse.museum", - "surgeonshall.museum", - "surrey.museum", - "svizzera.museum", - "sweden.museum", - "sydney.museum", - "tank.museum", - "tcm.museum", - "technology.museum", - "telekommunikation.museum", - "television.museum", - "texas.museum", - "textile.museum", - "theater.museum", - "time.museum", - "timekeeping.museum", - "topology.museum", - "torino.museum", - "touch.museum", - "town.museum", - "transport.museum", - "tree.museum", - "trolley.museum", - "trust.museum", - "trustee.museum", - "uhren.museum", - "ulm.museum", - "undersea.museum", - "university.museum", - "usa.museum", - "usantiques.museum", - "usarts.museum", - "uscountryestate.museum", - "usculture.museum", - "usdecorativearts.museum", - "usgarden.museum", - "ushistory.museum", - "ushuaia.museum", - "uslivinghistory.museum", - "utah.museum", - "uvic.museum", - "valley.museum", - "vantaa.museum", - "versailles.museum", - "viking.museum", - "village.museum", - "virginia.museum", - "virtual.museum", - "virtuel.museum", - "vlaanderen.museum", - "volkenkunde.museum", - "wales.museum", - "wallonie.museum", - "war.museum", - "washingtondc.museum", - "watchandclock.museum", - "watch-and-clock.museum", - "western.museum", - "westfalen.museum", - "whaling.museum", - "wildlife.museum", - "williamsburg.museum", - "windmill.museum", - "workshop.museum", - "york.museum", - "yorkshire.museum", - "yosemite.museum", - "youth.museum", - "zoological.museum", - "zoology.museum", - "xn--9dbhblg6di.museum", - "xn--h1aegh.museum", "mv", "aero.mv", "biz.mv", @@ -5133,52 +4604,60 @@ var rules = [...]string{ "turystyka.pl", "gov.pl", "ap.gov.pl", + "griw.gov.pl", "ic.gov.pl", "is.gov.pl", - "us.gov.pl", "kmpsp.gov.pl", + "konsulat.gov.pl", "kppsp.gov.pl", - "kwpsp.gov.pl", - "psp.gov.pl", - "wskr.gov.pl", "kwp.gov.pl", + "kwpsp.gov.pl", + "mup.gov.pl", "mw.gov.pl", - "ug.gov.pl", - "um.gov.pl", - "umig.gov.pl", - "ugim.gov.pl", - "upow.gov.pl", - "uw.gov.pl", - "starostwo.gov.pl", + "oia.gov.pl", + "oirm.gov.pl", + "oke.gov.pl", + "oow.gov.pl", + "oschr.gov.pl", + "oum.gov.pl", "pa.gov.pl", + "pinb.gov.pl", + "piw.gov.pl", "po.gov.pl", + "pr.gov.pl", + "psp.gov.pl", "psse.gov.pl", "pup.gov.pl", "rzgw.gov.pl", "sa.gov.pl", + "sdn.gov.pl", + "sko.gov.pl", "so.gov.pl", "sr.gov.pl", - "wsa.gov.pl", - "sko.gov.pl", + "starostwo.gov.pl", + "ug.gov.pl", + "ugim.gov.pl", + "um.gov.pl", + "umig.gov.pl", + "upow.gov.pl", + "uppo.gov.pl", + "us.gov.pl", + "uw.gov.pl", "uzs.gov.pl", + "wif.gov.pl", "wiih.gov.pl", "winb.gov.pl", - "pinb.gov.pl", "wios.gov.pl", "witd.gov.pl", - "wzmiuw.gov.pl", - "piw.gov.pl", "wiw.gov.pl", - "griw.gov.pl", - "wif.gov.pl", - "oum.gov.pl", - "sdn.gov.pl", - "zp.gov.pl", - "uppo.gov.pl", - "mup.gov.pl", + "wkz.gov.pl", + "wsa.gov.pl", + "wskr.gov.pl", + "wsse.gov.pl", "wuoz.gov.pl", - "konsulat.gov.pl", - "oirm.gov.pl", + "wzmiuw.gov.pl", + "zp.gov.pl", + "zpisdn.gov.pl", "augustow.pl", "babia-gora.pl", "bedzin.pl", @@ -5722,6 +5201,7 @@ var rules = [...]string{ "kirovograd.ua", "km.ua", "kr.ua", + "kropyvnytskyi.ua", "krym.ua", "ks.ua", "kv.ua", @@ -6063,18 +5543,84 @@ var rules = [...]string{ "net.vi", "org.vi", "vn", + "ac.vn", + "ai.vn", + "biz.vn", "com.vn", - "net.vn", - "org.vn", "edu.vn", "gov.vn", - "int.vn", - "ac.vn", - "biz.vn", + "health.vn", + "id.vn", "info.vn", + "int.vn", + "io.vn", "name.vn", + "net.vn", + "org.vn", "pro.vn", - "health.vn", + "angiang.vn", + "bacgiang.vn", + "backan.vn", + "baclieu.vn", + "bacninh.vn", + "baria-vungtau.vn", + "bentre.vn", + "binhdinh.vn", + "binhduong.vn", + "binhphuoc.vn", + "binhthuan.vn", + "camau.vn", + "cantho.vn", + "caobang.vn", + "daklak.vn", + "daknong.vn", + "danang.vn", + "dienbien.vn", + "dongnai.vn", + "dongthap.vn", + "gialai.vn", + "hagiang.vn", + "haiduong.vn", + "haiphong.vn", + "hanam.vn", + "hanoi.vn", + "hatinh.vn", + "haugiang.vn", + "hoabinh.vn", + "hungyen.vn", + "khanhhoa.vn", + "kiengiang.vn", + "kontum.vn", + "laichau.vn", + "lamdong.vn", + "langson.vn", + "laocai.vn", + "longan.vn", + "namdinh.vn", + "nghean.vn", + "ninhbinh.vn", + "ninhthuan.vn", + "phutho.vn", + "phuyen.vn", + "quangbinh.vn", + "quangnam.vn", + "quangngai.vn", + "quangninh.vn", + "quangtri.vn", + "soctrang.vn", + "sonla.vn", + "tayninh.vn", + "thaibinh.vn", + "thainguyen.vn", + "thanhhoa.vn", + "thanhphohochiminh.vn", + "thuathienhue.vn", + "tiengiang.vn", + "travinh.vn", + "tuyenquang.vn", + "vinhlong.vn", + "vinhphuc.vn", + "yenbai.vn", "vu", "com.vu", "edu.vu", @@ -6221,7 +5767,6 @@ var rules = [...]string{ "org.zw", "aaa", "aarp", - "abarth", "abb", "abbott", "abbvie", @@ -6235,7 +5780,6 @@ var rules = [...]string{ "accountants", "aco", "actor", - "adac", "ads", "adult", "aeg", @@ -6249,7 +5793,6 @@ var rules = [...]string{ "airforce", "airtel", "akdn", - "alfaromeo", "alibaba", "alipay", "allfinanz", @@ -6445,7 +5988,6 @@ var rules = [...]string{ "contact", "contractors", "cooking", - "cookingchannel", "cool", "corsica", "country", @@ -6554,7 +6096,6 @@ var rules = [...]string{ "feedback", "ferrari", "ferrero", - "fiat", "fidelity", "fido", "film", @@ -6576,7 +6117,6 @@ var rules = [...]string{ "fly", "foo", "food", - "foodnetwork", "football", "ford", "forex", @@ -6661,7 +6201,6 @@ var rules = [...]string{ "helsinki", "here", "hermes", - "hgtv", "hiphop", "hisamitsu", "hitachi", @@ -6680,7 +6219,6 @@ var rules = [...]string{ "host", "hosting", "hot", - "hoteles", "hotels", "hotmail", "house", @@ -6761,7 +6299,6 @@ var rules = [...]string{ "lamborghini", "lamer", "lancaster", - "lancia", "land", "landrover", "lanxess", @@ -6789,7 +6326,6 @@ var rules = [...]string{ "limited", "limo", "lincoln", - "linde", "link", "lipsy", "live", @@ -6800,7 +6336,6 @@ var rules = [...]string{ "loans", "locker", "locus", - "loft", "lol", "london", "lotte", @@ -6813,7 +6348,6 @@ var rules = [...]string{ "lundbeck", "luxe", "luxury", - "macys", "madrid", "maif", "maison", @@ -6827,7 +6361,6 @@ var rules = [...]string{ "markets", "marriott", "marshalls", - "maserati", "mattel", "mba", "mckinsey", @@ -6868,7 +6401,6 @@ var rules = [...]string{ "mtn", "mtr", "music", - "mutual", "nab", "nagoya", "natura", @@ -6933,7 +6465,6 @@ var rules = [...]string{ "partners", "parts", "party", - "passagens", "pay", "pccw", "pet", @@ -7063,7 +6594,6 @@ var rules = [...]string{ "select", "sener", "services", - "ses", "seven", "sew", "sex", @@ -7157,7 +6687,6 @@ var rules = [...]string{ "tiaa", "tickets", "tienda", - "tiffany", "tips", "tires", "tirol", @@ -7180,7 +6709,6 @@ var rules = [...]string{ "trading", "training", "travel", - "travelchannel", "travelers", "travelersinsurance", "trust", @@ -7225,7 +6753,6 @@ var rules = [...]string{ "voting", "voto", "voyage", - "vuelos", "wales", "walmart", "walter", @@ -7316,7 +6843,6 @@ var rules = [...]string{ "xn--io0a7i", "xn--j1aef", "xn--jlq480n2rg", - "xn--jlq61u9w7b", "xn--jvr189m", "xn--kcrx77d1x4a", "xn--kput3i", @@ -7379,17 +6905,35 @@ var rules = [...]string{ "graphox.us", "*.devcdnaccesso.com", "*.on-acorn.io", + "activetrail.biz", "adobeaemcloud.com", "*.dev.adobeaemcloud.com", "hlx.live", "adobeaemcloud.net", "hlx.page", "hlx3.page", + "adobeio-static.net", + "adobeioruntime.net", "beep.pl", "airkitapps.com", "airkitapps-au.com", "airkitapps.eu", "aivencloud.com", + "akadns.net", + "akamai.net", + "akamai-staging.net", + "akamaiedge.net", + "akamaiedge-staging.net", + "akamaihd.net", + "akamaihd-staging.net", + "akamaiorigin.net", + "akamaiorigin-staging.net", + "akamaized.net", + "akamaized-staging.net", + "edgekey.net", + "edgekey-staging.net", + "edgesuite.net", + "edgesuite-staging.net", "barsy.ca", "*.compute.estate", "*.alces.network", @@ -7456,46 +7000,72 @@ var rules = [...]string{ "s3.dualstack.us-east-2.amazonaws.com", "s3.us-east-2.amazonaws.com", "s3-website.us-east-2.amazonaws.com", + "analytics-gateway.ap-northeast-1.amazonaws.com", + "analytics-gateway.eu-west-1.amazonaws.com", + "analytics-gateway.us-east-1.amazonaws.com", + "analytics-gateway.us-east-2.amazonaws.com", + "analytics-gateway.us-west-2.amazonaws.com", + "webview-assets.aws-cloud9.af-south-1.amazonaws.com", "vfs.cloud9.af-south-1.amazonaws.com", "webview-assets.cloud9.af-south-1.amazonaws.com", + "webview-assets.aws-cloud9.ap-east-1.amazonaws.com", "vfs.cloud9.ap-east-1.amazonaws.com", "webview-assets.cloud9.ap-east-1.amazonaws.com", + "webview-assets.aws-cloud9.ap-northeast-1.amazonaws.com", "vfs.cloud9.ap-northeast-1.amazonaws.com", "webview-assets.cloud9.ap-northeast-1.amazonaws.com", + "webview-assets.aws-cloud9.ap-northeast-2.amazonaws.com", "vfs.cloud9.ap-northeast-2.amazonaws.com", "webview-assets.cloud9.ap-northeast-2.amazonaws.com", + "webview-assets.aws-cloud9.ap-northeast-3.amazonaws.com", "vfs.cloud9.ap-northeast-3.amazonaws.com", "webview-assets.cloud9.ap-northeast-3.amazonaws.com", + "webview-assets.aws-cloud9.ap-south-1.amazonaws.com", "vfs.cloud9.ap-south-1.amazonaws.com", "webview-assets.cloud9.ap-south-1.amazonaws.com", + "webview-assets.aws-cloud9.ap-southeast-1.amazonaws.com", "vfs.cloud9.ap-southeast-1.amazonaws.com", "webview-assets.cloud9.ap-southeast-1.amazonaws.com", + "webview-assets.aws-cloud9.ap-southeast-2.amazonaws.com", "vfs.cloud9.ap-southeast-2.amazonaws.com", "webview-assets.cloud9.ap-southeast-2.amazonaws.com", + "webview-assets.aws-cloud9.ca-central-1.amazonaws.com", "vfs.cloud9.ca-central-1.amazonaws.com", "webview-assets.cloud9.ca-central-1.amazonaws.com", + "webview-assets.aws-cloud9.eu-central-1.amazonaws.com", "vfs.cloud9.eu-central-1.amazonaws.com", "webview-assets.cloud9.eu-central-1.amazonaws.com", + "webview-assets.aws-cloud9.eu-north-1.amazonaws.com", "vfs.cloud9.eu-north-1.amazonaws.com", "webview-assets.cloud9.eu-north-1.amazonaws.com", + "webview-assets.aws-cloud9.eu-south-1.amazonaws.com", "vfs.cloud9.eu-south-1.amazonaws.com", "webview-assets.cloud9.eu-south-1.amazonaws.com", + "webview-assets.aws-cloud9.eu-west-1.amazonaws.com", "vfs.cloud9.eu-west-1.amazonaws.com", "webview-assets.cloud9.eu-west-1.amazonaws.com", + "webview-assets.aws-cloud9.eu-west-2.amazonaws.com", "vfs.cloud9.eu-west-2.amazonaws.com", "webview-assets.cloud9.eu-west-2.amazonaws.com", + "webview-assets.aws-cloud9.eu-west-3.amazonaws.com", "vfs.cloud9.eu-west-3.amazonaws.com", "webview-assets.cloud9.eu-west-3.amazonaws.com", + "webview-assets.aws-cloud9.me-south-1.amazonaws.com", "vfs.cloud9.me-south-1.amazonaws.com", "webview-assets.cloud9.me-south-1.amazonaws.com", + "webview-assets.aws-cloud9.sa-east-1.amazonaws.com", "vfs.cloud9.sa-east-1.amazonaws.com", "webview-assets.cloud9.sa-east-1.amazonaws.com", + "webview-assets.aws-cloud9.us-east-1.amazonaws.com", "vfs.cloud9.us-east-1.amazonaws.com", "webview-assets.cloud9.us-east-1.amazonaws.com", + "webview-assets.aws-cloud9.us-east-2.amazonaws.com", "vfs.cloud9.us-east-2.amazonaws.com", "webview-assets.cloud9.us-east-2.amazonaws.com", + "webview-assets.aws-cloud9.us-west-1.amazonaws.com", "vfs.cloud9.us-west-1.amazonaws.com", "webview-assets.cloud9.us-west-1.amazonaws.com", + "webview-assets.aws-cloud9.us-west-2.amazonaws.com", "vfs.cloud9.us-west-2.amazonaws.com", "webview-assets.cloud9.us-west-2.amazonaws.com", "cn-north-1.eb.amazonaws.com.cn", @@ -7542,6 +7112,7 @@ var rules = [...]string{ "myasustor.com", "cdn.prod.atlassian-dev.net", "translated.page", + "autocode.dev", "myfritz.net", "onavstack.net", "*.awdev.ca", @@ -7588,6 +7159,8 @@ var rules = [...]string{ "vm.bytemark.co.uk", "cafjs.com", "mycd.eu", + "canva-apps.cn", + "canva-apps.com", "drr.ac", "uwu.ai", "carrd.co", @@ -7653,8 +7226,11 @@ var rules = [...]string{ "cloudcontrolled.com", "cloudcontrolapp.com", "*.cloudera.site", - "pages.dev", + "cf-ipfs.com", + "cloudflare-ipfs.com", "trycloudflare.com", + "pages.dev", + "r2.dev", "workers.dev", "wnext.app", "co.ca", @@ -8227,6 +7803,7 @@ var rules = [...]string{ "channelsdvr.net", "u.channelsdvr.net", "edgecompute.app", + "fastly-edge.com", "fastly-terrarium.com", "fastlylb.net", "map.fastlylb.net", @@ -8566,6 +8143,7 @@ var rules = [...]string{ "ngo.ng", "edu.scot", "sch.so", + "ie.ua", "hostyhosting.io", "xn--hkkinen-5wa.fi", "*.moonscale.io", @@ -8633,7 +8211,6 @@ var rules = [...]string{ "iobb.net", "mel.cloudlets.com.au", "cloud.interhostsolutions.be", - "users.scale.virtualcloud.com.br", "mycloud.by", "alp1.ae.flow.ch", "appengine.flow.ch", @@ -8657,9 +8234,7 @@ var rules = [...]string{ "de.trendhosting.cloud", "jele.club", "amscompute.com", - "clicketcloud.com", "dopaas.com", - "hidora.com", "paas.hosted-by-previder.com", "rag-cloud.hosteur.com", "rag-cloud-ch.hosteur.com", @@ -8834,6 +8409,7 @@ var rules = [...]string{ "azurestaticapps.net", "1.azurestaticapps.net", "2.azurestaticapps.net", + "3.azurestaticapps.net", "centralus.azurestaticapps.net", "eastasia.azurestaticapps.net", "eastus2.azurestaticapps.net", @@ -8864,7 +8440,19 @@ var rules = [...]string{ "cloud.nospamproxy.com", "netlify.app", "4u.com", + "ngrok.app", + "ngrok-free.app", + "ngrok.dev", + "ngrok-free.dev", "ngrok.io", + "ap.ngrok.io", + "au.ngrok.io", + "eu.ngrok.io", + "in.ngrok.io", + "jp.ngrok.io", + "sa.ngrok.io", + "us.ngrok.io", + "ngrok.pizza", "nh-serv.co.uk", "nfshost.com", "*.developer.app", @@ -9084,6 +8672,7 @@ var rules = [...]string{ "eu.pythonanywhere.com", "qoto.io", "qualifioapp.com", + "ladesk.com", "qbuser.com", "cloudsite.builders", "instances.spawn.cc", @@ -9132,6 +8721,53 @@ var rules = [...]string{ "xn--h1aliz.xn--p1acf", "xn--90a1af.xn--p1acf", "xn--41a.xn--p1acf", + "180r.com", + "dojin.com", + "sakuratan.com", + "sakuraweb.com", + "x0.com", + "2-d.jp", + "bona.jp", + "crap.jp", + "daynight.jp", + "eek.jp", + "flop.jp", + "halfmoon.jp", + "jeez.jp", + "matrix.jp", + "mimoza.jp", + "ivory.ne.jp", + "mail-box.ne.jp", + "mints.ne.jp", + "mokuren.ne.jp", + "opal.ne.jp", + "sakura.ne.jp", + "sumomo.ne.jp", + "topaz.ne.jp", + "netgamers.jp", + "nyanta.jp", + "o0o0.jp", + "rdy.jp", + "rgr.jp", + "rulez.jp", + "s3.isk01.sakurastorage.jp", + "s3.isk02.sakurastorage.jp", + "saloon.jp", + "sblo.jp", + "skr.jp", + "tank.jp", + "uh-oh.jp", + "undo.jp", + "rs.webaccel.jp", + "user.webaccel.jp", + "websozai.jp", + "xii.jp", + "squares.net", + "jpn.org", + "kirara.st", + "x0.to", + "from.tv", + "sakura.tv", "*.builder.code.com", "*.dev-builder.code.com", "*.stg-builder.code.com", @@ -9204,6 +8840,9 @@ var rules = [...]string{ "beta.bounty-full.com", "small-web.org", "vp4.me", + "snowflake.app", + "privatelink.snowflake.app", + "streamlit.app", "streamlitapp.com", "try-snowplow.com", "srht.site", @@ -9243,6 +8882,7 @@ var rules = [...]string{ "myspreadshop.se", "myspreadshop.co.uk", "api.stdlib.com", + "storipress.app", "storj.farm", "utwente.io", "soc.srcf.net", @@ -9272,6 +8912,8 @@ var rules = [...]string{ "vpnplus.to", "direct.quickconnect.to", "tabitorder.co.il", + "mytabit.co.il", + "mytabit.com", "taifun-dns.de", "beta.tailscale.net", "ts.net", @@ -9350,6 +8992,7 @@ var rules = [...]string{ "hk.org", "ltd.hk", "inc.hk", + "it.com", "name.pm", "sch.tf", "biz.wf", @@ -9472,7 +9115,6 @@ var rules = [...]string{ var nodeLabels = [...]string{ "aaa", "aarp", - "abarth", "abb", "abbott", "abbvie", @@ -9488,7 +9130,6 @@ var nodeLabels = [...]string{ "aco", "actor", "ad", - "adac", "ads", "adult", "ae", @@ -9508,7 +9149,6 @@ var nodeLabels = [...]string{ "airtel", "akdn", "al", - "alfaromeo", "alibaba", "alipay", "allfinanz", @@ -9750,7 +9390,6 @@ var nodeLabels = [...]string{ "contact", "contractors", "cooking", - "cookingchannel", "cool", "coop", "corsica", @@ -9882,7 +9521,6 @@ var nodeLabels = [...]string{ "ferrari", "ferrero", "fi", - "fiat", "fidelity", "fido", "film", @@ -9908,7 +9546,6 @@ var nodeLabels = [...]string{ "fo", "foo", "food", - "foodnetwork", "football", "ford", "forex", @@ -10014,7 +9651,6 @@ var nodeLabels = [...]string{ "helsinki", "here", "hermes", - "hgtv", "hiphop", "hisamitsu", "hitachi", @@ -10036,7 +9672,6 @@ var nodeLabels = [...]string{ "host", "hosting", "hot", - "hoteles", "hotels", "hotmail", "house", @@ -10149,7 +9784,6 @@ var nodeLabels = [...]string{ "lamborghini", "lamer", "lancaster", - "lancia", "land", "landrover", "lanxess", @@ -10180,7 +9814,6 @@ var nodeLabels = [...]string{ "limited", "limo", "lincoln", - "linde", "link", "lipsy", "live", @@ -10192,7 +9825,6 @@ var nodeLabels = [...]string{ "loans", "locker", "locus", - "loft", "lol", "london", "lotte", @@ -10212,7 +9844,6 @@ var nodeLabels = [...]string{ "lv", "ly", "ma", - "macys", "madrid", "maif", "maison", @@ -10226,7 +9857,6 @@ var nodeLabels = [...]string{ "markets", "marriott", "marshalls", - "maserati", "mattel", "mba", "mc", @@ -10286,7 +9916,6 @@ var nodeLabels = [...]string{ "mu", "museum", "music", - "mutual", "mv", "mw", "mx", @@ -10374,7 +10003,6 @@ var nodeLabels = [...]string{ "partners", "parts", "party", - "passagens", "pay", "pccw", "pe", @@ -10530,7 +10158,6 @@ var nodeLabels = [...]string{ "select", "sener", "services", - "ses", "seven", "sew", "sex", @@ -10647,7 +10274,6 @@ var nodeLabels = [...]string{ "tiaa", "tickets", "tienda", - "tiffany", "tips", "tires", "tirol", @@ -10677,7 +10303,6 @@ var nodeLabels = [...]string{ "trading", "training", "travel", - "travelchannel", "travelers", "travelersinsurance", "trust", @@ -10739,7 +10364,6 @@ var nodeLabels = [...]string{ "voto", "voyage", "vu", - "vuelos", "wales", "walmart", "walter", @@ -10856,7 +10480,6 @@ var nodeLabels = [...]string{ "xn--j1amh", "xn--j6w193g", "xn--jlq480n2rg", - "xn--jlq61u9w7b", "xn--jvr189m", "xn--kcrx77d1x4a", "xn--kprw13d", @@ -11119,18 +10742,24 @@ var nodeLabels = [...]string{ "loginline", "messerli", "netlify", + "ngrok", + "ngrok-free", "noop", "northflank", "ondigitalocean", "onflashdrive", "platform0", "run", + "snowflake", + "storipress", + "streamlit", "telebit", "typedream", "vercel", "web", "wnext", "a", + "privatelink", "bet", "com", "coop", @@ -11316,6 +10945,7 @@ var nodeLabels = [...]string{ "edu", "or", "org", + "activetrail", "cloudns", "dscloud", "dyndns", @@ -11330,10 +10960,27 @@ var nodeLabels = [...]string{ "orx", "selfip", "webhop", - "asso", - "barreau", + "africa", + "agro", + "architectes", + "assur", + "avocats", "blogspot", - "gouv", + "co", + "com", + "eco", + "econo", + "edu", + "info", + "loisirs", + "money", + "net", + "org", + "ote", + "restaurant", + "resto", + "tourism", + "univ", "com", "edu", "gov", @@ -11529,9 +11176,6 @@ var nodeLabels = [...]string{ "zlg", "blogspot", "simplesite", - "virtualcloud", - "scale", - "users", "ac", "al", "am", @@ -11772,6 +11416,7 @@ var nodeLabels = [...]string{ "ac", "ah", "bj", + "canva-apps", "com", "cq", "edu", @@ -11853,6 +11498,7 @@ var nodeLabels = [...]string{ "owo", "001www", "0emm", + "180r", "1kapp", "3utilities", "4u", @@ -11888,11 +11534,13 @@ var nodeLabels = [...]string{ "br", "builtwithdark", "cafjs", + "canva-apps", "cechire", + "cf-ipfs", "ciscofreak", - "clicketcloud", "cloudcontrolapp", "cloudcontrolled", + "cloudflare-ipfs", "cn", "co", "code", @@ -11919,6 +11567,7 @@ var nodeLabels = [...]string{ "dnsdojo", "dnsiskinky", "doesntexist", + "dojin", "dontexist", "doomdns", "dopaas", @@ -11951,6 +11600,7 @@ var nodeLabels = [...]string{ "eu", "evennode", "familyds", + "fastly-edge", "fastly-terrarium", "fastvps-server", "fbsbx", @@ -12024,7 +11674,6 @@ var nodeLabels = [...]string{ "health-carereform", "herokuapp", "herokussl", - "hidora", "hk", "hobby-site", "homelinux", @@ -12098,6 +11747,7 @@ var nodeLabels = [...]string{ "isa-geek", "isa-hockeynut", "issmarterthanyou", + "it", "jdevcloud", "jelastic", "joyent", @@ -12107,6 +11757,7 @@ var nodeLabels = [...]string{ "kozow", "kr", "ktistory", + "ladesk", "likes-pie", "likescandy", "linode", @@ -12133,6 +11784,7 @@ var nodeLabels = [...]string{ "myshopblocks", "myshopify", "myspreadshop", + "mytabit", "mythic-beasts", "mytuleap", "myvnc", @@ -12179,6 +11831,8 @@ var nodeLabels = [...]string{ "rhcloud", "ru", "sa", + "sakuratan", + "sakuraweb", "saves-the-whales", "scrysec", "securitytactics", @@ -12241,6 +11895,7 @@ var nodeLabels = [...]string{ "wphostedmail", "wpmucdn", "writesthisblog", + "x0", "xnbay", "yolasite", "za", @@ -12295,107 +11950,154 @@ var nodeLabels = [...]string{ "us-east-2", "us-west-1", "us-west-2", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "analytics-gateway", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "analytics-gateway", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "analytics-gateway", + "aws-cloud9", "cloud9", "dualstack", + "webview-assets", "vfs", "webview-assets", "s3", + "analytics-gateway", + "aws-cloud9", "cloud9", "dualstack", "s3", "s3-website", + "webview-assets", "vfs", "webview-assets", "s3", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", + "analytics-gateway", + "aws-cloud9", "cloud9", + "webview-assets", "vfs", "webview-assets", "r", @@ -12610,6 +12312,7 @@ var nodeLabels = [...]string{ "pages", "customer", "bss", + "autocode", "curv", "deno", "deno-staging", @@ -12623,8 +12326,11 @@ var nodeLabels = [...]string{ "localcert", "loginline", "mediatech", + "ngrok", + "ngrok-free", "pages", "platter-app", + "r2", "shiftcrypto", "stg", "stgstage", @@ -13016,6 +12722,7 @@ var nodeLabels = [...]string{ "net", "org", "blogspot", + "mytabit", "ravpage", "tabitorder", "ac", @@ -13176,6 +12883,13 @@ var nodeLabels = [...]string{ "dyndns", "id", "apps", + "ap", + "au", + "eu", + "in", + "jp", + "sa", + "us", "stage", "mock", "sys", @@ -13649,6 +13363,7 @@ var nodeLabels = [...]string{ "net", "org", "sch", + "2-d", "ac", "ad", "aichi", @@ -13662,6 +13377,7 @@ var nodeLabels = [...]string{ "bitter", "blogspot", "blush", + "bona", "boo", "boy", "boyfriend", @@ -13682,18 +13398,22 @@ var nodeLabels = [...]string{ "cocotte", "coolblog", "cranky", + "crap", "cutegirl", "daa", + "daynight", "deca", "deci", "digick", "ed", + "eek", "egoism", "ehime", "fakefur", "fashionstore", "fem", "flier", + "flop", "floppy", "fool", "frenchkiss", @@ -13710,6 +13430,7 @@ var nodeLabels = [...]string{ "greater", "gunma", "hacca", + "halfmoon", "handcrafted", "heavy", "her", @@ -13725,6 +13446,7 @@ var nodeLabels = [...]string{ "ishikawa", "itigo", "iwate", + "jeez", "jellybean", "kagawa", "kagoshima", @@ -13748,7 +13470,9 @@ var nodeLabels = [...]string{ "lovepop", "lovesick", "main", + "matrix", "mie", + "mimoza", "miyagi", "miyazaki", "mods", @@ -13761,10 +13485,13 @@ var nodeLabels = [...]string{ "namaste", "nara", "ne", + "netgamers", "niigata", "nikita", "nobushi", "noor", + "nyanta", + "o0o0", "oita", "okayama", "okinawa", @@ -13785,22 +13512,30 @@ var nodeLabels = [...]string{ "pussycat", "pya", "raindrop", + "rdy", "readymade", + "rgr", + "rulez", "sadist", "saga", "saitama", + "sakurastorage", + "saloon", "sapporo", + "sblo", "schoolbus", "secret", "sendai", "shiga", "shimane", "shizuoka", + "skr", "staba", "stripper", "sub", "sunnyday", "supersale", + "tank", "theshop", "thick", "tochigi", @@ -13809,7 +13544,9 @@ var nodeLabels = [...]string{ "tonkotsu", "tottori", "toyama", + "uh-oh", "under", + "undo", "upper", "usercontent", "velvet", @@ -13818,8 +13555,11 @@ var nodeLabels = [...]string{ "vivian", "wakayama", "watson", + "webaccel", "weblike", + "websozai", "whitesnow", + "xii", "xn--0trq7p7nn", "xn--1ctwo", "xn--1lqs03n", @@ -14954,6 +14694,14 @@ var nodeLabels = [...]string{ "yoshino", "aseinet", "gehirn", + "ivory", + "mail-box", + "mints", + "mokuren", + "opal", + "sakura", + "sumomo", + "topaz", "user", "aga", "agano", @@ -15221,6 +14969,10 @@ var nodeLabels = [...]string{ "yoshida", "yoshikawa", "yoshimi", + "isk01", + "isk02", + "s3", + "s3", "city", "city", "aisho", @@ -15476,6 +15228,8 @@ var nodeLabels = [...]string{ "wakayama", "yuasa", "yura", + "rs", + "user", "asahi", "funagata", "higashine", @@ -15865,552 +15619,6 @@ var nodeLabels = [...]string{ "net", "or", "org", - "academy", - "agriculture", - "air", - "airguard", - "alabama", - "alaska", - "amber", - "ambulance", - "american", - "americana", - "americanantiques", - "americanart", - "amsterdam", - "and", - "annefrank", - "anthro", - "anthropology", - "antiques", - "aquarium", - "arboretum", - "archaeological", - "archaeology", - "architecture", - "art", - "artanddesign", - "artcenter", - "artdeco", - "arteducation", - "artgallery", - "arts", - "artsandcrafts", - "asmatart", - "assassination", - "assisi", - "association", - "astronomy", - "atlanta", - "austin", - "australia", - "automotive", - "aviation", - "axis", - "badajoz", - "baghdad", - "bahn", - "bale", - "baltimore", - "barcelona", - "baseball", - "basel", - "baths", - "bauern", - "beauxarts", - "beeldengeluid", - "bellevue", - "bergbau", - "berkeley", - "berlin", - "bern", - "bible", - "bilbao", - "bill", - "birdart", - "birthplace", - "bonn", - "boston", - "botanical", - "botanicalgarden", - "botanicgarden", - "botany", - "brandywinevalley", - "brasil", - "bristol", - "british", - "britishcolumbia", - "broadcast", - "brunel", - "brussel", - "brussels", - "bruxelles", - "building", - "burghof", - "bus", - "bushey", - "cadaques", - "california", - "cambridge", - "can", - "canada", - "capebreton", - "carrier", - "cartoonart", - "casadelamoneda", - "castle", - "castres", - "celtic", - "center", - "chattanooga", - "cheltenham", - "chesapeakebay", - "chicago", - "children", - "childrens", - "childrensgarden", - "chiropractic", - "chocolate", - "christiansburg", - "cincinnati", - "cinema", - "circus", - "civilisation", - "civilization", - "civilwar", - "clinton", - "clock", - "coal", - "coastaldefence", - "cody", - "coldwar", - "collection", - "colonialwilliamsburg", - "coloradoplateau", - "columbia", - "columbus", - "communication", - "communications", - "community", - "computer", - "computerhistory", - "contemporary", - "contemporaryart", - "convent", - "copenhagen", - "corporation", - "corvette", - "costume", - "countryestate", - "county", - "crafts", - "cranbrook", - "creation", - "cultural", - "culturalcenter", - "culture", - "cyber", - "cymru", - "dali", - "dallas", - "database", - "ddr", - "decorativearts", - "delaware", - "delmenhorst", - "denmark", - "depot", - "design", - "detroit", - "dinosaur", - "discovery", - "dolls", - "donostia", - "durham", - "eastafrica", - "eastcoast", - "education", - "educational", - "egyptian", - "eisenbahn", - "elburg", - "elvendrell", - "embroidery", - "encyclopedic", - "england", - "entomology", - "environment", - "environmentalconservation", - "epilepsy", - "essex", - "estate", - "ethnology", - "exeter", - "exhibition", - "family", - "farm", - "farmequipment", - "farmers", - "farmstead", - "field", - "figueres", - "filatelia", - "film", - "fineart", - "finearts", - "finland", - "flanders", - "florida", - "force", - "fortmissoula", - "fortworth", - "foundation", - "francaise", - "frankfurt", - "franziskaner", - "freemasonry", - "freiburg", - "fribourg", - "frog", - "fundacio", - "furniture", - "gallery", - "garden", - "gateway", - "geelvinck", - "gemological", - "geology", - "georgia", - "giessen", - "glas", - "glass", - "gorge", - "grandrapids", - "graz", - "guernsey", - "halloffame", - "hamburg", - "handson", - "harvestcelebration", - "hawaii", - "health", - "heimatunduhren", - "hellas", - "helsinki", - "hembygdsforbund", - "heritage", - "histoire", - "historical", - "historicalsociety", - "historichouses", - "historisch", - "historisches", - "history", - "historyofscience", - "horology", - "house", - "humanities", - "illustration", - "imageandsound", - "indian", - "indiana", - "indianapolis", - "indianmarket", - "intelligence", - "interactive", - "iraq", - "iron", - "isleofman", - "jamison", - "jefferson", - "jerusalem", - "jewelry", - "jewish", - "jewishart", - "jfk", - "journalism", - "judaica", - "judygarland", - "juedisches", - "juif", - "karate", - "karikatur", - "kids", - "koebenhavn", - "koeln", - "kunst", - "kunstsammlung", - "kunstunddesign", - "labor", - "labour", - "lajolla", - "lancashire", - "landes", - "lans", - "larsson", - "lewismiller", - "lincoln", - "linz", - "living", - "livinghistory", - "localhistory", - "london", - "losangeles", - "louvre", - "loyalist", - "lucerne", - "luxembourg", - "luzern", - "mad", - "madrid", - "mallorca", - "manchester", - "mansion", - "mansions", - "manx", - "marburg", - "maritime", - "maritimo", - "maryland", - "marylhurst", - "media", - "medical", - "medizinhistorisches", - "meeres", - "memorial", - "mesaverde", - "michigan", - "midatlantic", - "military", - "mill", - "miners", - "mining", - "minnesota", - "missile", - "missoula", - "modern", - "moma", - "money", - "monmouth", - "monticello", - "montreal", - "moscow", - "motorcycle", - "muenchen", - "muenster", - "mulhouse", - "muncie", - "museet", - "museumcenter", - "museumvereniging", - "music", - "national", - "nationalfirearms", - "nationalheritage", - "nativeamerican", - "naturalhistory", - "naturalhistorymuseum", - "naturalsciences", - "nature", - "naturhistorisches", - "natuurwetenschappen", - "naumburg", - "naval", - "nebraska", - "neues", - "newhampshire", - "newjersey", - "newmexico", - "newport", - "newspaper", - "newyork", - "niepce", - "norfolk", - "north", - "nrw", - "nyc", - "nyny", - "oceanographic", - "oceanographique", - "omaha", - "online", - "ontario", - "openair", - "oregon", - "oregontrail", - "otago", - "oxford", - "pacific", - "paderborn", - "palace", - "paleo", - "palmsprings", - "panama", - "paris", - "pasadena", - "pharmacy", - "philadelphia", - "philadelphiaarea", - "philately", - "phoenix", - "photography", - "pilots", - "pittsburgh", - "planetarium", - "plantation", - "plants", - "plaza", - "portal", - "portland", - "portlligat", - "posts-and-telecommunications", - "preservation", - "presidio", - "press", - "project", - "public", - "pubol", - "quebec", - "railroad", - "railway", - "research", - "resistance", - "riodejaneiro", - "rochester", - "rockart", - "roma", - "russia", - "saintlouis", - "salem", - "salvadordali", - "salzburg", - "sandiego", - "sanfrancisco", - "santabarbara", - "santacruz", - "santafe", - "saskatchewan", - "satx", - "savannahga", - "schlesisches", - "schoenbrunn", - "schokoladen", - "school", - "schweiz", - "science", - "science-fiction", - "scienceandhistory", - "scienceandindustry", - "sciencecenter", - "sciencecenters", - "sciencehistory", - "sciences", - "sciencesnaturelles", - "scotland", - "seaport", - "settlement", - "settlers", - "shell", - "sherbrooke", - "sibenik", - "silk", - "ski", - "skole", - "society", - "sologne", - "soundandvision", - "southcarolina", - "southwest", - "space", - "spy", - "square", - "stadt", - "stalbans", - "starnberg", - "state", - "stateofdelaware", - "station", - "steam", - "steiermark", - "stjohn", - "stockholm", - "stpetersburg", - "stuttgart", - "suisse", - "surgeonshall", - "surrey", - "svizzera", - "sweden", - "sydney", - "tank", - "tcm", - "technology", - "telekommunikation", - "television", - "texas", - "textile", - "theater", - "time", - "timekeeping", - "topology", - "torino", - "touch", - "town", - "transport", - "tree", - "trolley", - "trust", - "trustee", - "uhren", - "ulm", - "undersea", - "university", - "usa", - "usantiques", - "usarts", - "uscountryestate", - "usculture", - "usdecorativearts", - "usgarden", - "ushistory", - "ushuaia", - "uslivinghistory", - "utah", - "uvic", - "valley", - "vantaa", - "versailles", - "viking", - "village", - "virginia", - "virtual", - "virtuel", - "vlaanderen", - "volkenkunde", - "wales", - "wallonie", - "war", - "washingtondc", - "watch-and-clock", - "watchandclock", - "western", - "westfalen", - "whaling", - "wildlife", - "williamsburg", - "windmill", - "workshop", - "xn--9dbhblg6di", - "xn--comunicaes-v6a2o", - "xn--correios-e-telecomunicaes-ghc29a", - "xn--h1aegh", - "xn--lns-qla", - "york", - "yorkshire", - "yosemite", - "youth", - "zoological", - "zoology", "aero", "biz", "com", @@ -16483,6 +15691,19 @@ var nodeLabels = [...]string{ "asso", "nom", "adobeaemcloud", + "adobeio-static", + "adobeioruntime", + "akadns", + "akamai", + "akamai-staging", + "akamaiedge", + "akamaiedge-staging", + "akamaihd", + "akamaihd-staging", + "akamaiorigin", + "akamaiorigin-staging", + "akamaized", + "akamaized-staging", "alwaysdata", "appudo", "at-band-camp", @@ -16532,6 +15753,10 @@ var nodeLabels = [...]string{ "dynv6", "eating-organic", "edgeapp", + "edgekey", + "edgekey-staging", + "edgesuite", + "edgesuite-staging", "elastx", "endofinternet", "familyds", @@ -16612,6 +15837,7 @@ var nodeLabels = [...]string{ "shopselect", "siteleaf", "square7", + "squares", "srcf", "static-access", "supabase", @@ -16634,6 +15860,7 @@ var nodeLabels = [...]string{ "cdn", "1", "2", + "3", "centralus", "eastasia", "eastus2", @@ -17619,6 +16846,7 @@ var nodeLabels = [...]string{ "is-very-nice", "is-very-sweet", "isa-geek", + "jpn", "js", "kicks-ass", "mayfirst", @@ -17774,6 +17002,7 @@ var nodeLabels = [...]string{ "org", "framer", "1337", + "ngrok", "biz", "com", "edu", @@ -17978,12 +17207,17 @@ var nodeLabels = [...]string{ "kwpsp", "mup", "mw", + "oia", "oirm", + "oke", + "oow", + "oschr", "oum", "pa", "pinb", "piw", "po", + "pr", "psp", "psse", "pup", @@ -18009,11 +17243,14 @@ var nodeLabels = [...]string{ "wios", "witd", "wiw", + "wkz", "wsa", "wskr", + "wsse", "wuoz", "wzmiuw", "zp", + "zpisdn", "co", "name", "own", @@ -18355,6 +17592,7 @@ var nodeLabels = [...]string{ "consulado", "edu", "embaixada", + "kirara", "mil", "net", "noho", @@ -18501,6 +17739,7 @@ var nodeLabels = [...]string{ "quickconnect", "rdv", "vpnplus", + "x0", "direct", "prequalifyme", "now-dns", @@ -18549,7 +17788,9 @@ var nodeLabels = [...]string{ "travel", "better-than", "dyndns", + "from", "on-the-web", + "sakura", "worse-than", "blogspot", "club", @@ -18602,6 +17843,7 @@ var nodeLabels = [...]string{ "dp", "edu", "gov", + "ie", "if", "in", "inf", @@ -18616,6 +17858,7 @@ var nodeLabels = [...]string{ "kirovograd", "km", "kr", + "kropyvnytskyi", "krym", "ks", "kv", @@ -19010,18 +18253,84 @@ var nodeLabels = [...]string{ "net", "org", "ac", + "ai", + "angiang", + "bacgiang", + "backan", + "baclieu", + "bacninh", + "baria-vungtau", + "bentre", + "binhdinh", + "binhduong", + "binhphuoc", + "binhthuan", "biz", "blogspot", + "camau", + "cantho", + "caobang", "com", + "daklak", + "daknong", + "danang", + "dienbien", + "dongnai", + "dongthap", "edu", + "gialai", "gov", + "hagiang", + "haiduong", + "haiphong", + "hanam", + "hanoi", + "hatinh", + "haugiang", "health", + "hoabinh", + "hungyen", + "id", "info", "int", + "io", + "khanhhoa", + "kiengiang", + "kontum", + "laichau", + "lamdong", + "langson", + "laocai", + "longan", + "namdinh", "name", "net", + "nghean", + "ninhbinh", + "ninhthuan", "org", + "phutho", + "phuyen", "pro", + "quangbinh", + "quangnam", + "quangngai", + "quangninh", + "quangtri", + "soctrang", + "sonla", + "tayninh", + "thaibinh", + "thainguyen", + "thanhhoa", + "thanhphohochiminh", + "thuathienhue", + "tiengiang", + "travinh", + "tuyenquang", + "vinhlong", + "vinhphuc", + "yenbai", "blog", "cn", "com", From f09e75378f8ee74a1ff2e883d590eac175d93fea Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 2 Aug 2023 12:41:31 -0700 Subject: [PATCH 26/46] quic: send and receive stream data Send and receive data in STREAM frames. Write-close streams and communicate the final size in a STREAM frame with the FIN bit. Return io.EOF on reads at the end of a stream. Handle stream-level flow control. Send window updates in MAX_STREAM_DATA frames, send STREAM_DATA_BLOCKED when flow control is not available. Does not include connection-level flow control, read-closing, aborting, or removing streams from a conn after both sides have closed the stream. For golang/go#58547 Change-Id: Ib2b449bf54eb6cf200c4f6e2dd2c33274dda3387 Reviewed-on: https://go-review.googlesource.com/c/net/+/515815 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/config.go | 25 + internal/quic/conn.go | 3 + internal/quic/conn_loss.go | 8 + internal/quic/conn_loss_test.go | 270 +++++++++++ internal/quic/conn_recv.go | 13 +- internal/quic/conn_streams.go | 31 +- internal/quic/conn_test.go | 2 + internal/quic/crypto_stream.go | 23 +- internal/quic/gate.go | 12 +- internal/quic/quic_test.go | 37 ++ internal/quic/stream.go | 394 ++++++++++++++-- internal/quic/stream_test.go | 794 ++++++++++++++++++++++++++++++++ 12 files changed, 1549 insertions(+), 63 deletions(-) create mode 100644 internal/quic/quic_test.go diff --git a/internal/quic/config.go b/internal/quic/config.go index 7d1b7433af..df493579f2 100644 --- a/internal/quic/config.go +++ b/internal/quic/config.go @@ -17,4 +17,29 @@ type Config struct { // TLSConfig is the endpoint's TLS configuration. // It must be non-nil and include at least one certificate or else set GetCertificate. TLSConfig *tls.Config + + // StreamReadBufferSize is the maximum amount of data sent by the peer that a + // stream will buffer for reading. + // If zero, the default value of 1MiB is used. + // If negative, the limit is zero. + StreamReadBufferSize int64 + + // StreamWriteBufferSize is the maximum amount of data a stream will buffer for + // sending to the peer. + // If zero, the default value of 1MiB is used. + // If negative, the limit is zero. + StreamWriteBufferSize int64 } + +func configDefault(v, def int64) int64 { + switch v { + case -1: + return 0 + case 0: + return def + } + return v +} + +func (c *Config) streamReadBufferSize() int64 { return configDefault(c.StreamReadBufferSize, 1<<20) } +func (c *Config) streamWriteBufferSize() int64 { return configDefault(c.StreamWriteBufferSize, 1<<20) } diff --git a/internal/quic/conn.go b/internal/quic/conn.go index 90e6739630..0952a79e81 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -160,6 +160,9 @@ func (c *Conn) discardKeys(now time.Time, space numberSpace) { // receiveTransportParameters applies transport parameters sent by the peer. func (c *Conn) receiveTransportParameters(p transportParameters) error { + c.streams.peerInitialMaxStreamDataBidiLocal = p.initialMaxStreamDataBidiLocal + c.streams.peerInitialMaxStreamDataRemote[bidiStream] = p.initialMaxStreamDataBidiRemote + c.streams.peerInitialMaxStreamDataRemote[uniStream] = p.initialMaxStreamDataUni c.peerAckDelayExponent = p.ackDelayExponent c.loss.setMaxAckDelay(p.maxAckDelay) if err := c.connIDState.setPeerActiveConnIDLimit(p.activeConnIDLimit, c.newConnIDFunc()); err != nil { diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go index ca178089d2..f42f7e5282 100644 --- a/internal/quic/conn_loss.go +++ b/internal/quic/conn_loss.go @@ -44,6 +44,14 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF case frameTypeCrypto: start, end := sent.nextRange() c.crypto[space].ackOrLoss(start, end, fate) + case frameTypeMaxStreamData, + frameTypeStreamDataBlocked: + id := streamID(sent.nextInt()) + s := c.streamForID(id) + if s == nil { + continue + } + s.ackOrLoss(sent.num, f, fate) case frameTypeStreamBase, frameTypeStreamBase | streamFinBit: id := streamID(sent.nextInt()) diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index e3d16a7baa..d9445150a3 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -7,7 +7,9 @@ package quic import ( + "context" "crypto/tls" + "fmt" "testing" ) @@ -145,7 +147,275 @@ func TestLostStreamFrameEmpty(t *testing.T) { data: []byte{}, }) }) +} + +func TestLostStreamWithData(t *testing.T) { + // "Application data sent in STREAM frames is retransmitted in new STREAM + // frames unless the endpoint has sent a RESET_STREAM for that stream." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.2 + // + // TODO: Lost stream frame after RESET_STREAM + lostFrameTest(t, func(t *testing.T, pto bool) { + data := []byte{0, 1, 2, 3, 4, 5, 6, 7} + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { + p.initialMaxStreamsUni = 1 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataUni = 1 << 20 + }) + s.Write(data[:4]) + tc.wantFrame("send [0,4)", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: data[:4], + }) + s.Write(data[4:8]) + tc.wantFrame("send [4,8)", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 4, + data: data[4:8], + }) + s.Close() + tc.wantFrame("send FIN", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 8, + fin: true, + data: []byte{}, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resend data", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + fin: true, + data: data[:8], + }) + }) +} + +func TestLostStreamPartialLoss(t *testing.T) { + // Conn sends four STREAM packets. + // ACKs are received for the packets containing bytes 0 and 2. + // The remaining packets are declared lost. + // The Conn resends only the lost data. + // + // This test doesn't have a PTO mode, because the ACK for the packet containing byte 2 + // starts the loss timer for the packet containing byte 1, and the PTO timer is not + // armed when the loss timer is. + data := []byte{0, 1, 2, 3} + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { + p.initialMaxStreamsUni = 1 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataUni = 1 << 20 + }) + for i := range data { + s.Write(data[i : i+1]) + tc.wantFrame(fmt.Sprintf("send STREAM frame with byte %v", i), + packetType1RTT, debugFrameStream{ + id: s.id, + off: int64(i), + data: data[i : i+1], + }) + if i%2 == 0 { + num := tc.sentFramePacket.num + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{ + {num, num + 1}, + }, + }) + } + } + const pto = false + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resend byte 1", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 1, + data: data[1:2], + }) + tc.wantFrame("resend byte 3", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 3, + data: data[3:4], + }) + tc.wantIdle("no more frames sent after packet loss") +} + +func TestLostMaxStreamDataFrame(t *testing.T) { + // "[...] an updated value is sent when the packet containing + // the most recent MAX_STREAM_DATA frame for a stream is lost" + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.8 + lostFrameTest(t, func(t *testing.T, pto bool) { + const maxWindowSize = 10 + buf := make([]byte, maxWindowSize) + tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) { + c.StreamReadBufferSize = maxWindowSize + }) + + // We send MAX_STREAM_DATA = 19. + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: make([]byte, maxWindowSize), + }) + if n, err := s.Read(buf[:maxWindowSize-1]); err != nil || n != maxWindowSize-1 { + t.Fatalf("Read() = %v, %v; want %v, nil", n, err, maxWindowSize-1) + } + tc.wantFrame("stream window is extended after reading data", + packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: (maxWindowSize * 2) - 1, + }) + + // MAX_STREAM_DATA = 20, which is only one more byte, so we don't send the frame. + if n, err := s.Read(buf); err != nil || n != 1 { + t.Fatalf("Read() = %v, %v; want %v, nil", n, err, 1) + } + tc.wantIdle("read doesn't extend window enough to send another MAX_STREAM_DATA") + + // The MAX_STREAM_DATA = 19 packet was lost, so we send 20. + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resent MAX_STREAM_DATA includes most current value", + packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: maxWindowSize * 2, + }) + }) +} + +func TestLostMaxStreamDataFrameAfterStreamFinReceived(t *testing.T) { + // "An endpoint SHOULD stop sending MAX_STREAM_DATA frames when + // the receiving part of the stream enters a "Size Known" or "Reset Recvd" state." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.8 + lostFrameTest(t, func(t *testing.T, pto bool) { + const maxWindowSize = 10 + buf := make([]byte, maxWindowSize) + tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) { + c.StreamReadBufferSize = maxWindowSize + }) + + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: make([]byte, maxWindowSize), + }) + if n, err := s.Read(buf); err != nil || n != maxWindowSize { + t.Fatalf("Read() = %v, %v; want %v, nil", n, err, maxWindowSize) + } + tc.wantFrame("stream window is extended after reading data", + packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 2 * maxWindowSize, + }) + + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + off: maxWindowSize, + fin: true, + }) + + tc.ignoreFrame(frameTypePing) + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantIdle("lost MAX_STREAM_DATA not resent for stream in 'size known'") + }) +} + +func TestLostStreamDataBlockedFrame(t *testing.T) { + // "A new [STREAM_DATA_BLOCKED] frame is sent if a packet containing + // the most recent frame for a scope is lost [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.10 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { + p.initialMaxStreamsUni = 1 + p.initialMaxData = 1 << 20 + }) + + w := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, []byte{0, 1, 2, 3}) + }) + defer w.cancel() + tc.wantFrame("write is blocked by flow control", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 0, + }) + + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 1, + }) + tc.wantFrame("write makes some progress, but is still blocked by flow control", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 1, + }) + tc.wantFrame("write consuming available window", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: []byte{0}, + }) + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("STREAM_DATA_BLOCKED is resent", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 1, + }) + tc.wantFrame("STREAM is resent as well", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: []byte{0}, + }) + }) +} + +func TestLostStreamDataBlockedFrameAfterStreamUnblocked(t *testing.T) { + // "A new [STREAM_DATA_BLOCKED] frame is sent [...] only while + // the endpoint is blocked on the corresponding limit." + // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.10 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { + p.initialMaxStreamsUni = 1 + p.initialMaxData = 1 << 20 + }) + + data := []byte{0, 1, 2, 3} + w := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, data) + }) + defer w.cancel() + tc.wantFrame("write is blocked by flow control", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 0, + }) + + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 10, + }) + tc.wantFrame("write completes after flow control available", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: data, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("STREAM data is resent", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: data, + }) + tc.wantIdle("STREAM_DATA_BLOCKED is not resent, since the stream is not blocked") + }) } func TestLostNewConnectionIDFrame(t *testing.T) { diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 45ef3844e8..00985b6703 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -191,7 +191,7 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, if !frameOK(c, ptype, __01) { return } - _, _, n = consumeMaxStreamDataFrame(payload) + n = c.handleMaxStreamDataFrame(now, payload) case frameTypeMaxStreamsBidi, frameTypeMaxStreamsUni: if !frameOK(c, ptype, __01) { return @@ -280,6 +280,17 @@ func (c *Conn) handleAckFrame(now time.Time, space numberSpace, payload []byte) return n } +func (c *Conn) handleMaxStreamDataFrame(now time.Time, payload []byte) int { + id, maxStreamData, n := consumeMaxStreamDataFrame(payload) + if s := c.streamForFrame(now, id, sendStream); s != nil { + if err := s.handleMaxStreamData(maxStreamData); err != nil { + c.abort(now, err) + return -1 + } + } + return n +} + func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byte) int { off, data, n := consumeCryptoFrame(payload) err := c.handleCrypto(now, space, off, data) diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go index f626323b5a..7a531f52b2 100644 --- a/internal/quic/conn_streams.go +++ b/internal/quic/conn_streams.go @@ -20,6 +20,10 @@ type streamsState struct { streams map[streamID]*Stream opened [streamTypeCount]int64 // number of streams opened by us + // Peer configuration provided in transport parameters. + peerInitialMaxStreamDataRemote [streamTypeCount]int64 // streams opened by us + peerInitialMaxStreamDataBidiLocal int64 // streams opened by them + // Streams with frames to send are stored in a circular linked list. // sendHead is the next stream to write, or nil if there are no streams // with data to send. sendTail is the last stream to write. @@ -55,15 +59,24 @@ func (c *Conn) NewSendOnlyStream(ctx context.Context) (*Stream, error) { return c.newLocalStream(ctx, uniStream) } -func (c *Conn) newLocalStream(ctx context.Context, typ streamType) (*Stream, error) { +func (c *Conn) newLocalStream(ctx context.Context, styp streamType) (*Stream, error) { // TODO: Stream limits. c.streams.streamsMu.Lock() defer c.streams.streamsMu.Unlock() - num := c.streams.opened[typ] - c.streams.opened[typ]++ + num := c.streams.opened[styp] + c.streams.opened[styp]++ + + s := newStream(c, newStreamID(c.side, styp, num)) + s.outmaxbuf = c.config.streamWriteBufferSize() + s.outwin = c.streams.peerInitialMaxStreamDataRemote[styp] + if styp == bidiStream { + s.inmaxbuf = c.config.streamReadBufferSize() + s.inwin = c.config.streamReadBufferSize() + } + s.inUnlock() + s.outUnlock() - s := newStream(c, newStreamID(c.side, typ, num)) c.streams.streams[s.id] = s return s, nil } @@ -117,7 +130,17 @@ func (c *Conn) streamForFrame(now time.Time, id streamID, ftype streamFrameType) c.abort(now, localTransportError(errStreamState)) return nil } + s := newStream(c, id) + s.inmaxbuf = c.config.streamReadBufferSize() + s.inwin = c.config.streamReadBufferSize() + if id.streamType() == bidiStream { + s.outmaxbuf = c.config.streamWriteBufferSize() + s.outwin = c.streams.peerInitialMaxStreamDataBidiLocal + } + s.inUnlock() + s.outUnlock() + c.streams.streams[id] = s c.streams.queue.put(s) return s diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 5aad69f4d1..2480f9cb0a 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -179,6 +179,8 @@ func newTestConn(t *testing.T, side connSide, opts ...any) *testConn { peerProvidedParams := defaultTransportParameters() for _, o := range opts { switch o := o.(type) { + case func(*Config): + o(config) case func(*tls.Config): o(config.TLSConfig) case func(p *transportParameters): diff --git a/internal/quic/crypto_stream.go b/internal/quic/crypto_stream.go index 6cda6578c1..75dea87d0d 100644 --- a/internal/quic/crypto_stream.go +++ b/internal/quic/crypto_stream.go @@ -118,28 +118,7 @@ func (s *cryptoStream) ackOrLoss(start, end int64, fate packetFate) { // copy the data it wants into position. func (s *cryptoStream) dataToSend(pto bool, f func(off, size int64) (sent int64)) { for { - var off, size int64 - if pto { - // On PTO, resend unacked data that fits in the probe packet. - // For simplicity, we send the range starting at s.out.start - // (which is definitely unacked, or else we would have discarded it) - // up to the next acked byte (if any). - // - // This may miss unacked data starting after that acked byte, - // but avoids resending data the peer has acked. - off = s.out.start - end := s.out.end - for _, r := range s.outacked { - if r.start > off { - end = r.start - break - } - } - size = end - s.out.start - } else if s.outunsent.numRanges() > 0 { - off = s.outunsent.min() - size = s.outunsent[0].size() - } + off, size := dataToSend(s.out, s.outunsent, s.outacked, pto) if size == 0 { return } diff --git a/internal/quic/gate.go b/internal/quic/gate.go index efb28daf8f..27ab07a6f9 100644 --- a/internal/quic/gate.go +++ b/internal/quic/gate.go @@ -20,13 +20,19 @@ type gate struct { unset chan struct{} } +// newGate returns a new, unlocked gate with the condition unset. func newGate() gate { - g := gate{ + g := newLockedGate() + g.unlock(false) + return g +} + +// newLocked gate returns a new, locked gate. +func newLockedGate() gate { + return gate{ set: make(chan struct{}, 1), unset: make(chan struct{}, 1), } - g.unset <- struct{}{} - return g } // lock acquires the gate unconditionally. diff --git a/internal/quic/quic_test.go b/internal/quic/quic_test.go new file mode 100644 index 0000000000..1281b54eec --- /dev/null +++ b/internal/quic/quic_test.go @@ -0,0 +1,37 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "testing" +) + +func testSides(t *testing.T, name string, f func(*testing.T, connSide)) { + if name != "" { + name += "/" + } + t.Run(name+"server", func(t *testing.T) { f(t, serverSide) }) + t.Run(name+"client", func(t *testing.T) { f(t, clientSide) }) +} + +func testStreamTypes(t *testing.T, name string, f func(*testing.T, streamType)) { + if name != "" { + name += "/" + } + t.Run(name+"bidi", func(t *testing.T) { f(t, bidiStream) }) + t.Run(name+"uni", func(t *testing.T) { f(t, uniStream) }) +} + +func testSidesAndStreamTypes(t *testing.T, name string, f func(*testing.T, connSide, streamType)) { + if name != "" { + name += "/" + } + t.Run(name+"server/bidi", func(t *testing.T) { f(t, serverSide, bidiStream) }) + t.Run(name+"client/bidi", func(t *testing.T) { f(t, clientSide, bidiStream) }) + t.Run(name+"server/uni", func(t *testing.T) { f(t, serverSide, uniStream) }) + t.Run(name+"client/uni", func(t *testing.T) { f(t, clientSide, uniStream) }) +} diff --git a/internal/quic/stream.go b/internal/quic/stream.go index b55f927e02..83215dfd3e 100644 --- a/internal/quic/stream.go +++ b/internal/quic/stream.go @@ -9,34 +9,57 @@ package quic import ( "context" "errors" + "io" ) type Stream struct { id streamID conn *Conn + // ingate's lock guards all receive-related state. + // + // The gate condition is set if a read from the stream will not block, + // either because the stream has available data or because the read will fail. + ingate gate + in pipe // received data + inwin int64 // last MAX_STREAM_DATA sent to the peer + insendmax sentVal // set when we should send MAX_STREAM_DATA to the peer + inmaxbuf int64 // maximum amount of data we will buffer + insize int64 // stream final size; -1 before this is known + inset rangeset[int64] // received ranges + // outgate's lock guards all send-related state. // // The gate condition is set if a write to the stream will not block, // either because the stream has available flow control or because // the write will fail. - outgate gate - outopened sentVal // set if we should open the stream + outgate gate + out pipe // buffered data to send + outwin int64 // maximum MAX_STREAM_DATA received from the peer + outmaxbuf int64 // maximum amount of data we will buffer + outunsent rangeset[int64] // ranges buffered but not yet sent + outacked rangeset[int64] // ranges sent and acknowledged + outopened sentVal // set if we should open the stream + outclosed sentVal // set by CloseWrite + outblocked sentVal // set when a write to the stream is blocked by flow control prev, next *Stream // guarded by streamsState.sendMu } +// newStream returns a new stream. +// +// The stream's ingate and outgate are locked. +// (We create the stream with locked gates so after the caller +// initializes the flow control window, +// unlocking outgate will set the stream writability state.) func newStream(c *Conn, id streamID) *Stream { s := &Stream{ conn: c, id: id, - outgate: newGate(), + insize: -1, // -1 indicates the stream size is unknown + ingate: newLockedGate(), + outgate: newLockedGate(), } - - // Lock and unlock outgate to update the stream writability state. - s.outgate.lock() - s.outUnlock() - return s } @@ -66,8 +89,48 @@ func (s *Stream) Read(b []byte) (n int, err error) { // returning all data sent by the peer. // If the peer terminates reads abruptly, ReadContext returns StreamResetError. func (s *Stream) ReadContext(ctx context.Context, b []byte) (n int, err error) { - // TODO: implement - return 0, errors.New("unimplemented") + if s.IsWriteOnly() { + return 0, errors.New("read from write-only stream") + } + // Wait until data is available. + if err := s.conn.waitAndLockGate(ctx, &s.ingate); err != nil { + return 0, err + } + defer s.inUnlock() + if s.insize == s.in.start { + return 0, io.EOF + } + // Getting here indicates the stream contains data to be read. + if len(s.inset) < 1 || s.inset[0].start != 0 || s.inset[0].end <= s.in.start { + panic("BUG: inconsistent input stream state") + } + if size := int(s.inset[0].end - s.in.start); size < len(b) { + b = b[:size] + } + start := s.in.start + end := start + int64(len(b)) + s.in.copy(start, b) + s.in.discardBefore(end) + if s.insize == -1 || s.insize > s.inwin { + if shouldUpdateFlowControl(s.inwin-s.in.start, s.inmaxbuf) { + // Update stream flow control with a STREAM_MAX_DATA frame. + s.insendmax.setUnsent() + } + } + if end == s.insize { + return len(b), io.EOF + } + return len(b), nil +} + +// shouldUpdateFlowControl determines whether to send a flow control window update. +// +// We want to balance keeping the peer well-supplied with flow control with not sending +// many small updates. +func shouldUpdateFlowControl(curwin, maxwin int64) bool { + // Update flow control if doing so gives the peer at least 64k tokens, + // or if it will double the current window. + return maxwin-curwin >= 64<<10 || curwin*2 < maxwin } // Write writes data to the stream. @@ -87,65 +150,330 @@ func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) if s.IsReadOnly() { return 0, errors.New("write to read-only stream") } - if len(b) > 0 { - // TODO: implement - return 0, errors.New("unimplemented") + canWrite := s.outgate.lock() + if s.outclosed.isSet() { + s.outUnlock() + return 0, errors.New("write to closed stream") } - if err := s.outgate.waitAndLockContext(ctx); err != nil { - return 0, err + if len(b) == 0 { + // We aren't writing any data, but send a STREAM frame to open the stream + // if we haven't done so already. + s.outopened.set() + } + for len(b) > 0 { + // The first time through this loop, we may or may not be write blocked. + // We exit the loop after writing all data, so on subsequent passes through + // the loop we are always write blocked. + if !canWrite { + // We're blocked, either by flow control or by our own buffer limit. + // We either need the peer to extend our flow control window, + // or ack some of our outstanding packets. + if s.out.end == s.outwin { + // We're blocked by flow control. + // Send a STREAM_DATA_BLOCKED frame to let the peer know. + s.outblocked.setUnsent() + } + s.outUnlock() + if err := s.conn.waitAndLockGate(ctx, &s.outgate); err != nil { + return n, err + } + // Successfully returning from waitAndLockGate means we are no longer + // write blocked. (Unlike traditional condition variables, gates do not + // have spurious wakeups.) + } + s.outblocked.clear() + // Write limit is min(our own buffer limit, the peer-provided flow control window). + // This is a stream offset. + lim := min(s.out.start+s.outmaxbuf, s.outwin) + // Amount to write is min(the full buffer, data up to the write limit). + // This is a number of bytes. + nn := min(int64(len(b)), lim-s.out.end) + // Copy the data into the output buffer and mark it as unsent. + s.outunsent.add(s.out.end, s.out.end+nn) + s.out.writeAt(b[:nn], s.out.end) + s.outopened.set() + b = b[nn:] + n += int(nn) + // If we have bytes left to send, we're blocked. + canWrite = false } + s.outUnlock() + return n, nil +} + +// Close closes the stream. +// See CloseContext for more details. +func (s *Stream) Close() error { + return s.CloseContext(context.Background()) +} + +// CloseContext closes the stream. +// Any blocked stream operations will be unblocked and return errors. +// +// CloseContext flushes any data in the stream write buffer and waits for the peer to +// acknowledge receipt of the data. +// If the stream has been reset, it waits for the peer to acknowledge the reset. +// If the context expires before the peer receives the stream's data, +// CloseContext discards the buffer and returns the context error. +func (s *Stream) CloseContext(ctx context.Context) error { + s.CloseRead() + s.CloseWrite() + // TODO: wait for peer to acknowledge data + // TODO: Return code from peer's RESET_STREAM frame? + return nil +} + +// CloseRead aborts reads on the stream. +// Any blocked reads will be unblocked and return errors. +// +// CloseRead notifies the peer that the stream has been closed for reading. +// It does not wait for the peer to acknowledge the closure. +// Use CloseContext to wait for the peer's acknowledgement. +func (s *Stream) CloseRead() { + if s.IsWriteOnly() { + return + } + // TODO: support read-closing streams with a STOP_SENDING frame +} + +// CloseWrite aborts writes on the stream. +// Any blocked writes will be unblocked and return errors. +// +// CloseWrite sends any data in the stream write buffer to the peer. +// It does not wait for the peer to acknowledge receipt of the data. +// Use CloseContext to wait for the peer's acknowledgement. +func (s *Stream) CloseWrite() { + if s.IsReadOnly() { + return + } + s.outgate.lock() defer s.outUnlock() + s.outclosed.set() +} - // Set outopened to send a STREAM frame with no data, - // opening the stream on the peer. - s.outopened.set() +// inUnlock unlocks s.ingate. +// It sets the gate condition if reads from s will not block. +// If s has receive-related frames to write, it notifies the Conn. +func (s *Stream) inUnlock() { + if s.inUnlockNoQueue() { + s.conn.queueStreamForSend(s) + } +} - return n, nil +// inUnlockNoQueue is inUnlock, +// but reports whether s has frames to write rather than notifying the Conn. +func (s *Stream) inUnlockNoQueue() (shouldSend bool) { + // TODO: STOP_SENDING + canRead := s.inset.contains(s.in.start) || // data available to read + s.insize == s.in.start // at EOF + s.ingate.unlock(canRead) + return s.insendmax.shouldSend() // STREAM_MAX_DATA } // outUnlock unlocks s.outgate. // It sets the gate condition if writes to s will not block. -// If s has frames to write, it notifies the Conn. +// If s has send-related frames to write, it notifies the Conn. func (s *Stream) outUnlock() { - if s.outopened.shouldSend() { + if s.outUnlockNoQueue() { s.conn.queueStreamForSend(s) } - canSend := true // TODO: set sendability status based on flow control - s.outgate.unlock(canSend) +} + +// outUnlockNoQueue is outUnlock, +// but reports whether s has frames to write rather than notifying the Conn. +func (s *Stream) outUnlockNoQueue() (shouldSend bool) { + lim := min(s.out.start+s.outmaxbuf, s.outwin) + canWrite := lim > s.out.end || // available flow control + s.outclosed.isSet() // closed + s.outgate.unlock(canWrite) + return len(s.outunsent) > 0 || // STREAM frame with data + s.outclosed.shouldSend() || // STREAM frame with FIN bit + s.outopened.shouldSend() || // STREAM frame with no data + s.outblocked.shouldSend() // STREAM_DATA_BLOCKED } // handleData handles data received in a STREAM frame. func (s *Stream) handleData(off int64, b []byte, fin bool) error { - // TODO + s.ingate.lock() + defer s.inUnlock() + end := off + int64(len(b)) + if end > s.inwin { + // The peer sent us data past the maximum flow control window we gave them. + return localTransportError(errFlowControl) + } + if s.insize != -1 && end > s.insize { + // The peer sent us data past the final size of the stream they previously gave us. + return localTransportError(errFinalSize) + } + s.in.writeAt(b, off) + s.inset.add(off, end) + if fin { + if s.insize != -1 && s.insize != end { + // The peer changed the final size of the stream. + return localTransportError(errFinalSize) + } + s.insize = end + // The peer has enough flow control window to send the entire stream. + s.insendmax.clear() + } + return nil +} + +// handleMaxStreamData handles an update received in a MAX_STREAM_DATA frame. +func (s *Stream) handleMaxStreamData(maxStreamData int64) error { + s.outgate.lock() + defer s.outUnlock() + s.outwin = max(maxStreamData, s.outwin) return nil } +// ackOrLoss handles the fate of stream frames other than STREAM. +func (s *Stream) ackOrLoss(pnum packetNumber, ftype byte, fate packetFate) { + // Frames which carry new information each time they are sent + // (MAX_STREAM_DATA, STREAM_DATA_BLOCKED) must only be marked + // as received if the most recent packet carrying this frame is acked. + // + // Frames which are always the same (STOP_SENDING, RESET_STREAM) + // can be marked as received if any packet carrying this frame is acked. + switch ftype { + case frameTypeMaxStreamData: + s.ingate.lock() + s.insendmax.ackLatestOrLoss(pnum, fate) + s.inUnlock() + case frameTypeStreamDataBlocked: + s.outgate.lock() + s.outblocked.ackLatestOrLoss(pnum, fate) + s.outUnlock() + default: + // TODO: Handle STOP_SENDING, RESET_STREAM. + panic("unhandled frame type") + } +} + // ackOrLossData handles the fate of a STREAM frame. func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fate packetFate) { s.outgate.lock() defer s.outUnlock() s.outopened.ackOrLoss(pnum, fate) + if fin { + s.outclosed.ackOrLoss(pnum, fate) + } + switch fate { + case packetAcked: + s.outacked.add(start, end) + s.outunsent.sub(start, end) + // If this ack is for data at the start of the send buffer, we can now discard it. + if s.outacked.contains(s.out.start) { + s.out.discardBefore(s.outacked[0].end) + } + case packetLost: + // Mark everything lost, but not previously acked, as needing retransmission. + // We do this by adding all the lost bytes to outunsent, and then + // removing everything already acked. + s.outunsent.add(start, end) + for _, a := range s.outacked { + s.outunsent.sub(a.start, a.end) + } + } } +// appendInFrames appends STOP_SENDING and MAX_STREAM_DATA frames +// to the current packet. +// +// It returns true if no more frames need appending, +// false if not everything fit in the current packet. func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + s.ingate.lock() + defer s.inUnlockNoQueue() // TODO: STOP_SENDING - // TODO: MAX_STREAM_DATA + if s.insendmax.shouldSendPTO(pto) { + // MAX_STREAM_DATA + maxStreamData := s.in.start + s.inmaxbuf + if !w.appendMaxStreamDataFrame(s.id, maxStreamData) { + return false + } + s.inwin = maxStreamData + s.insendmax.setSent(pnum) + } return true } +// appendOutFrames appends RESET_STREAM, STREAM_DATA_BLOCKED, and STREAM frames +// to the current packet. +// +// It returns true if no more frames need appending, +// false if not everything fit in the current packet. func (s *Stream) appendOutFrames(w *packetWriter, pnum packetNumber, pto bool) bool { + s.outgate.lock() + defer s.outUnlockNoQueue() // TODO: RESET_STREAM - // TODO: STREAM_DATA_BLOCKED - // TODO: STREAM frames with data - if s.outopened.shouldSendPTO(pto) { - off := int64(0) - size := 0 - fin := false - _, added := w.appendStreamFrame(s.id, off, size, fin) + if s.outblocked.shouldSendPTO(pto) { + // STREAM_DATA_BLOCKED + if !w.appendStreamDataBlockedFrame(s.id, s.out.end) { + return false + } + s.outblocked.setSent(pnum) + s.frameOpensStream(pnum) + } + // STREAM + for { + off, size := dataToSend(s.out, s.outunsent, s.outacked, pto) + fin := s.outclosed.isSet() && off+size == s.out.end + shouldSend := size > 0 || // have data to send + s.outopened.shouldSendPTO(pto) || // should open the stream + (fin && s.outclosed.shouldSendPTO(pto)) // should close the stream + if !shouldSend { + return true + } + b, added := w.appendStreamFrame(s.id, off, int(size), fin) if !added { return false } + s.out.copy(off, b) + s.outunsent.sub(off, off+int64(len(b))) + s.frameOpensStream(pnum) + if fin { + s.outclosed.setSent(pnum) + } + if pto { + return true + } + if int64(len(b)) < size { + return false + } + } +} + +// frameOpensStream records that we're sending a frame that will open the stream. +// +// If we don't have an acknowledgement from the peer for a previous frame opening the stream, +// record this packet as being the latest one to open it. +func (s *Stream) frameOpensStream(pnum packetNumber) { + if !s.outopened.isReceived() { s.outopened.setSent(pnum) } - return true +} + +// dataToSend returns the next range of data to send in a STREAM or CRYPTO_STREAM. +func dataToSend(out pipe, outunsent, outacked rangeset[int64], pto bool) (start, size int64) { + switch { + case pto: + // On PTO, resend unacked data that fits in the probe packet. + // For simplicity, we send the range starting at s.out.start + // (which is definitely unacked, or else we would have discarded it) + // up to the next acked byte (if any). + // + // This may miss unacked data starting after that acked byte, + // but avoids resending data the peer has acked. + for _, r := range outacked { + if r.start > out.start { + return out.start, r.start - out.start + } + } + return out.start, out.end - out.start + case outunsent.numRanges() > 0: + return outunsent.min(), outunsent[0].size() + default: + return out.end, 0 + } } diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index 8ae9dbc825..d158e72afb 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -7,10 +7,703 @@ package quic import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "io" "reflect" + "strings" "testing" ) +func TestStreamWriteBlockedByStreamFlowControl(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + ctx := canceledContext() + want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + tc := newTestConn(t, clientSide, func(p *transportParameters) { + p.initialMaxStreamsBidi = 100 + p.initialMaxStreamsUni = 100 + p.initialMaxData = 1 << 20 + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + // Non-blocking write with no flow control. + s, err := tc.conn.newLocalStream(ctx, styp) + if err != nil { + t.Fatal(err) + } + _, err = s.WriteContext(ctx, want) + if err != context.Canceled { + t.Fatalf("write to stream with no flow control: err = %v, want context.Canceled", err) + } + tc.wantFrame("write blocked by flow control triggers a STREAM_DATA_BLOCKED frame", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 0, + }) + + // Blocking write waiting for flow control. + w := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, want) + }) + tc.wantFrame("second blocked write triggers another STREAM_DATA_BLOCKED", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 0, + }) + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 4, + }) + tc.wantFrame("stream window extended, but still more data to write", + packetType1RTT, debugFrameStreamDataBlocked{ + id: s.id, + max: 4, + }) + tc.wantFrame("stream window extended to 4, expect blocked write to progress", + packetType1RTT, debugFrameStream{ + id: s.id, + data: want[:4], + }) + + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: int64(len(want)), + }) + tc.wantFrame("stream window extended further, expect blocked write to finish", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 4, + data: want[4:], + }) + n, err := w.result() + if n != len(want) || err != nil { + t.Errorf("Write() = %v, %v; want %v, nil", n, err, len(want)) + } + }) +} + +func TestStreamIgnoresMaxStreamDataReduction(t *testing.T) { + // "A sender MUST ignore any MAX_STREAM_DATA [...] frames that + // do not increase flow control limits." + // https://www.rfc-editor.org/rfc/rfc9000#section-4.1-9 + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + ctx := canceledContext() + want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + tc := newTestConn(t, clientSide, func(p *transportParameters) { + if styp == uniStream { + p.initialMaxStreamsUni = 1 + p.initialMaxStreamDataUni = 4 + } else { + p.initialMaxStreamsBidi = 1 + p.initialMaxStreamDataBidiRemote = 4 + } + p.initialMaxData = 1 << 20 + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeStreamDataBlocked) + + // Write [0,1). + s, err := tc.conn.newLocalStream(ctx, styp) + if err != nil { + t.Fatal(err) + } + s.WriteContext(ctx, want[:1]) + tc.wantFrame("sent data (1 byte) fits within flow control limit", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: want[:1], + }) + + // MAX_STREAM_DATA tries to decrease limit, and is ignored. + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 2, + }) + + // Write [1,4). + s.WriteContext(ctx, want[1:]) + tc.wantFrame("stream limit is 4 bytes, ignoring decrease in MAX_STREAM_DATA", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 1, + data: want[1:4], + }) + + // MAX_STREAM_DATA increases limit. + // Second MAX_STREAM_DATA decreases it, and is ignored. + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 8, + }) + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 6, + }) + + // Write [1,4). + s.WriteContext(ctx, want[4:]) + tc.wantFrame("stream limit is 8 bytes, ignoring decrease in MAX_STREAM_DATA", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 4, + data: want[4:8], + }) + }) +} + +func TestStreamWriteBlockedByWriteBufferLimit(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + ctx := canceledContext() + want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + const maxWriteBuffer = 4 + tc := newTestConn(t, clientSide, func(p *transportParameters) { + p.initialMaxStreamsBidi = 100 + p.initialMaxStreamsUni = 100 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataBidiRemote = 1 << 20 + p.initialMaxStreamDataUni = 1 << 20 + }, func(c *Config) { + c.StreamWriteBufferSize = maxWriteBuffer + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + // Write more data than StreamWriteBufferSize. + // The peer has given us plenty of flow control, + // so we're just blocked by our local limit. + s, err := tc.conn.newLocalStream(ctx, styp) + if err != nil { + t.Fatal(err) + } + w := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, want) + }) + tc.wantFrame("stream write should send as much data as write buffer allows", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: want[:maxWriteBuffer], + }) + tc.wantIdle("no STREAM_DATA_BLOCKED, we're blocked locally not by flow control") + + // ACK for previously-sent data allows making more progress. + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, + }) + tc.wantFrame("ACK for previous data allows making progress", + packetType1RTT, debugFrameStream{ + id: s.id, + off: maxWriteBuffer, + data: want[maxWriteBuffer:][:maxWriteBuffer], + }) + + // Cancel the write with data left to send. + w.cancel() + n, err := w.result() + if n != 2*maxWriteBuffer || err == nil { + t.Fatalf("WriteContext() = %v, %v; want %v bytes, error", n, err, 2*maxWriteBuffer) + } + }) +} + +func TestStreamReceive(t *testing.T) { + // "Endpoints MUST be able to deliver stream data to an application as + // an ordered byte stream." + // https://www.rfc-editor.org/rfc/rfc9000#section-2.2-2 + want := make([]byte, 5000) + for i := range want { + want[i] = byte(i) + } + type frame struct { + start int64 + end int64 + fin bool + want int + wantEOF bool + } + for _, test := range []struct { + name string + frames []frame + }{{ + name: "linear", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 1000, + end: 2000, + want: 2000, + }, { + start: 2000, + end: 3000, + want: 3000, + fin: true, + wantEOF: true, + }}, + }, { + name: "out of order", + frames: []frame{{ + start: 1000, + end: 2000, + }, { + start: 2000, + end: 3000, + }, { + start: 0, + end: 1000, + want: 3000, + }}, + }, { + name: "resent", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 0, + end: 1000, + want: 1000, + }, { + start: 1000, + end: 2000, + want: 2000, + }, { + start: 0, + end: 1000, + want: 2000, + }, { + start: 1000, + end: 2000, + want: 2000, + }}, + }, { + name: "overlapping", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 3000, + end: 4000, + want: 1000, + }, { + start: 2000, + end: 3000, + want: 1000, + }, { + start: 1000, + end: 3000, + want: 4000, + }}, + }, { + name: "early eof", + frames: []frame{{ + start: 3000, + end: 3000, + fin: true, + want: 0, + }, { + start: 1000, + end: 2000, + want: 0, + }, { + start: 0, + end: 1000, + want: 2000, + }, { + start: 2000, + end: 3000, + want: 3000, + wantEOF: true, + }}, + }, { + name: "empty eof", + frames: []frame{{ + start: 0, + end: 1000, + want: 1000, + }, { + start: 1000, + end: 1000, + fin: true, + want: 1000, + wantEOF: true, + }}, + }} { + testStreamTypes(t, test.name, func(t *testing.T, styp streamType) { + ctx := canceledContext() + tc := newTestConn(t, serverSide) + tc.handshake() + sid := newStreamID(clientSide, styp, 0) + var s *Stream + got := make([]byte, len(want)) + var total int + for _, f := range test.frames { + t.Logf("receive [%v,%v)", f.start, f.end) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: f.start, + data: want[f.start:f.end], + fin: f.fin, + }) + if s == nil { + var err error + s, err = tc.conn.AcceptStream(ctx) + if err != nil { + tc.t.Fatalf("conn.AcceptStream() = %v", err) + } + } + for { + n, err := s.ReadContext(ctx, got[total:]) + t.Logf("s.ReadContext() = %v, %v", n, err) + total += n + if f.wantEOF && err != io.EOF { + t.Fatalf("ReadContext() error = %v; want io.EOF", err) + } + if !f.wantEOF && err == io.EOF { + t.Fatalf("ReadContext() error = io.EOF, want something else") + } + if err != nil { + break + } + } + if total != f.want { + t.Fatalf("total bytes read = %v, want %v", total, f.want) + } + for i := 0; i < total; i++ { + if got[i] != want[i] { + t.Fatalf("byte %v differs: got %v, want %v", i, got[i], want[i]) + } + } + } + }) + } + +} + +func TestStreamReceiveExtendsStreamWindow(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + const maxWindowSize = 20 + ctx := canceledContext() + tc := newTestConn(t, serverSide, func(c *Config) { + c.StreamReadBufferSize = maxWindowSize + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + sid := newStreamID(clientSide, styp, 0) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 0, + data: make([]byte, maxWindowSize), + }) + s, err := tc.conn.AcceptStream(ctx) + if err != nil { + t.Fatalf("AcceptStream: %v", err) + } + tc.wantIdle("stream window is not extended before data is read") + buf := make([]byte, maxWindowSize+1) + if n, err := s.ReadContext(ctx, buf); n != maxWindowSize || err != nil { + t.Fatalf("s.ReadContext() = %v, %v; want %v, nil", n, err, maxWindowSize) + } + tc.wantFrame("stream window is extended after reading data", + packetType1RTT, debugFrameMaxStreamData{ + id: sid, + max: maxWindowSize * 2, + }) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: maxWindowSize, + data: make([]byte, maxWindowSize), + fin: true, + }) + if n, err := s.ReadContext(ctx, buf); n != maxWindowSize || err != io.EOF { + t.Fatalf("s.ReadContext() = %v, %v; want %v, io.EOF", n, err, maxWindowSize) + } + tc.wantIdle("stream window is not extended after FIN") + }) +} + +func TestStreamReceiveViolatesStreamDataLimit(t *testing.T) { + // "A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if + // the sender violates the advertised [...] stream data limits [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-4.1-8 + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + const maxStreamData = 10 + for _, test := range []struct { + off int64 + size int64 + }{{ + off: maxStreamData, + size: 1, + }, { + off: 0, + size: maxStreamData + 1, + }, { + off: maxStreamData - 1, + size: 2, + }} { + tc := newTestConn(t, serverSide, func(c *Config) { + c.StreamReadBufferSize = maxStreamData + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: newStreamID(clientSide, styp, 0), + off: test.off, + data: make([]byte, test.size), + }) + tc.wantFrame( + fmt.Sprintf("data [%v,%v) violates stream data limit and closes connection", + test.off, test.off+test.size), + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errFlowControl, + }, + ) + } + }) +} + +func TestStreamReceiveDuplicateDataDoesNotViolateLimits(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + const maxData = 10 + tc := newTestConn(t, serverSide, func(c *Config) { + // TODO: Add connection-level maximum data here as well. + c.StreamReadBufferSize = maxData + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + for i := 0; i < 3; i++ { + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: newStreamID(clientSide, styp, 0), + off: 0, + data: make([]byte, maxData), + }) + tc.wantIdle(fmt.Sprintf("conn sends no frames after receiving data frame %v", i)) + } + }) +} + +func TestStreamFinalSizeChangedByStreamFrame(t *testing.T) { + // "If a [...] STREAM frame is received indicating a change + // in the final size for the stream, an endpoint SHOULD + // respond with an error of type FINAL_SIZE_ERROR [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5 + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc := newTestConn(t, serverSide) + tc.handshake() + sid := newStreamID(clientSide, styp, 0) + + const write1size = 4 + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 10, + fin: true, + }) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 9, + fin: true, + }) + tc.wantFrame("change in final size of stream is an error", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errFinalSize, + }, + ) + }) +} + +func TestStreamDataBeyondFinalSize(t *testing.T) { + // "A receiver SHOULD treat receipt of data at or beyond + // the final size as an error of type FINAL_SIZE_ERROR [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5 + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc := newTestConn(t, serverSide) + tc.handshake() + sid := newStreamID(clientSide, styp, 0) + + const write1size = 4 + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 0, + data: make([]byte, 16), + fin: true, + }) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 16, + data: []byte{0}, + }) + tc.wantFrame("received data past final size of stream", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errFinalSize, + }, + ) + }) +} + +func TestStreamReceiveUnblocksReader(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc := newTestConn(t, serverSide) + tc.handshake() + want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + sid := newStreamID(clientSide, styp, 0) + + // AcceptStream blocks until a STREAM frame is received. + accept := runAsync(tc, func(ctx context.Context) (*Stream, error) { + return tc.conn.AcceptStream(ctx) + }) + const write1size = 4 + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: 0, + data: want[:write1size], + }) + s, err := accept.result() + if err != nil { + t.Fatalf("AcceptStream() = %v", err) + } + + // ReadContext succeeds immediately, since we already have data. + got := make([]byte, len(want)) + read := runAsync(tc, func(ctx context.Context) (int, error) { + return s.ReadContext(ctx, got) + }) + if n, err := read.result(); n != write1size || err != nil { + t.Fatalf("ReadContext = %v, %v; want %v, nil", n, err, write1size) + } + + // ReadContext blocks waiting for more data. + read = runAsync(tc, func(ctx context.Context) (int, error) { + return s.ReadContext(ctx, got[write1size:]) + }) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: write1size, + data: want[write1size:], + fin: true, + }) + if n, err := read.result(); n != len(want)-write1size || err != io.EOF { + t.Fatalf("ReadContext = %v, %v; want %v, io.EOF", n, err, len(want)-write1size) + } + if !bytes.Equal(got, want) { + t.Fatalf("read bytes %x, want %x", got, want) + } + }) +} + +// testStreamSendFrameInvalidState calls the test func with a stream ID for: +// +// - a remote bidirectional stream that the peer has not created +// - a remote unidirectional stream +// +// It then sends the returned frame (STREAM, STREAM_DATA_BLOCKED, etc.) +// to the conn and expects a STREAM_STATE_ERROR. +func testStreamSendFrameInvalidState(t *testing.T, f func(sid streamID) debugFrame) { + testSides(t, "stream_not_created", func(t *testing.T, side connSide) { + tc := newTestConn(t, side) + tc.handshake() + tc.writeFrames(packetType1RTT, f(newStreamID(side, bidiStream, 0))) + tc.wantFrame("frame for local stream which has not been created", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) + }) + testSides(t, "uni_stream", func(t *testing.T, side connSide) { + ctx := canceledContext() + tc := newTestConn(t, side) + tc.handshake() + sid := newStreamID(side, uniStream, 0) + s, err := tc.conn.NewSendOnlyStream(ctx) + if err != nil { + t.Fatal(err) + } + s.Write(nil) // open the stream + tc.wantFrame("new stream is opened", + packetType1RTT, debugFrameStream{ + id: sid, + data: []byte{}, + }) + tc.writeFrames(packetType1RTT, f(sid)) + tc.wantFrame("send-oriented frame for send-only stream", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) + }) +} + +func TestStreamStreamFrameInvalidState(t *testing.T) { + // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR + // if it receives a STREAM frame for a locally initiated stream + // that has not yet been created, or for a send-only stream." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3 + testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame { + return debugFrameStream{ + id: sid, + } + }) +} + +func TestStreamDataBlockedInvalidState(t *testing.T) { + // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR + // if it receives a STREAM frame for a locally initiated stream + // that has not yet been created, or for a send-only stream." + // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3 + testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame { + return debugFrameStream{ + id: sid, + } + }) +} + +// testStreamReceiveFrameInvalidState calls the test func with a stream ID for: +// +// - a remote bidirectional stream that the peer has not created +// - a local unidirectional stream +// +// It then sends the returned frame (MAX_STREAM_DATA, STOP_SENDING, etc.) +// to the conn and expects a STREAM_STATE_ERROR. +func testStreamReceiveFrameInvalidState(t *testing.T, f func(sid streamID) debugFrame) { + testSides(t, "stream_not_created", func(t *testing.T, side connSide) { + tc := newTestConn(t, side) + tc.handshake() + tc.writeFrames(packetType1RTT, f(newStreamID(side, bidiStream, 0))) + tc.wantFrame("frame for local stream which has not been created", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) + }) + testSides(t, "uni_stream", func(t *testing.T, side connSide) { + tc := newTestConn(t, side) + tc.handshake() + tc.writeFrames(packetType1RTT, f(newStreamID(side.peer(), uniStream, 0))) + tc.wantFrame("receive-oriented frame for receive-only stream", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: errStreamState, + }) + }) +} + +func TestStreamMaxStreamDataInvalidState(t *testing.T) { + // "Receiving a MAX_STREAM_DATA frame for a locally initiated stream + // that has not yet been created MUST be treated as a connection error + // of type STREAM_STATE_ERROR. An endpoint that receives a MAX_STREAM_DATA + // frame for a receive-only stream MUST terminate the connection + // with error STREAM_STATE_ERROR." + // https://www.rfc-editor.org/rfc/rfc9000#section-19.10-2 + testStreamReceiveFrameInvalidState(t, func(sid streamID) debugFrame { + return debugFrameMaxStreamData{ + id: sid, + max: 1000, + } + }) +} + func TestStreamOffsetTooLarge(t *testing.T) { // "Receipt of a frame that exceeds [2^62-1] MUST be treated as a // connection error of type FRAME_ENCODING_ERROR or FLOW_CONTROL_ERROR." @@ -31,3 +724,104 @@ func TestStreamOffsetTooLarge(t *testing.T) { t.Fatalf("STREAM offset exceeds 2^62-1\ngot: %v\nwant: %v\n or: %v", got, want1, want2) } } + +func TestStreamReadFromWriteOnlyStream(t *testing.T) { + _, s := newTestConnAndLocalStream(t, serverSide, uniStream) + buf := make([]byte, 10) + wantErr := "read from write-only stream" + if n, err := s.Read(buf); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr) + } +} + +func TestStreamWriteToReadOnlyStream(t *testing.T) { + _, s := newTestConnAndRemoteStream(t, serverSide, uniStream) + buf := make([]byte, 10) + wantErr := "write to read-only stream" + if n, err := s.Write(buf); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr) + } +} + +func TestStreamWriteToClosedStream(t *testing.T) { + tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, func(p *transportParameters) { + p.initialMaxStreamsBidi = 1 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataBidiRemote = 1 << 20 + }) + s.Close() + tc.wantFrame("stream is opened after being closed", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + fin: true, + data: []byte{}, + }) + wantErr := "write to closed stream" + if n, err := s.Write([]byte{}); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr) + } +} + +func TestStreamWriteMoreThanOnePacketOfData(t *testing.T) { + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { + p.initialMaxStreamsUni = 1 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataUni = 1 << 20 + }) + want := make([]byte, 4096) + rand.Read(want) // doesn't need to be crypto/rand, but non-deprecated and harmless + w := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, want) + }) + got := make([]byte, 0, len(want)) + for { + f, _ := tc.readFrame() + if f == nil { + break + } + sf, ok := f.(debugFrameStream) + if !ok { + t.Fatalf("unexpected frame: %v", sf) + } + if len(got) != int(sf.off) { + t.Fatalf("got frame: %v\nwant offset %v", sf, len(got)) + } + got = append(got, sf.data...) + } + if n, err := w.result(); n != len(want) || err != nil { + t.Fatalf("s.WriteContext() = %v, %v; want %v, nil", n, err, len(want)) + } + if !bytes.Equal(got, want) { + t.Fatalf("mismatch in received stream data") + } +} + +func newTestConnAndLocalStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) { + t.Helper() + ctx := canceledContext() + tc := newTestConn(t, side, opts...) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + s, err := tc.conn.newLocalStream(ctx, styp) + if err != nil { + t.Fatalf("conn.newLocalStream(%v) = %v", styp, err) + } + return tc, s +} + +func newTestConnAndRemoteStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) { + t.Helper() + ctx := canceledContext() + tc := newTestConn(t, side, opts...) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: newStreamID(side.peer(), styp, 0), + }) + s, err := tc.conn.AcceptStream(ctx) + if err != nil { + t.Fatalf("conn.AcceptStream() = %v", err) + } + return tc, s +} From 126a5f3b343c940b1ce677f43b138556311b0999 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 9 Aug 2023 10:55:08 -0700 Subject: [PATCH 27/46] quic: fix some bugs in the sendable stream list Write a test for multiple streams simultaneously sending data. Exercise the stream send queue, verify that we fairly schedule sends among the available streams. Fix a couple bugs turned up by the test. For golang/go#58547 Change-Id: I6a56f121d5cb49e79c9e4ad043fb94d34a4dab40 Reviewed-on: https://go-review.googlesource.com/c/net/+/517859 Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- internal/quic/conn_streams.go | 10 ++-- internal/quic/conn_streams_test.go | 82 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go index 7a531f52b2..dd35e34cf6 100644 --- a/internal/quic/conn_streams.go +++ b/internal/quic/conn_streams.go @@ -163,6 +163,7 @@ func (c *Conn) queueStreamForSend(s *Stream) { // Insert this stream at the end of the queue. c.streams.sendTail.next = s c.streams.sendTail = s + s.next = c.streams.sendHead } c.streams.needSend.Store(true) c.wake() @@ -202,7 +203,11 @@ func (c *Conn) appendStreamFrames(w *packetWriter, pnum packetNumber, pto bool) } return false } + next := s.next s.next = nil + if (next == s) != (s == c.streams.sendTail) { + panic("BUG: sendable stream list state is inconsistent") + } if s == c.streams.sendTail { // This was the last stream. c.streams.sendHead = nil @@ -211,9 +216,8 @@ func (c *Conn) appendStreamFrames(w *packetWriter, pnum packetNumber, pto bool) return true } // We've sent all data for this stream, so remove it from the list. - c.streams.sendTail.next = s.next - c.streams.sendHead = s.next - s.next = nil + c.streams.sendTail.next = next + c.streams.sendHead = next } } diff --git a/internal/quic/conn_streams_test.go b/internal/quic/conn_streams_test.go index bcbbe81ce3..877dbb94fc 100644 --- a/internal/quic/conn_streams_test.go +++ b/internal/quic/conn_streams_test.go @@ -171,3 +171,85 @@ func TestStreamsStreamSendOnly(t *testing.T) { code: errStreamState, }) } + +func TestStreamsWriteQueueFairness(t *testing.T) { + ctx := canceledContext() + const dataLen = 1 << 20 + const numStreams = 3 + tc := newTestConn(t, clientSide, func(p *transportParameters) { + p.initialMaxStreamsBidi = numStreams + p.initialMaxData = 1<<62 - 1 + p.initialMaxStreamDataBidiRemote = dataLen + }, func(c *Config) { + c.StreamWriteBufferSize = dataLen + }) + tc.handshake() + tc.ignoreFrame(frameTypeAck) + + // Create a number of streams, and write a bunch of data to them. + // The streams are not limited by flow control. + // + // The first stream we create is going to immediately consume all + // available congestion window. + // + // Once we've created all the remaining streams, + // we start sending acks back to open up the congestion window. + // We verify that all streams can make progress. + data := make([]byte, dataLen) + var streams []*Stream + for i := 0; i < numStreams; i++ { + s, err := tc.conn.NewStream(ctx) + if err != nil { + t.Fatal(err) + } + streams = append(streams, s) + if n, err := s.WriteContext(ctx, data); n != len(data) || err != nil { + t.Fatalf("s.WriteContext() = %v, %v; want %v, nil", n, err, len(data)) + } + // Wait for the stream to finish writing whatever frames it can before + // congestion control blocks it. + tc.wait() + } + + sent := make([]int64, len(streams)) + for { + p := tc.readPacket() + if p == nil { + break + } + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, p.num}}, + }) + for _, f := range p.frames { + sf, ok := f.(debugFrameStream) + if !ok { + t.Fatalf("got unexpected frame (want STREAM): %v", sf) + } + if got, want := sf.off, sent[sf.id.num()]; got != want { + t.Fatalf("got frame: %v\nwant offset: %v", sf, want) + } + sent[sf.id.num()] = sf.off + int64(len(sf.data)) + // Look at the amount of data sent by all streams, excluding the first one. + // (The first stream got a head start when it consumed the initial window.) + // + // We expect that difference between the streams making the most and least progress + // so far will be less than the maximum datagram size. + minSent := sent[1] + maxSent := sent[1] + for _, s := range sent[2:] { + minSent = min(minSent, s) + maxSent = max(maxSent, s) + } + const maxDelta = maxUDPPayloadSize + if d := maxSent - minSent; d > maxDelta { + t.Fatalf("stream data sent: %v; delta=%v, want delta <= %v", sent, d, maxDelta) + } + } + } + // Final check that every stream sent the full amount of data expected. + for num, s := range sent { + if s != dataLen { + t.Errorf("stream %v sent %v bytes, want %v", num, s, dataLen) + } + } +} From 95cb3bb9eb72a38ad7817552051746cf41999f5a Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Sat, 19 Aug 2023 08:25:09 +0000 Subject: [PATCH 28/46] dns/dnsmessage: show AD and CD bit in Header.GoString() Change-Id: I7b973d255ec4ab1e1c0f8539b811ddc0503c2f48 GitHub-Last-Rev: 954434b6211a6c24d281cda61547070b586ea818 GitHub-Pull-Request: golang/net#188 Reviewed-on: https://go-review.googlesource.com/c/net/+/521075 Run-TryBot: Mateusz Poliwczak TryBot-Result: Gopher Robot Auto-Submit: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor --- dns/dnsmessage/message.go | 2 ++ dns/dnsmessage/message_test.go | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index 37da3de4d3..69938d54ff 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -361,6 +361,8 @@ func (m *Header) GoString() string { "Truncated: " + printBool(m.Truncated) + ", " + "RecursionDesired: " + printBool(m.RecursionDesired) + ", " + "RecursionAvailable: " + printBool(m.RecursionAvailable) + ", " + + "AuthenticData: " + printBool(m.AuthenticData) + ", " + + "CheckingDisabled: " + printBool(m.CheckingDisabled) + ", " + "RCode: " + m.RCode.GoString() + "}" } diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index 64c6db86d1..83fac78128 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1185,8 +1185,7 @@ func TestGoString(t *testing.T) { t.Error("Message.GoString lost information or largeTestMsg changed: msg != largeTestMsg()") } got := msg.GoString() - - want := `dnsmessage.Message{Header: dnsmessage.Header{ID: 0, Response: true, OpCode: 0, Authoritative: true, Truncated: false, RecursionDesired: false, RecursionAvailable: false, RCode: dnsmessage.RCodeSuccess}, Questions: []dnsmessage.Question{dnsmessage.Question{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, Answers: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 1}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 2}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeAAAA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AAAAResource{AAAA: [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeCNAME, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.CNAMEResource{CNAME: dnsmessage.MustNewName("alias.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeSOA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.SOAResource{NS: dnsmessage.MustNewName("ns1.example.com."), MBox: dnsmessage.MustNewName("mb.example.com."), Serial: 1, Refresh: 2, Retry: 3, Expire: 4, MinTTL: 5}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypePTR, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.PTRResource{PTR: dnsmessage.MustNewName("ptr.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeMX, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.MXResource{Pref: 7, MX: dnsmessage.MustNewName("mx.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeSRV, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.SRVResource{Priority: 8, Weight: 9, Port: 11, Target: dnsmessage.MustNewName("srv.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: 65362, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.UnknownResource{Type: 65362, Data: []byte{42, 0, 43, 44}}}}, Authorities: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeNS, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.NSResource{NS: dnsmessage.MustNewName("ns1.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeNS, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.NSResource{NS: dnsmessage.MustNewName("ns2.example.com.")}}}, Additionals: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeTXT, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.TXTResource{TXT: []string{"So Long\x2c and Thanks for All the Fish"}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeTXT, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.TXTResource{TXT: []string{"Hamster Huey and the Gooey Kablooie"}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("."), Type: dnsmessage.TypeOPT, Class: 4096, TTL: 4261412864, Length: 0}, Body: &dnsmessage.OPTResource{Options: []dnsmessage.Option{dnsmessage.Option{Code: 10, Data: []byte{1, 35, 69, 103, 137, 171, 205, 239}}}}}}}` + want := `dnsmessage.Message{Header: dnsmessage.Header{ID: 0, Response: true, OpCode: 0, Authoritative: true, Truncated: false, RecursionDesired: false, RecursionAvailable: false, AuthenticData: false, CheckingDisabled: false, RCode: dnsmessage.RCodeSuccess}, Questions: []dnsmessage.Question{dnsmessage.Question{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, Answers: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 1}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 2}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeAAAA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.AAAAResource{AAAA: [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeCNAME, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.CNAMEResource{CNAME: dnsmessage.MustNewName("alias.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeSOA, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.SOAResource{NS: dnsmessage.MustNewName("ns1.example.com."), MBox: dnsmessage.MustNewName("mb.example.com."), Serial: 1, Refresh: 2, Retry: 3, Expire: 4, MinTTL: 5}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypePTR, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.PTRResource{PTR: dnsmessage.MustNewName("ptr.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeMX, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.MXResource{Pref: 7, MX: dnsmessage.MustNewName("mx.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeSRV, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.SRVResource{Priority: 8, Weight: 9, Port: 11, Target: dnsmessage.MustNewName("srv.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: 65362, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.UnknownResource{Type: 65362, Data: []byte{42, 0, 43, 44}}}}, Authorities: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeNS, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.NSResource{NS: dnsmessage.MustNewName("ns1.example.com.")}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeNS, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.NSResource{NS: dnsmessage.MustNewName("ns2.example.com.")}}}, Additionals: []dnsmessage.Resource{dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeTXT, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.TXTResource{TXT: []string{"So Long\x2c and Thanks for All the Fish"}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("foo.bar.example.com."), Type: dnsmessage.TypeTXT, Class: dnsmessage.ClassINET, TTL: 0, Length: 0}, Body: &dnsmessage.TXTResource{TXT: []string{"Hamster Huey and the Gooey Kablooie"}}}, dnsmessage.Resource{Header: dnsmessage.ResourceHeader{Name: dnsmessage.MustNewName("."), Type: dnsmessage.TypeOPT, Class: 4096, TTL: 4261412864, Length: 0}, Body: &dnsmessage.OPTResource{Options: []dnsmessage.Option{dnsmessage.Option{Code: 10, Data: []byte{1, 35, 69, 103, 137, 171, 205, 239}}}}}}}` if got != want { t.Errorf("got msg1.GoString() = %s\nwant = %s", got, want) From 9cde5a081510f83ae10bc2bf88231babd81ef2d5 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 12 Jul 2023 19:16:51 +0000 Subject: [PATCH 29/46] net/http2: remove awaitGracefulShutdown It was added by https://golang.org/cl/43455 and its usage was removed by https://golang.org/cl/43230 Updates golang/go#20302 Change-Id: I5072c3d9cbf9a33d2ac613bc5a3c059dc54e9d29 GitHub-Last-Rev: 68a32fb702168992427174c41c5d4638f4e567ad GitHub-Pull-Request: golang/net#184 Reviewed-on: https://go-review.googlesource.com/c/net/+/509117 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Reviewed-by: Cherry Mui --- http2/server.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/http2/server.go b/http2/server.go index 033b6e6db6..6d5e008874 100644 --- a/http2/server.go +++ b/http2/server.go @@ -1012,14 +1012,6 @@ func (sc *serverConn) serve() { } } -func (sc *serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, privateCh chan struct{}) { - select { - case <-sc.doneServing: - case <-sharedCh: - close(privateCh) - } -} - type serverMessage int // Message values sent to serveMsgCh. From f89417cca1f18e39ab1db1bb80c42728f99d6143 Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Tue, 22 Aug 2023 07:54:22 +0000 Subject: [PATCH 30/46] dns/dnsmessage: reduce Parser size In the net package the Parser is copied a lot, the size of the Parser can be reduced easily by not storing the entire ResourceHeader in the Parser. It reduces the size from 328B to 80B. Also it makes sure that the resource header parsing methods don't return stale headers (from different sections). Change-Id: If05b03ba654ca5c03d536e86446c5a2a7dc79ec3 GitHub-Last-Rev: dacd25cc355269ff2a89d855d2094bb8f152c83c GitHub-Pull-Request: golang/net#186 Reviewed-on: https://go-review.googlesource.com/c/net/+/514855 Reviewed-by: Matthew Dempsky Auto-Submit: Matthew Dempsky TryBot-Result: Gopher Robot Run-TryBot: Mateusz Poliwczak Run-TryBot: Damien Neil Reviewed-by: Damien Neil --- dns/dnsmessage/message.go | 69 +++++++++++--------- dns/dnsmessage/message_test.go | 114 +++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 32 deletions(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index 69938d54ff..19ea8f17ce 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -542,11 +542,13 @@ type Parser struct { msg []byte header header - section section - off int - index int - resHeaderValid bool - resHeader ResourceHeader + section section + off int + index int + resHeaderValid bool + resHeaderOffset int + resHeaderType Type + resHeaderLength uint16 } // Start parses the header and enables the parsing of Questions. @@ -597,8 +599,9 @@ func (p *Parser) resource(sec section) (Resource, error) { func (p *Parser) resourceHeader(sec section) (ResourceHeader, error) { if p.resHeaderValid { - return p.resHeader, nil + p.off = p.resHeaderOffset } + if err := p.checkAdvance(sec); err != nil { return ResourceHeader{}, err } @@ -608,14 +611,16 @@ func (p *Parser) resourceHeader(sec section) (ResourceHeader, error) { return ResourceHeader{}, err } p.resHeaderValid = true - p.resHeader = hdr + p.resHeaderOffset = p.off + p.resHeaderType = hdr.Type + p.resHeaderLength = hdr.Length p.off = off return hdr, nil } func (p *Parser) skipResource(sec section) error { if p.resHeaderValid { - newOff := p.off + int(p.resHeader.Length) + newOff := p.off + int(p.resHeaderLength) if newOff > len(p.msg) { return errResourceLen } @@ -866,14 +871,14 @@ func (p *Parser) SkipAllAdditionals() error { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) CNAMEResource() (CNAMEResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeCNAME { + if !p.resHeaderValid || p.resHeaderType != TypeCNAME { return CNAMEResource{}, ErrNotStarted } r, err := unpackCNAMEResource(p.msg, p.off) if err != nil { return CNAMEResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -884,14 +889,14 @@ func (p *Parser) CNAMEResource() (CNAMEResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) MXResource() (MXResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeMX { + if !p.resHeaderValid || p.resHeaderType != TypeMX { return MXResource{}, ErrNotStarted } r, err := unpackMXResource(p.msg, p.off) if err != nil { return MXResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -902,14 +907,14 @@ func (p *Parser) MXResource() (MXResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) NSResource() (NSResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeNS { + if !p.resHeaderValid || p.resHeaderType != TypeNS { return NSResource{}, ErrNotStarted } r, err := unpackNSResource(p.msg, p.off) if err != nil { return NSResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -920,14 +925,14 @@ func (p *Parser) NSResource() (NSResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) PTRResource() (PTRResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypePTR { + if !p.resHeaderValid || p.resHeaderType != TypePTR { return PTRResource{}, ErrNotStarted } r, err := unpackPTRResource(p.msg, p.off) if err != nil { return PTRResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -938,14 +943,14 @@ func (p *Parser) PTRResource() (PTRResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) SOAResource() (SOAResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeSOA { + if !p.resHeaderValid || p.resHeaderType != TypeSOA { return SOAResource{}, ErrNotStarted } r, err := unpackSOAResource(p.msg, p.off) if err != nil { return SOAResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -956,14 +961,14 @@ func (p *Parser) SOAResource() (SOAResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) TXTResource() (TXTResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeTXT { + if !p.resHeaderValid || p.resHeaderType != TypeTXT { return TXTResource{}, ErrNotStarted } - r, err := unpackTXTResource(p.msg, p.off, p.resHeader.Length) + r, err := unpackTXTResource(p.msg, p.off, p.resHeaderLength) if err != nil { return TXTResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -974,14 +979,14 @@ func (p *Parser) TXTResource() (TXTResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) SRVResource() (SRVResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeSRV { + if !p.resHeaderValid || p.resHeaderType != TypeSRV { return SRVResource{}, ErrNotStarted } r, err := unpackSRVResource(p.msg, p.off) if err != nil { return SRVResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -992,14 +997,14 @@ func (p *Parser) SRVResource() (SRVResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) AResource() (AResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeA { + if !p.resHeaderValid || p.resHeaderType != TypeA { return AResource{}, ErrNotStarted } r, err := unpackAResource(p.msg, p.off) if err != nil { return AResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -1010,14 +1015,14 @@ func (p *Parser) AResource() (AResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) AAAAResource() (AAAAResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeAAAA { + if !p.resHeaderValid || p.resHeaderType != TypeAAAA { return AAAAResource{}, ErrNotStarted } r, err := unpackAAAAResource(p.msg, p.off) if err != nil { return AAAAResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -1028,14 +1033,14 @@ func (p *Parser) AAAAResource() (AAAAResource, error) { // One of the XXXHeader methods must have been called before calling this // method. func (p *Parser) OPTResource() (OPTResource, error) { - if !p.resHeaderValid || p.resHeader.Type != TypeOPT { + if !p.resHeaderValid || p.resHeaderType != TypeOPT { return OPTResource{}, ErrNotStarted } - r, err := unpackOPTResource(p.msg, p.off, p.resHeader.Length) + r, err := unpackOPTResource(p.msg, p.off, p.resHeaderLength) if err != nil { return OPTResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil @@ -1049,11 +1054,11 @@ func (p *Parser) UnknownResource() (UnknownResource, error) { if !p.resHeaderValid { return UnknownResource{}, ErrNotStarted } - r, err := unpackUnknownResource(p.resHeader.Type, p.msg, p.off, p.resHeader.Length) + r, err := unpackUnknownResource(p.resHeaderType, p.msg, p.off, p.resHeaderLength) if err != nil { return UnknownResource{}, err } - p.off += int(p.resHeader.Length) + p.off += int(p.resHeaderLength) p.resHeaderValid = false p.index++ return r, nil diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index 83fac78128..ddb062b1e3 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1670,3 +1670,117 @@ func FuzzUnpackPack(f *testing.F) { } }) } + +func TestParseResourceHeaderMultipleTimes(t *testing.T) { + msg := Message{ + Header: Header{Response: true, Authoritative: true}, + Answers: []Resource{ + { + ResourceHeader{ + Name: MustNewName("go.dev."), + Type: TypeA, + Class: ClassINET, + }, + &AResource{[4]byte{127, 0, 0, 1}}, + }, + }, + Authorities: []Resource{ + { + ResourceHeader{ + Name: MustNewName("go.dev."), + Type: TypeA, + Class: ClassINET, + }, + &AResource{[4]byte{127, 0, 0, 1}}, + }, + }, + } + + raw, err := msg.Pack() + if err != nil { + t.Fatal(err) + } + + var p Parser + + if _, err := p.Start(raw); err != nil { + t.Fatal(err) + } + + if err := p.SkipAllQuestions(); err != nil { + t.Fatal(err) + } + + hdr1, err := p.AnswerHeader() + if err != nil { + t.Fatal(err) + } + + hdr2, err := p.AnswerHeader() + if err != nil { + t.Fatal(err) + } + + if hdr1 != hdr2 { + t.Fatal("AnswerHeader called multiple times without parsing the RData returned different headers") + } + + if _, err := p.AResource(); err != nil { + t.Fatal(err) + } + + if _, err := p.AnswerHeader(); err != ErrSectionDone { + t.Fatalf("unexpected error: %v, want: %v", err, ErrSectionDone) + } + + hdr3, err := p.AuthorityHeader() + if err != nil { + t.Fatal(err) + } + + hdr4, err := p.AuthorityHeader() + if err != nil { + t.Fatal(err) + } + + if hdr3 != hdr4 { + t.Fatal("AuthorityHeader called multiple times without parsing the RData returned different headers") + } + + if _, err := p.AResource(); err != nil { + t.Fatal(err) + } + + if _, err := p.AuthorityHeader(); err != ErrSectionDone { + t.Fatalf("unexpected error: %v, want: %v", err, ErrSectionDone) + } +} + +func TestParseDifferentResourceHeadersWithoutParsingRData(t *testing.T) { + msg := smallTestMsg() + raw, err := msg.Pack() + if err != nil { + t.Fatal(err) + } + + var p Parser + if _, err := p.Start(raw); err != nil { + t.Fatal(err) + } + + if err := p.SkipAllQuestions(); err != nil { + t.Fatal(err) + } + + if _, err := p.AnswerHeader(); err != nil { + t.Fatal(err) + } + + if _, err := p.AdditionalHeader(); err == nil { + t.Errorf("p.AdditionalHeader() unexpected success") + } + + if _, err := p.AuthorityHeader(); err == nil { + t.Errorf("p.AuthorityHeader() unexpected success") + } +} From 0f7767ccf469d91c5c628723ad5971768b33b981 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 22 Aug 2023 14:20:27 -0700 Subject: [PATCH 31/46] dns/dnsmessage: validate cached section when skipping sections When skipping a section when p.resHeaderValid is set, verify that the cached resource header is for the right section. Fixes golang/go#62220 Change-Id: I8731dfdb5ad3cca94221b58f8be830bd2e16cff3 Reviewed-on: https://go-review.googlesource.com/c/net/+/521995 Reviewed-by: Mateusz Poliwczak Reviewed-by: Ian Lance Taylor Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- dns/dnsmessage/message.go | 2 +- dns/dnsmessage/message_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index 19ea8f17ce..cd997bab07 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -619,7 +619,7 @@ func (p *Parser) resourceHeader(sec section) (ResourceHeader, error) { } func (p *Parser) skipResource(sec section) error { - if p.resHeaderValid { + if p.resHeaderValid && p.section == sec { newOff := p.off + int(p.resHeaderLength) if newOff > len(p.msg) { return errResourceLen diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index ddb062b1e3..1b7f3cb35a 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1784,3 +1784,32 @@ func TestParseDifferentResourceHeadersWithoutParsingRData(t *testing.T) { t.Errorf("p.AuthorityHeader() unexpected success") } } + +func TestParseWrongSection(t *testing.T) { + msg := smallTestMsg() + raw, err := msg.Pack() + if err != nil { + t.Fatal(err) + } + + var p Parser + if _, err := p.Start(raw); err != nil { + t.Fatal(err) + } + + if err := p.SkipAllQuestions(); err != nil { + t.Fatalf("p.SkipAllQuestions() = %v", err) + } + if _, err := p.AnswerHeader(); err != nil { + t.Fatalf("p.AnswerHeader() = %v", err) + } + if _, err := p.AuthorityHeader(); err == nil { + t.Fatalf("p.AuthorityHeader(): unexpected success in Answer section") + } + if err := p.SkipAuthority(); err == nil { + t.Fatalf("p.SkipAuthority(): unexpected success in Answer section") + } + if err := p.SkipAllAuthorities(); err == nil { + t.Fatalf("p.SkipAllAuthorities(): unexpected success in Answer section") + } +} From 3d2be970e8ac4df2e4e5f0dd892c668bafad41cc Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 25 Aug 2023 12:46:28 -0700 Subject: [PATCH 32/46] quic: fix testConn.uncheckedHandshake This test helper was sending the connection-under-test the wrong TLS 1-RTT data: It was resending the handshake-level data rather than the application-level data. This error was hidden by a crypto/tls bug, fixed in CL 522595. Change-Id: Ib672b174ddb1dfa5763f1eb3dd830932a0d26cad Reviewed-on: https://go-review.googlesource.com/c/net/+/522678 Run-TryBot: Damien Neil Reviewed-by: Bryan Mills Auto-Submit: Damien Neil TryBot-Result: Gopher Robot --- internal/quic/tls_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go index 1e3d6b6223..35cb8bf008 100644 --- a/internal/quic/tls_test.go +++ b/internal/quic/tls_test.go @@ -242,6 +242,7 @@ func fillCryptoFrames(d *testDatagram, data map[tls.QUICEncryptionLevel][]byte) // Useful for testing scenarios where configuration has // changed the handshake responses in some way. func (tc *testConn) uncheckedHandshake() { + tc.t.Helper() defer func(saved map[byte]bool) { tc.ignoreFrames = saved }(tc.ignoreFrames) @@ -268,6 +269,7 @@ func (tc *testConn) uncheckedHandshake() { ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, }) } else { + tc.wantIdle("initial frames are ignored") tc.writeFrames(packetTypeInitial, debugFrameCrypto{ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], @@ -285,7 +287,7 @@ func (tc *testConn) uncheckedHandshake() { debugFrameHandshakeDone{}) tc.writeFrames(packetType1RTT, debugFrameCrypto{ - data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + data: tc.cryptoDataIn[tls.QUICEncryptionLevelApplication], }) tc.wantFrame("client ACKs server's first 1-RTT packet", packetType1RTT, debugFrameAck{ From d8d84787ad6422cae430ca5e2455b1e0abf99225 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 9 Aug 2023 13:31:38 -0700 Subject: [PATCH 33/46] quic: read-closing and reset streams, wait on close s.Close waits for the peer to acknowledge receipt of sent data before returning. s.ReadClose closes the receive end of a stream, discarding buffered data and sending a STOP_SENDING frame to the peer. s.Reset(code) closes the send end of a stream with an error, which is sent to the peer in a RESET_STREAM frame. Receipt of a STOP_SENDING frame resets the stream locally and causes future writes to fail. Receipt of a RESET_STREAM frame causes future reads to fail. Stream state is currently retained even after a stream has been completely closed. A future CL will add cleanup. For golang/go#58547 Change-Id: I29088ae570db4079926ad426be6e85dace2122da Reviewed-on: https://go-review.googlesource.com/c/net/+/518435 Run-TryBot: Damien Neil Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot --- internal/quic/conn.go | 21 ++ internal/quic/conn_async_test.go | 46 +++- internal/quic/conn_loss.go | 4 +- internal/quic/conn_loss_test.go | 55 ++++- internal/quic/conn_recv.go | 30 ++- internal/quic/errors.go | 8 + internal/quic/stream.go | 253 ++++++++++++++----- internal/quic/stream_test.go | 404 +++++++++++++++++++++++++++++-- internal/quic/wire.go | 5 +- 9 files changed, 732 insertions(+), 94 deletions(-) diff --git a/internal/quic/conn.go b/internal/quic/conn.go index 0952a79e81..ee8f011f8a 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -73,6 +73,7 @@ type connTestHooks interface { handleTLSEvent(tls.QUICEvent) newConnID(seq int64) ([]byte, error) waitAndLockGate(ctx context.Context, g *gate) error + waitOnDone(ctx context.Context, ch <-chan struct{}) error } func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, config *Config, l connListener, hooks connTestHooks) (*Conn, error) { @@ -311,6 +312,26 @@ func (c *Conn) waitAndLockGate(ctx context.Context, g *gate) error { return g.waitAndLockContext(ctx) } +func (c *Conn) waitOnDone(ctx context.Context, ch <-chan struct{}) error { + if c.testHooks != nil { + return c.testHooks.waitOnDone(ctx, ch) + } + // Check the channel before the context. + // We always prefer to return results when available, + // even when provided with an already-canceled context. + select { + case <-ch: + return nil + default: + } + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + } + return nil +} + // abort terminates a connection with an error. func (c *Conn) abort(now time.Time, err error) { if c.errForPeer == nil { diff --git a/internal/quic/conn_async_test.go b/internal/quic/conn_async_test.go index 2078325a53..0da3ddb452 100644 --- a/internal/quic/conn_async_test.go +++ b/internal/quic/conn_async_test.go @@ -82,10 +82,11 @@ func (a *asyncOp[T]) result() (v T, err error) { } // A blockedAsync is a blocked async operation. -// -// Currently, the only type of blocked operation is one waiting on a gate. type blockedAsync struct { - g *gate + // Exactly one of these will be set, depending on the type of blocked operation. + g *gate + ch <-chan struct{} + donec chan struct{} // closed when the operation is unblocked } @@ -133,6 +134,25 @@ func (as *asyncTestState) waitAndLockGate(ctx context.Context, g *gate) error { // Gate can be acquired without blocking. return nil } + return as.block(ctx, &blockedAsync{ + g: g, + }) +} + +// waitOnDone replaces receiving from a chan struct{} in tests. +func (as *asyncTestState) waitOnDone(ctx context.Context, ch <-chan struct{}) error { + select { + case <-ch: + return nil // read without blocking + default: + } + return as.block(ctx, &blockedAsync{ + ch: ch, + }) +} + +// block waits for a blocked async operation to complete. +func (as *asyncTestState) block(ctx context.Context, b *blockedAsync) error { if err := ctx.Err(); err != nil { // Context has already expired. return err @@ -144,12 +164,9 @@ func (as *asyncTestState) waitAndLockGate(ctx context.Context, g *gate) error { // which may have unpredictable results. panic("blocking async point with unexpected Context") } + b.donec = make(chan struct{}) // Record this as a pending blocking operation. as.mu.Lock() - b := &blockedAsync{ - g: g, - donec: make(chan struct{}), - } as.blocked[b] = struct{}{} as.mu.Unlock() // Notify the creator of the operation that we're blocked, @@ -169,8 +186,19 @@ func (as *asyncTestState) wakeAsync() bool { as.mu.Lock() var woken *blockedAsync for w := range as.blocked { - if w.g.lockIfSet() { - woken = w + switch { + case w.g != nil: + if w.g.lockIfSet() { + woken = w + } + case w.ch != nil: + select { + case <-w.ch: + woken = w + default: + } + } + if woken != nil { delete(as.blocked, woken) break } diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go index f42f7e5282..103db9fa41 100644 --- a/internal/quic/conn_loss.go +++ b/internal/quic/conn_loss.go @@ -44,7 +44,9 @@ func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetF case frameTypeCrypto: start, end := sent.nextRange() c.crypto[space].ackOrLoss(start, end, fate) - case frameTypeMaxStreamData, + case frameTypeResetStream, + frameTypeStopSending, + frameTypeMaxStreamData, frameTypeStreamDataBlocked: id := streamID(sent.nextInt()) s := c.streamForID(id) diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index d9445150a3..dc0dc6cd33 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -75,7 +75,58 @@ func (tc *testConn) triggerLossOrPTO(ptype packetType, pto bool) { }) } -func TestLostCRYPTOFrame(t *testing.T) { +func TestLostResetStreamFrame(t *testing.T) { + // "Cancellation of stream transmission, as carried in a RESET_STREAM frame, + // is sent until acknowledged or until all stream data is acknowledged by the peer [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.4 + lostFrameTest(t, func(t *testing.T, pto bool) { + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters) + tc.ignoreFrame(frameTypeAck) + + s.Reset(1) + tc.wantFrame("reset stream", + packetType1RTT, debugFrameResetStream{ + id: s.id, + code: 1, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resent RESET_STREAM frame", + packetType1RTT, debugFrameResetStream{ + id: s.id, + code: 1, + }) + }) +} + +func TestLostStopSendingFrame(t *testing.T) { + // "[...] a request to cancel stream transmission, as encoded in a STOP_SENDING frame, + // is sent until the receiving part of the stream enters either a "Data Recvd" or + // "Reset Recvd" state [...]" + // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.5 + // + // Technically, we can stop sending a STOP_SENDING frame if the peer sends + // us all the data for the stream or resets it. We don't bother tracking this, + // however, so we'll keep sending the frame until it is acked. This is harmless. + lostFrameTest(t, func(t *testing.T, pto bool) { + tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, permissiveTransportParameters) + tc.ignoreFrame(frameTypeAck) + + s.CloseRead() + tc.wantFrame("stream is read-closed", + packetType1RTT, debugFrameStopSending{ + id: s.id, + }) + + tc.triggerLossOrPTO(packetType1RTT, pto) + tc.wantFrame("resent STOP_SENDING frame", + packetType1RTT, debugFrameStopSending{ + id: s.id, + }) + }) +} + +func TestLostCryptoFrame(t *testing.T) { // "Data sent in CRYPTO frames is retransmitted [...] until all data has been acknowledged." // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.1 lostFrameTest(t, func(t *testing.T, pto bool) { @@ -176,7 +227,7 @@ func TestLostStreamWithData(t *testing.T) { off: 4, data: data[4:8], }) - s.Close() + s.CloseWrite() tc.wantFrame("send FIN", packetType1RTT, debugFrameStream{ id: s.id, diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 00985b6703..e0a91ab00c 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -161,12 +161,12 @@ func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, if !frameOK(c, ptype, __01) { return } - _, _, _, n = consumeResetStreamFrame(payload) + n = c.handleResetStreamFrame(now, space, payload) case frameTypeStopSending: if !frameOK(c, ptype, __01) { return } - _, _, n = consumeStopSendingFrame(payload) + n = c.handleStopSendingFrame(now, space, payload) case frameTypeCrypto: if !frameOK(c, ptype, IH_1) { return @@ -291,6 +291,32 @@ func (c *Conn) handleMaxStreamDataFrame(now time.Time, payload []byte) int { return n } +func (c *Conn) handleResetStreamFrame(now time.Time, space numberSpace, payload []byte) int { + id, code, finalSize, n := consumeResetStreamFrame(payload) + if n < 0 { + return -1 + } + if s := c.streamForFrame(now, id, recvStream); s != nil { + if err := s.handleReset(code, finalSize); err != nil { + c.abort(now, err) + } + } + return n +} + +func (c *Conn) handleStopSendingFrame(now time.Time, space numberSpace, payload []byte) int { + id, code, n := consumeStopSendingFrame(payload) + if n < 0 { + return -1 + } + if s := c.streamForFrame(now, id, sendStream); s != nil { + if err := s.handleStopSending(code); err != nil { + c.abort(now, err) + } + } + return n +} + func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byte) int { off, data, n := consumeCryptoFrame(payload) err := c.handleCrypto(now, space, off, data) diff --git a/internal/quic/errors.go b/internal/quic/errors.go index 55d32f3106..f156859325 100644 --- a/internal/quic/errors.go +++ b/internal/quic/errors.go @@ -99,6 +99,14 @@ func (e peerTransportError) Error() string { return fmt.Sprintf("peer closed connection: %v: %q", e.code, e.reason) } +// A StreamErrorCode is an application protocol error code (RFC 9000, Section 20.2) +// indicating whay a stream is being closed. +type StreamErrorCode uint64 + +func (e StreamErrorCode) Error() string { + return fmt.Sprintf("stream error code %v", uint64(e)) +} + // An ApplicationError is an application protocol error code (RFC 9000, Section 20.2). // Application protocol errors may be sent when terminating a stream or connection. type ApplicationError struct { diff --git a/internal/quic/stream.go b/internal/quic/stream.go index 83215dfd3e..12117dbd39 100644 --- a/internal/quic/stream.go +++ b/internal/quic/stream.go @@ -9,6 +9,7 @@ package quic import ( "context" "errors" + "fmt" "io" ) @@ -20,28 +21,33 @@ type Stream struct { // // The gate condition is set if a read from the stream will not block, // either because the stream has available data or because the read will fail. - ingate gate - in pipe // received data - inwin int64 // last MAX_STREAM_DATA sent to the peer - insendmax sentVal // set when we should send MAX_STREAM_DATA to the peer - inmaxbuf int64 // maximum amount of data we will buffer - insize int64 // stream final size; -1 before this is known - inset rangeset[int64] // received ranges + ingate gate + in pipe // received data + inwin int64 // last MAX_STREAM_DATA sent to the peer + insendmax sentVal // set when we should send MAX_STREAM_DATA to the peer + inmaxbuf int64 // maximum amount of data we will buffer + insize int64 // stream final size; -1 before this is known + inset rangeset[int64] // received ranges + inclosed sentVal // set by CloseRead + inresetcode int64 // RESET_STREAM code received from the peer; -1 if not reset // outgate's lock guards all send-related state. // // The gate condition is set if a write to the stream will not block, // either because the stream has available flow control or because // the write will fail. - outgate gate - out pipe // buffered data to send - outwin int64 // maximum MAX_STREAM_DATA received from the peer - outmaxbuf int64 // maximum amount of data we will buffer - outunsent rangeset[int64] // ranges buffered but not yet sent - outacked rangeset[int64] // ranges sent and acknowledged - outopened sentVal // set if we should open the stream - outclosed sentVal // set by CloseWrite - outblocked sentVal // set when a write to the stream is blocked by flow control + outgate gate + out pipe // buffered data to send + outwin int64 // maximum MAX_STREAM_DATA received from the peer + outmaxbuf int64 // maximum amount of data we will buffer + outunsent rangeset[int64] // ranges buffered but not yet sent + outacked rangeset[int64] // ranges sent and acknowledged + outopened sentVal // set if we should open the stream + outclosed sentVal // set by CloseWrite + outblocked sentVal // set when a write to the stream is blocked by flow control + outreset sentVal // set by Reset + outresetcode uint64 // reset code to send in RESET_STREAM + outdone chan struct{} // closed when all data sent prev, next *Stream // guarded by streamsState.sendMu } @@ -54,11 +60,13 @@ type Stream struct { // unlocking outgate will set the stream writability state.) func newStream(c *Conn, id streamID) *Stream { s := &Stream{ - conn: c, - id: id, - insize: -1, // -1 indicates the stream size is unknown - ingate: newLockedGate(), - outgate: newLockedGate(), + conn: c, + id: id, + insize: -1, // -1 indicates the stream size is unknown + inresetcode: -1, // -1 indicates no RESET_STREAM received + ingate: newLockedGate(), + outgate: newLockedGate(), + outdone: make(chan struct{}), } return s } @@ -87,7 +95,8 @@ func (s *Stream) Read(b []byte) (n int, err error) { // // If the peer closes the stream cleanly, ReadContext returns io.EOF after // returning all data sent by the peer. -// If the peer terminates reads abruptly, ReadContext returns StreamResetError. +// If the peer aborts reads on the stream, ReadContext returns +// an error wrapping StreamResetCode. func (s *Stream) ReadContext(ctx context.Context, b []byte) (n int, err error) { if s.IsWriteOnly() { return 0, errors.New("read from write-only stream") @@ -97,6 +106,12 @@ func (s *Stream) ReadContext(ctx context.Context, b []byte) (n int, err error) { return 0, err } defer s.inUnlock() + if s.inresetcode != -1 { + return 0, fmt.Errorf("stream reset by peer: %w", StreamErrorCode(s.inresetcode)) + } + if s.inclosed.isSet() { + return 0, errors.New("read from closed stream") + } if s.insize == s.in.start { return 0, io.EOF } @@ -145,26 +160,17 @@ func (s *Stream) Write(b []byte) (n int, err error) { // Buffered data is only sent when the buffer is sufficiently full. // Call the Flush method to ensure buffered data is sent. // -// If the peer aborts reads on the stream, ReadContext returns StreamResetError. +// TODO: Implement Flush. func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) { if s.IsReadOnly() { return 0, errors.New("write to read-only stream") } canWrite := s.outgate.lock() - if s.outclosed.isSet() { - s.outUnlock() - return 0, errors.New("write to closed stream") - } - if len(b) == 0 { - // We aren't writing any data, but send a STREAM frame to open the stream - // if we haven't done so already. - s.outopened.set() - } - for len(b) > 0 { + for { // The first time through this loop, we may or may not be write blocked. // We exit the loop after writing all data, so on subsequent passes through // the loop we are always write blocked. - if !canWrite { + if len(b) > 0 && !canWrite { // We're blocked, either by flow control or by our own buffer limit. // We either need the peer to extend our flow control window, // or ack some of our outstanding packets. @@ -181,6 +187,21 @@ func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) // write blocked. (Unlike traditional condition variables, gates do not // have spurious wakeups.) } + if s.outreset.isSet() { + s.outUnlock() + return n, errors.New("write to reset stream") + } + if s.outclosed.isSet() { + s.outUnlock() + return n, errors.New("write to closed stream") + } + // We set outopened here rather than below, + // so if this is a zero-length write we still + // open the stream despite not writing any data to it. + s.outopened.set() + if len(b) == 0 { + break + } s.outblocked.clear() // Write limit is min(our own buffer limit, the peer-provided flow control window). // This is a stream offset. @@ -191,7 +212,6 @@ func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) // Copy the data into the output buffer and mark it as unsent. s.outunsent.add(s.out.end, s.out.end+nn) s.out.writeAt(b[:nn], s.out.end) - s.outopened.set() b = b[nn:] n += int(nn) // If we have bytes left to send, we're blocked. @@ -218,9 +238,8 @@ func (s *Stream) Close() error { func (s *Stream) CloseContext(ctx context.Context) error { s.CloseRead() s.CloseWrite() - // TODO: wait for peer to acknowledge data // TODO: Return code from peer's RESET_STREAM frame? - return nil + return s.conn.waitOnDone(ctx, s.outdone) } // CloseRead aborts reads on the stream. @@ -233,7 +252,17 @@ func (s *Stream) CloseRead() { if s.IsWriteOnly() { return } - // TODO: support read-closing streams with a STOP_SENDING frame + s.ingate.lock() + defer s.inUnlock() + if s.inset.isrange(0, s.insize) || s.inresetcode != -1 { + // We've already received all data from the peer, + // so there's no need to send STOP_SENDING. + // This is the same as saying we sent one and they got it. + s.inclosed.setReceived() + } else { + s.inclosed.set() + } + s.in.discardBefore(s.in.end) } // CloseWrite aborts writes on the stream. @@ -251,6 +280,29 @@ func (s *Stream) CloseWrite() { s.outclosed.set() } +// Reset aborts writes on the stream and notifies the peer +// that the stream was terminated abruptly. +// Any blocked writes will be unblocked and return errors. +// +// Reset sends the application protocol error code to the peer. +// It does not wait for the peer to acknowledge receipt of the error. +// Use CloseContext to wait for the peer's acknowledgement. +func (s *Stream) Reset(code uint64) { + s.outgate.lock() + defer s.outUnlock() + if s.outreset.isSet() { + return + } + // We could check here to see if the stream is closed and the + // peer has acked all the data and the FIN, but sending an + // extra RESET_STREAM in this case is harmless. + s.outreset.set() + s.outresetcode = code + s.out.discardBefore(s.out.end) + s.outunsent = rangeset[int64]{} + s.outblocked.clear() +} + // inUnlock unlocks s.ingate. // It sets the gate condition if reads from s will not block. // If s has receive-related frames to write, it notifies the Conn. @@ -263,11 +315,13 @@ func (s *Stream) inUnlock() { // inUnlockNoQueue is inUnlock, // but reports whether s has frames to write rather than notifying the Conn. func (s *Stream) inUnlockNoQueue() (shouldSend bool) { - // TODO: STOP_SENDING canRead := s.inset.contains(s.in.start) || // data available to read - s.insize == s.in.start // at EOF - s.ingate.unlock(canRead) - return s.insendmax.shouldSend() // STREAM_MAX_DATA + s.insize == s.in.start || // at EOF + s.inresetcode != -1 || // reset by peer + s.inclosed.isSet() // closed locally + defer s.ingate.unlock(canRead) + return s.insendmax.shouldSend() || // STREAM_MAX_DATA + s.inclosed.shouldSend() // STOP_SENDING } // outUnlock unlocks s.outgate. @@ -282,10 +336,24 @@ func (s *Stream) outUnlock() { // outUnlockNoQueue is outUnlock, // but reports whether s has frames to write rather than notifying the Conn. func (s *Stream) outUnlockNoQueue() (shouldSend bool) { + isDone := s.outclosed.isReceived() && s.outacked.isrange(0, s.out.end) || // all data acked + s.outreset.isSet() // reset locally + if isDone { + select { + case <-s.outdone: + default: + close(s.outdone) + } + } lim := min(s.out.start+s.outmaxbuf, s.outwin) canWrite := lim > s.out.end || // available flow control - s.outclosed.isSet() // closed - s.outgate.unlock(canWrite) + s.outclosed.isSet() || // closed locally + s.outreset.isSet() // reset locally + defer s.outgate.unlock(canWrite) + if s.outreset.isSet() { + // If the stream is reset locally, the only frame we'll send is RESET_STREAM. + return s.outreset.shouldSend() + } return len(s.outunsent) > 0 || // STREAM frame with data s.outclosed.shouldSend() || // STREAM frame with FIN bit s.outopened.shouldSend() || // STREAM frame with no data @@ -297,21 +365,17 @@ func (s *Stream) handleData(off int64, b []byte, fin bool) error { s.ingate.lock() defer s.inUnlock() end := off + int64(len(b)) - if end > s.inwin { - // The peer sent us data past the maximum flow control window we gave them. - return localTransportError(errFlowControl) + if err := s.checkStreamBounds(end, fin); err != nil { + return err } - if s.insize != -1 && end > s.insize { - // The peer sent us data past the final size of the stream they previously gave us. - return localTransportError(errFinalSize) + if s.inclosed.isSet() || s.inresetcode != -1 { + // The user read-closed the stream, or the peer reset it. + // Either way, we can discard this frame. + return nil } s.in.writeAt(b, off) s.inset.add(off, end) if fin { - if s.insize != -1 && s.insize != end { - // The peer changed the final size of the stream. - return localTransportError(errFinalSize) - } s.insize = end // The peer has enough flow control window to send the entire stream. s.insendmax.clear() @@ -319,6 +383,53 @@ func (s *Stream) handleData(off int64, b []byte, fin bool) error { return nil } +// handleReset handles a RESET_STREAM frame. +func (s *Stream) handleReset(code uint64, finalSize int64) error { + s.ingate.lock() + defer s.inUnlock() + const fin = true + if err := s.checkStreamBounds(finalSize, fin); err != nil { + return err + } + if s.inresetcode != -1 { + // The stream was already reset. + return nil + } + s.in.discardBefore(s.in.end) + s.inresetcode = int64(code) + s.insize = finalSize + return nil +} + +// checkStreamBounds validates the stream offset in a STREAM or RESET_STREAM frame. +func (s *Stream) checkStreamBounds(end int64, fin bool) error { + if end > s.inwin { + // The peer sent us data past the maximum flow control window we gave them. + return localTransportError(errFlowControl) + } + if s.insize != -1 && end > s.insize { + // The peer sent us data past the final size of the stream they previously gave us. + return localTransportError(errFinalSize) + } + if fin && s.insize != -1 && end != s.insize { + // The peer changed the final size of the stream. + return localTransportError(errFinalSize) + } + if fin && end < s.in.end { + // The peer has previously sent us data past the final size. + return localTransportError(errFinalSize) + } + return nil +} + +// handleStopSending handles a STOP_SENDING frame. +func (s *Stream) handleStopSending(code uint64) error { + // Peer requests that we reset this stream. + // https://www.rfc-editor.org/rfc/rfc9000#section-3.5-4 + s.Reset(code) + return nil +} + // handleMaxStreamData handles an update received in a MAX_STREAM_DATA frame. func (s *Stream) handleMaxStreamData(maxStreamData int64) error { s.outgate.lock() @@ -336,6 +447,14 @@ func (s *Stream) ackOrLoss(pnum packetNumber, ftype byte, fate packetFate) { // Frames which are always the same (STOP_SENDING, RESET_STREAM) // can be marked as received if any packet carrying this frame is acked. switch ftype { + case frameTypeResetStream: + s.outgate.lock() + s.outreset.ackOrLoss(pnum, fate) + s.outUnlock() + case frameTypeStopSending: + s.ingate.lock() + s.inclosed.ackOrLoss(pnum, fate) + s.inUnlock() case frameTypeMaxStreamData: s.ingate.lock() s.insendmax.ackLatestOrLoss(pnum, fate) @@ -345,7 +464,6 @@ func (s *Stream) ackOrLoss(pnum packetNumber, ftype byte, fate packetFate) { s.outblocked.ackLatestOrLoss(pnum, fate) s.outUnlock() default: - // TODO: Handle STOP_SENDING, RESET_STREAM. panic("unhandled frame type") } } @@ -358,6 +476,10 @@ func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fa if fin { s.outclosed.ackOrLoss(pnum, fate) } + if s.outreset.isSet() { + // If the stream has been reset, we don't care any more. + return + } switch fate { case packetAcked: s.outacked.add(start, end) @@ -385,6 +507,15 @@ func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fa func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bool { s.ingate.lock() defer s.inUnlockNoQueue() + if s.inclosed.shouldSendPTO(pto) { + // We don't currently have an API for setting the error code. + // Just send zero. + code := uint64(0) + if !w.appendStopSendingFrame(s.id, code) { + return false + } + s.inclosed.setSent(pnum) + } // TODO: STOP_SENDING if s.insendmax.shouldSendPTO(pto) { // MAX_STREAM_DATA @@ -406,7 +537,17 @@ func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bo func (s *Stream) appendOutFrames(w *packetWriter, pnum packetNumber, pto bool) bool { s.outgate.lock() defer s.outUnlockNoQueue() - // TODO: RESET_STREAM + if s.outreset.isSet() { + // RESET_STREAM + if s.outreset.shouldSendPTO(pto) { + if !w.appendResetStreamFrame(s.id, s.outresetcode, s.out.end) { + return false + } + s.outreset.setSent(pnum) + s.frameOpensStream(pnum) + } + return true + } if s.outblocked.shouldSendPTO(pto) { // STREAM_DATA_BLOCKED if !w.appendStreamDataBlockedFrame(s.id, s.out.end) { diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index d158e72afb..5904a93428 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "crypto/rand" + "errors" "fmt" "io" "reflect" @@ -489,32 +490,76 @@ func TestStreamReceiveDuplicateDataDoesNotViolateLimits(t *testing.T) { }) } -func TestStreamFinalSizeChangedByStreamFrame(t *testing.T) { - // "If a [...] STREAM frame is received indicating a change - // in the final size for the stream, an endpoint SHOULD - // respond with an error of type FINAL_SIZE_ERROR [...]" - // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5 +func finalSizeTest(t *testing.T, wantErr transportError, f func(tc *testConn, sid streamID) (finalSize int64), opts ...any) { testStreamTypes(t, "", func(t *testing.T, styp streamType) { - tc := newTestConn(t, serverSide) - tc.handshake() - sid := newStreamID(clientSide, styp, 0) + for _, test := range []struct { + name string + finalFrame func(tc *testConn, sid streamID, finalSize int64) + }{{ + name: "FIN", + finalFrame: func(tc *testConn, sid streamID, finalSize int64) { + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: sid, + off: finalSize, + fin: true, + }) + }, + }, { + name: "RESET_STREAM", + finalFrame: func(tc *testConn, sid streamID, finalSize int64) { + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: sid, + finalSize: finalSize, + }) + }, + }} { + t.Run(test.name, func(t *testing.T) { + tc := newTestConn(t, serverSide, opts...) + tc.handshake() + sid := newStreamID(clientSide, styp, 0) + finalSize := f(tc, sid) + test.finalFrame(tc, sid, finalSize) + tc.wantFrame("change in final size of stream is an error", + packetType1RTT, debugFrameConnectionCloseTransport{ + code: wantErr, + }, + ) + }) + } + }) +} - const write1size = 4 +func TestStreamFinalSizeChangedAfterFin(t *testing.T) { + // "If a RESET_STREAM or STREAM frame is received indicating a change + // in the final size for the stream, an endpoint SHOULD respond with + // an error of type FINAL_SIZE_ERROR [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5 + finalSizeTest(t, errFinalSize, func(tc *testConn, sid streamID) (finalSize int64) { tc.writeFrames(packetType1RTT, debugFrameStream{ id: sid, off: 10, fin: true, }) + return 9 + }) +} + +func TestStreamFinalSizeBeforePreviousData(t *testing.T) { + finalSizeTest(t, errFinalSize, func(tc *testConn, sid streamID) (finalSize int64) { tc.writeFrames(packetType1RTT, debugFrameStream{ - id: sid, - off: 9, - fin: true, + id: sid, + off: 10, + data: []byte{0}, }) - tc.wantFrame("change in final size of stream is an error", - packetType1RTT, debugFrameConnectionCloseTransport{ - code: errFinalSize, - }, - ) + return 9 + }) +} + +func TestStreamFinalSizePastMaxStreamData(t *testing.T) { + finalSizeTest(t, errFlowControl, func(tc *testConn, sid streamID) (finalSize int64) { + return 11 + }, func(c *Config) { + c.StreamReadBufferSize = 10 }) } @@ -637,6 +682,19 @@ func testStreamSendFrameInvalidState(t *testing.T, f func(sid streamID) debugFra }) } +func TestStreamResetStreamInvalidState(t *testing.T) { + // "An endpoint that receives a RESET_STREAM frame for a send-only + // stream MUST terminate the connection with error STREAM_STATE_ERROR." + // https://www.rfc-editor.org/rfc/rfc9000#section-19.4-3 + testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame { + return debugFrameResetStream{ + id: sid, + code: 0, + finalSize: 0, + } + }) +} + func TestStreamStreamFrameInvalidState(t *testing.T) { // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR // if it receives a STREAM frame for a locally initiated stream @@ -689,6 +747,20 @@ func testStreamReceiveFrameInvalidState(t *testing.T, f func(sid streamID) debug }) } +func TestStreamStopSendingInvalidState(t *testing.T) { + // "Receiving a STOP_SENDING frame for a locally initiated stream + // that has not yet been created MUST be treated as a connection error + // of type STREAM_STATE_ERROR. An endpoint that receives a STOP_SENDING + // frame for a receive-only stream MUST terminate the connection with + // error STREAM_STATE_ERROR." + // https://www.rfc-editor.org/rfc/rfc9000#section-19.5-2 + testStreamReceiveFrameInvalidState(t, func(sid streamID) debugFrame { + return debugFrameStopSending{ + id: sid, + } + }) +} + func TestStreamMaxStreamDataInvalidState(t *testing.T) { // "Receiving a MAX_STREAM_DATA frame for a locally initiated stream // that has not yet been created MUST be treated as a connection error @@ -743,13 +815,47 @@ func TestStreamWriteToReadOnlyStream(t *testing.T) { } } -func TestStreamWriteToClosedStream(t *testing.T) { - tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, func(p *transportParameters) { - p.initialMaxStreamsBidi = 1 - p.initialMaxData = 1 << 20 - p.initialMaxStreamDataBidiRemote = 1 << 20 +func TestStreamReadFromClosedStream(t *testing.T) { + tc, s := newTestConnAndRemoteStream(t, serverSide, bidiStream, permissiveTransportParameters) + s.CloseRead() + tc.wantFrame("CloseRead sends a STOP_SENDING frame", + packetType1RTT, debugFrameStopSending{ + id: s.id, + }) + wantErr := "read from closed stream" + if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr) + } + // Data which shows up after STOP_SENDING is discarded. + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + data: []byte{1, 2, 3}, + fin: true, + }) + if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr) + } +} + +func TestStreamCloseReadWithAllDataReceived(t *testing.T) { + tc, s := newTestConnAndRemoteStream(t, serverSide, bidiStream, permissiveTransportParameters) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + data: []byte{1, 2, 3}, + fin: true, }) - s.Close() + s.CloseRead() + tc.wantIdle("CloseRead in Data Recvd state doesn't need to send STOP_SENDING") + // We had all the data for the stream, but CloseRead discarded it. + wantErr := "read from closed stream" + if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr) + } +} + +func TestStreamWriteToClosedStream(t *testing.T) { + tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, permissiveTransportParameters) + s.CloseWrite() tc.wantFrame("stream is opened after being closed", packetType1RTT, debugFrameStream{ id: s.id, @@ -763,6 +869,45 @@ func TestStreamWriteToClosedStream(t *testing.T) { } } +func TestStreamResetBlockedStream(t *testing.T) { + tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, func(p *transportParameters) { + p.initialMaxStreamsBidi = 1 + p.initialMaxData = 1 << 20 + p.initialMaxStreamDataBidiRemote = 4 + }) + tc.ignoreFrame(frameTypeStreamDataBlocked) + writing := runAsync(tc, func(ctx context.Context) (int, error) { + return s.WriteContext(ctx, []byte{0, 1, 2, 3, 4, 5, 6, 7}) + }) + tc.wantFrame("stream writes data until blocked by flow control", + packetType1RTT, debugFrameStream{ + id: s.id, + off: 0, + data: []byte{0, 1, 2, 3}, + }) + s.Reset(42) + tc.wantFrame("stream is reset", + packetType1RTT, debugFrameResetStream{ + id: s.id, + code: 42, + finalSize: 4, + }) + wantErr := "write to reset stream" + if n, err := writing.result(); n != 4 || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Write() interrupted by Reset: %v, %q; want 4, %q", n, err, wantErr) + } + tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{ + id: s.id, + max: 1 << 20, + }) + tc.wantIdle("flow control is available, but stream has been reset") + s.Reset(100) + tc.wantIdle("resetting stream a second time has no effect") + if n, err := s.Write([]byte{}); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr) + } +} + func TestStreamWriteMoreThanOnePacketOfData(t *testing.T) { tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) { p.initialMaxStreamsUni = 1 @@ -797,6 +942,209 @@ func TestStreamWriteMoreThanOnePacketOfData(t *testing.T) { } } +func TestStreamCloseWaitsForAcks(t *testing.T) { + ctx := canceledContext() + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters) + data := make([]byte, 100) + s.WriteContext(ctx, data) + tc.wantFrame("conn sends data for the stream", + packetType1RTT, debugFrameStream{ + id: s.id, + data: data, + }) + if err := s.CloseContext(ctx); err != context.Canceled { + t.Fatalf("s.Close() = %v, want context.Canceled (data not acked yet)", err) + } + tc.wantFrame("conn sends FIN for closed stream", + packetType1RTT, debugFrameStream{ + id: s.id, + off: int64(len(data)), + fin: true, + data: []byte{}, + }) + closing := runAsync(tc, func(ctx context.Context) (struct{}, error) { + return struct{}{}, s.CloseContext(ctx) + }) + if _, err := closing.result(); err != errNotDone { + t.Fatalf("s.CloseContext() = %v, want it to block waiting for acks", err) + } + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, + }) + if _, err := closing.result(); err != nil { + t.Fatalf("s.CloseContext() = %v, want nil (all data acked)", err) + } +} + +func TestStreamCloseUnblocked(t *testing.T) { + for _, test := range []struct { + name string + unblock func(tc *testConn, s *Stream) + }{{ + name: "data received", + unblock: func(tc *testConn, s *Stream) { + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, + }) + }, + }, { + name: "stop sending received", + unblock: func(tc *testConn, s *Stream) { + tc.writeFrames(packetType1RTT, debugFrameStopSending{ + id: s.id, + }) + }, + }, { + name: "stream reset", + unblock: func(tc *testConn, s *Stream) { + s.Reset(0) + tc.wait() // wait for test conn to process the Reset + }, + }} { + t.Run(test.name, func(t *testing.T) { + ctx := canceledContext() + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters) + data := make([]byte, 100) + s.WriteContext(ctx, data) + tc.wantFrame("conn sends data for the stream", + packetType1RTT, debugFrameStream{ + id: s.id, + data: data, + }) + if err := s.CloseContext(ctx); err != context.Canceled { + t.Fatalf("s.Close() = %v, want context.Canceled (data not acked yet)", err) + } + tc.wantFrame("conn sends FIN for closed stream", + packetType1RTT, debugFrameStream{ + id: s.id, + off: int64(len(data)), + fin: true, + data: []byte{}, + }) + closing := runAsync(tc, func(ctx context.Context) (struct{}, error) { + return struct{}{}, s.CloseContext(ctx) + }) + if _, err := closing.result(); err != errNotDone { + t.Fatalf("s.CloseContext() = %v, want it to block waiting for acks", err) + } + test.unblock(tc, s) + if _, err := closing.result(); err != nil { + t.Fatalf("s.CloseContext() = %v, want nil (all data acked)", err) + } + }) + } +} + +func TestStreamPeerResetsWithUnreadAndUnsentData(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + ctx := canceledContext() + tc, s := newTestConnAndRemoteStream(t, serverSide, styp) + data := []byte{0, 1, 2, 3, 4, 5, 6, 7} + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + data: data, + }) + got := make([]byte, 4) + if n, err := s.ReadContext(ctx, got); n != len(got) || err != nil { + t.Fatalf("Read start of stream: got %v, %v; want %v, nil", n, err, len(got)) + } + const sentCode = 42 + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + finalSize: 20, + code: sentCode, + }) + wantErr := StreamErrorCode(sentCode) + if n, err := s.ReadContext(ctx, got); n != 0 || !errors.Is(err, wantErr) { + t.Fatalf("Read reset stream: got %v, %v; want 0, %v", n, err, wantErr) + } + }) +} + +func TestStreamPeerResetWakesBlockedRead(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc, s := newTestConnAndRemoteStream(t, serverSide, styp) + reader := runAsync(tc, func(ctx context.Context) (int, error) { + got := make([]byte, 4) + return s.ReadContext(ctx, got) + }) + const sentCode = 42 + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + finalSize: 20, + code: sentCode, + }) + wantErr := StreamErrorCode(sentCode) + if n, err := reader.result(); n != 0 || !errors.Is(err, wantErr) { + t.Fatalf("Read reset stream: got %v, %v; want 0, %v", n, err, wantErr) + } + }) +} + +func TestStreamPeerResetFollowedByData(t *testing.T) { + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc, s := newTestConnAndRemoteStream(t, serverSide, styp) + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + finalSize: 4, + code: 1, + }) + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + data: []byte{0, 1, 2, 3}, + }) + // Another reset with a different code, for good measure. + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + finalSize: 4, + code: 2, + }) + wantErr := StreamErrorCode(1) + if n, err := s.Read(make([]byte, 16)); n != 0 || !errors.Is(err, wantErr) { + t.Fatalf("Read from reset stream: got %v, %v; want 0, %v", n, err, wantErr) + } + }) +} + +func TestStreamPeerStopSendingForActiveStream(t *testing.T) { + // "An endpoint that receives a STOP_SENDING frame MUST send a RESET_STREAM frame if + // the stream is in the "Ready" or "Send" state." + // https://www.rfc-editor.org/rfc/rfc9000#section-3.5-4 + testStreamTypes(t, "", func(t *testing.T, styp streamType) { + tc, s := newTestConnAndLocalStream(t, serverSide, styp, permissiveTransportParameters) + for i := 0; i < 4; i++ { + s.Write([]byte{byte(i)}) + tc.wantFrame("write sends a STREAM frame to peer", + packetType1RTT, debugFrameStream{ + id: s.id, + off: int64(i), + data: []byte{byte(i)}, + }) + } + tc.writeFrames(packetType1RTT, debugFrameStopSending{ + id: s.id, + code: 42, + }) + tc.wantFrame("receiving STOP_SENDING causes stream reset", + packetType1RTT, debugFrameResetStream{ + id: s.id, + code: 42, + finalSize: 4, + }) + if n, err := s.Write([]byte{0}); err == nil { + t.Errorf("s.Write() after STOP_SENDING = %v, %v; want error", n, err) + } + // This ack will result in some of the previous frames being marked as lost. + tc.writeFrames(packetType1RTT, debugFrameAck{ + ranges: []i64range[packetNumber]{{ + tc.sentFramePacket.num, + tc.sentFramePacket.num + 1, + }}, + }) + tc.wantIdle("lost STREAM frames for reset stream are not resent") + }) +} + func newTestConnAndLocalStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) { t.Helper() ctx := canceledContext() @@ -825,3 +1173,13 @@ func newTestConnAndRemoteStream(t *testing.T, side connSide, styp streamType, op } return tc, s } + +// permissiveTransportParameters may be passed as an option to newTestConn. +func permissiveTransportParameters(p *transportParameters) { + p.initialMaxStreamsBidi = maxVarint + p.initialMaxStreamsUni = maxVarint + p.initialMaxData = maxVarint + p.initialMaxStreamDataBidiRemote = maxVarint + p.initialMaxStreamDataBidiLocal = maxVarint + p.initialMaxStreamDataUni = maxVarint +} diff --git a/internal/quic/wire.go b/internal/quic/wire.go index f0643c9229..8486029151 100644 --- a/internal/quic/wire.go +++ b/internal/quic/wire.go @@ -8,7 +8,10 @@ package quic import "encoding/binary" -const maxVarintSize = 8 +const ( + maxVarintSize = 8 // encoded size in bytes + maxVarint = (1 << 62) - 1 +) // consumeVarint parses a variable-length integer, reporting its length. // It returns a negative length upon an error. From efb8d7ab942d2b798abae11dfebfb9043cac78be Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Sat, 26 Aug 2023 16:55:17 +0000 Subject: [PATCH 34/46] dns/dnsmessage: don't include bytes after name.Length in the compression map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This improves the performance of name compression and makes the name.Data[name.Length:] not included in the compression map, it is unnecessary and might cause issues (i.e. reusing the Name struct, without using the NewName function). goos: linux goarch: amd64 pkg: golang.org/x/net/dns/dnsmessage cpu: Intel(R) Core(TM) i5-4200M CPU @ 2.50GHz │ before │ after │ │ sec/op │ sec/op vs base │ Pack-4 15.672µ ± 13% 5.470µ ± 14% -65.10% (p=0.000 n=10) AppendPack-4 15.144µ ± 12% 5.330µ ± 10% -64.80% (p=0.000 n=10) geomean 15.41µ 5.400µ -64.95% │ before │ after │ │ B/op │ B/op vs base │ Pack-4 6.051Ki ± 0% 1.285Ki ± 0% -78.76% (p=0.000 n=10) AppendPack-4 5684.0 ± 0% 804.0 ± 0% -85.86% (p=0.000 n=10) geomean 5.795Ki 1.005Ki -82.67% │ before │ after │ │ allocs/op │ allocs/op vs base │ Pack-4 21.00 ± 0% 11.00 ± 0% -47.62% (p=0.000 n=10) AppendPack-4 20.00 ± 0% 10.00 ± 0% -50.00% (p=0.000 n=10) geomean 20.49 10.49 -48.82% Change-Id: Idf40d5d4790d37eb7253214f089eff859a937c60 GitHub-Last-Rev: a3182830e27086a0e12e116c7f7916468eb1edf2 GitHub-Pull-Request: golang/net#190 Reviewed-on: https://go-review.googlesource.com/c/net/+/522817 Run-TryBot: Mateusz Poliwczak TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Damien Neil Auto-Submit: Ian Lance Taylor --- dns/dnsmessage/message.go | 11 +++++++++-- dns/dnsmessage/message_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index cd997bab07..9ddf2c2292 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -1961,6 +1961,8 @@ func (n *Name) pack(msg []byte, compression map[string]int, compressionOff int) return append(msg, 0), nil } + var nameAsStr string + // Emit sequence of counted strings, chopping at dots. for i, begin := 0, 0; i < int(n.Length); i++ { // Check for the end of the segment. @@ -1991,7 +1993,7 @@ func (n *Name) pack(msg []byte, compression map[string]int, compressionOff int) // segment. A pointer is two bytes with the two most significant // bits set to 1 to indicate that it is a pointer. if (i == 0 || n.Data[i-1] == '.') && compression != nil { - if ptr, ok := compression[string(n.Data[i:])]; ok { + if ptr, ok := compression[string(n.Data[i:n.Length])]; ok { // Hit. Emit a pointer instead of the rest of // the domain. return append(msg, byte(ptr>>8|0xC0), byte(ptr)), nil @@ -2000,7 +2002,12 @@ func (n *Name) pack(msg []byte, compression map[string]int, compressionOff int) // Miss. Add the suffix to the compression table if the // offset can be stored in the available 14 bytes. if len(msg) <= int(^uint16(0)>>2) { - compression[string(n.Data[i:])] = len(msg) - compressionOff + if nameAsStr == "" { + // allocate n.Data on the heap once, to avoid allocating it + // multiple times (for next labels). + nameAsStr = string(n.Data[:n.Length]) + } + compression[nameAsStr[i:]] = len(msg) - compressionOff } } } diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index 1b7f3cb35a..ee42febbc2 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1813,3 +1813,37 @@ func TestParseWrongSection(t *testing.T) { t.Fatalf("p.SkipAllAuthorities(): unexpected success in Answer section") } } + +func TestBuilderNameCompressionWithNonZeroedName(t *testing.T) { + b := NewBuilder(nil, Header{}) + b.EnableCompression() + if err := b.StartQuestions(); err != nil { + t.Fatalf("b.StartQuestions() unexpected error: %v", err) + } + + name := MustNewName("go.dev.") + if err := b.Question(Question{Name: name}); err != nil { + t.Fatalf("b.Question() unexpected error: %v", err) + } + + // Character that is not part of the name (name.Data[:name.Length]), + // shouldn't affect the compression algorithm. + name.Data[name.Length] = '1' + if err := b.Question(Question{Name: name}); err != nil { + t.Fatalf("b.Question() unexpected error: %v", err) + } + + msg, err := b.Finish() + if err != nil { + t.Fatalf("b.Finish() unexpected error: %v", err) + } + + expect := []byte{ + 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, // header + 2, 'g', 'o', 3, 'd', 'e', 'v', 0, 0, 0, 0, 0, // question 1 + 0xC0, 12, 0, 0, 0, 0, // question 2 + } + if !bytes.Equal(msg, expect) { + t.Fatalf("b.Finish() = %v, want: %v", msg, expect) + } +} From 4a2d37ed365334ff00b166660d7c497fcfeaef1b Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 31 Jul 2023 15:26:38 -0700 Subject: [PATCH 35/46] http2: remove Docker-requiring tests Remove two tests, one of which uses curl and the other which uses h2load. These tests don't seem worth the complexity of keeping around a Dockerfile and curl/h2load dependencies. Change-Id: I0370af061168e46d8110fa40eba8dabe68acecc3 Reviewed-on: https://go-review.googlesource.com/c/net/+/514597 Reviewed-by: Brad Fitzpatrick Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills --- http2/Dockerfile | 51 ------------------------ http2/Makefile | 3 -- http2/http2_test.go | 62 ----------------------------- http2/server_test.go | 92 -------------------------------------------- 4 files changed, 208 deletions(-) delete mode 100644 http2/Dockerfile delete mode 100644 http2/Makefile diff --git a/http2/Dockerfile b/http2/Dockerfile deleted file mode 100644 index 8512245952..0000000000 --- a/http2/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# -# This Dockerfile builds a recent curl with HTTP/2 client support, using -# a recent nghttp2 build. -# -# See the Makefile for how to tag it. If Docker and that image is found, the -# Go tests use this curl binary for integration tests. -# - -FROM ubuntu:trusty - -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y git-core build-essential wget - -RUN apt-get install -y --no-install-recommends \ - autotools-dev libtool pkg-config zlib1g-dev \ - libcunit1-dev libssl-dev libxml2-dev libevent-dev \ - automake autoconf - -# The list of packages nghttp2 recommends for h2load: -RUN apt-get install -y --no-install-recommends make binutils \ - autoconf automake autotools-dev \ - libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev \ - libev-dev libevent-dev libjansson-dev libjemalloc-dev \ - cython python3.4-dev python-setuptools - -# Note: setting NGHTTP2_VER before the git clone, so an old git clone isn't cached: -ENV NGHTTP2_VER 895da9a -RUN cd /root && git clone https://github.com/tatsuhiro-t/nghttp2.git - -WORKDIR /root/nghttp2 -RUN git reset --hard $NGHTTP2_VER -RUN autoreconf -i -RUN automake -RUN autoconf -RUN ./configure -RUN make -RUN make install - -WORKDIR /root -RUN wget https://curl.se/download/curl-7.45.0.tar.gz -RUN tar -zxvf curl-7.45.0.tar.gz -WORKDIR /root/curl-7.45.0 -RUN ./configure --with-ssl --with-nghttp2=/usr/local -RUN make -RUN make install -RUN ldconfig - -CMD ["-h"] -ENTRYPOINT ["/usr/local/bin/curl"] - diff --git a/http2/Makefile b/http2/Makefile deleted file mode 100644 index 55fd826f77..0000000000 --- a/http2/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -curlimage: - docker build -t gohttp2/curl . - diff --git a/http2/http2_test.go b/http2/http2_test.go index f77c08a107..a16774b7ff 100644 --- a/http2/http2_test.go +++ b/http2/http2_test.go @@ -6,16 +6,13 @@ package http2 import ( "bytes" - "errors" "flag" "fmt" "io/ioutil" "net/http" "os" - "os/exec" "path/filepath" "regexp" - "strconv" "strings" "testing" "time" @@ -85,44 +82,6 @@ func encodeHeaderNoImplicit(t *testing.T, headers ...string) []byte { return buf.Bytes() } -// Verify that curl has http2. -func requireCurl(t *testing.T) { - out, err := dockerLogs(curl(t, "--version")) - if err != nil { - t.Skipf("failed to determine curl features; skipping test") - } - if !strings.Contains(string(out), "HTTP2") { - t.Skip("curl doesn't support HTTP2; skipping test") - } -} - -func curl(t *testing.T, args ...string) (container string) { - out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "gohttp2/curl"}, args...)...).Output() - if err != nil { - t.Skipf("Failed to run curl in docker: %v, %s", err, out) - } - return strings.TrimSpace(string(out)) -} - -// Verify that h2load exists. -func requireH2load(t *testing.T) { - out, err := dockerLogs(h2load(t, "--version")) - if err != nil { - t.Skipf("failed to probe h2load; skipping test: %s", out) - } - if !strings.Contains(string(out), "h2load nghttp2/") { - t.Skipf("h2load not present; skipping test. (Output=%q)", out) - } -} - -func h2load(t *testing.T, args ...string) (container string) { - out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "--entrypoint=/usr/local/bin/h2load", "gohttp2/curl"}, args...)...).Output() - if err != nil { - t.Skipf("Failed to run h2load in docker: %v, %s", err, out) - } - return strings.TrimSpace(string(out)) -} - type puppetCommand struct { fn func(w http.ResponseWriter, r *http.Request) done chan<- bool @@ -151,27 +110,6 @@ func (p *handlerPuppet) do(fn func(http.ResponseWriter, *http.Request)) { p.ch <- puppetCommand{fn, done} <-done } -func dockerLogs(container string) ([]byte, error) { - out, err := exec.Command("docker", "wait", container).CombinedOutput() - if err != nil { - return out, err - } - exitStatus, err := strconv.Atoi(strings.TrimSpace(string(out))) - if err != nil { - return out, errors.New("unexpected exit status from docker wait") - } - out, err = exec.Command("docker", "logs", container).CombinedOutput() - exec.Command("docker", "rm", container).Run() - if err == nil && exitStatus != 0 { - err = fmt.Errorf("exit status %d: %s", exitStatus, out) - } - return out, err -} - -func kill(container string) { - exec.Command("docker", "kill", container).Run() - exec.Command("docker", "rm", container).Run() -} func cleanDate(res *http.Response) { if d := res.Header["Date"]; len(d) == 1 { diff --git a/http2/server_test.go b/http2/server_test.go index cd73291ea0..b99c5af549 100644 --- a/http2/server_test.go +++ b/http2/server_test.go @@ -20,13 +20,11 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "reflect" "runtime" "strconv" "strings" "sync" - "sync/atomic" "testing" "time" @@ -2704,96 +2702,6 @@ func readBodyHandler(t *testing.T, want string) func(w http.ResponseWriter, r *h } } -// TestServerWithCurl currently fails, hence the LenientCipherSuites test. See: -// -// https://github.com/tatsuhiro-t/nghttp2/issues/140 & -// http://sourceforge.net/p/curl/bugs/1472/ -func TestServerWithCurl(t *testing.T) { testServerWithCurl(t, false) } -func TestServerWithCurl_LenientCipherSuites(t *testing.T) { testServerWithCurl(t, true) } - -func testServerWithCurl(t *testing.T, permitProhibitedCipherSuites bool) { - if runtime.GOOS != "linux" { - t.Skip("skipping Docker test when not on Linux; requires --net which won't work with boot2docker anyway") - } - if testing.Short() { - t.Skip("skipping curl test in short mode") - } - requireCurl(t) - var gotConn int32 - testHookOnConn = func() { atomic.StoreInt32(&gotConn, 1) } - - const msg = "Hello from curl!\n" - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Foo", "Bar") - w.Header().Set("Client-Proto", r.Proto) - io.WriteString(w, msg) - })) - ConfigureServer(ts.Config, &Server{ - PermitProhibitedCipherSuites: permitProhibitedCipherSuites, - }) - ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config - ts.StartTLS() - defer ts.Close() - - t.Logf("Running test server for curl to hit at: %s", ts.URL) - container := curl(t, "--silent", "--http2", "--insecure", "-v", ts.URL) - defer kill(container) - res, err := dockerLogs(container) - if err != nil { - t.Fatal(err) - } - - body := string(res) - // Search for both "key: value" and "key:value", since curl changed their format - // Our Dockerfile contains the latest version (no space), but just in case people - // didn't rebuild, check both. - if !strings.Contains(body, "foo: Bar") && !strings.Contains(body, "foo:Bar") { - t.Errorf("didn't see foo: Bar header") - t.Logf("Got: %s", body) - } - if !strings.Contains(body, "client-proto: HTTP/2") && !strings.Contains(body, "client-proto:HTTP/2") { - t.Errorf("didn't see client-proto: HTTP/2 header") - t.Logf("Got: %s", res) - } - if !strings.Contains(string(res), msg) { - t.Errorf("didn't see %q content", msg) - t.Logf("Got: %s", res) - } - - if atomic.LoadInt32(&gotConn) == 0 { - t.Error("never saw an http2 connection") - } -} - -var doh2load = flag.Bool("h2load", false, "Run h2load test") - -func TestServerWithH2Load(t *testing.T) { - if !*doh2load { - t.Skip("Skipping without --h2load flag.") - } - if runtime.GOOS != "linux" { - t.Skip("skipping Docker test when not on Linux; requires --net which won't work with boot2docker anyway") - } - requireH2load(t) - - msg := strings.Repeat("Hello, h2load!\n", 5000) - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - io.WriteString(w, msg) - w.(http.Flusher).Flush() - io.WriteString(w, msg) - })) - ts.StartTLS() - defer ts.Close() - - cmd := exec.Command("docker", "run", "--net=host", "--entrypoint=/usr/local/bin/h2load", "gohttp2/curl", - "-n100000", "-c100", "-m100", ts.URL) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - t.Fatal(err) - } -} - func TestServer_MaxDecoderHeaderTableSize(t *testing.T) { wantHeaderTableSize := uint32(initialHeaderTableSize * 2) st := newServerTester(t, func(w http.ResponseWriter, r *http.Request) {}, func(s *Server) { From 52fbe3731bc7b6873c58d80aae59dc20abbf89c9 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Sun, 13 Aug 2023 10:33:31 -0400 Subject: [PATCH 36/46] quic: add test helpers for acking packets Add connTest methods to send the conn-under-test an ACK for the latest packet it sent, or for all packets in the number space it last sent in. For golang/go#58547 Change-Id: Id35cad9bddf9dd32074dc121fd360a65b989fb4b Reviewed-on: https://go-review.googlesource.com/c/net/+/522055 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam --- internal/quic/conn_id_test.go | 4 ++-- internal/quic/conn_loss_test.go | 7 +------ internal/quic/conn_test.go | 34 +++++++++++++++++++++++++++------ internal/quic/stream_test.go | 19 ++++-------------- internal/quic/tls_test.go | 2 +- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/internal/quic/conn_id_test.go b/internal/quic/conn_id_test.go index 04baf0edaf..d479cd4a87 100644 --- a/internal/quic/conn_id_test.go +++ b/internal/quic/conn_id_test.go @@ -264,7 +264,7 @@ func TestConnIDPeerRequestsRetirement(t *testing.T) { packetType1RTT, debugFrameRetireConnectionID{ seq: 0, }) - if got, want := tc.sentFramePacket.dstConnID, testPeerConnID(1); !bytes.Equal(got, want) { + if got, want := tc.lastPacket.dstConnID, testPeerConnID(1); !bytes.Equal(got, want) { t.Fatalf("used destination conn id {%x}, want {%x}", got, want) } } @@ -467,7 +467,7 @@ func TestConnIDUsePreferredAddressConnID(t *testing.T) { packetType1RTT, debugFrameRetireConnectionID{ seq: 0, }) - if got, want := tc.sentFramePacket.dstConnID, cid; !bytes.Equal(got, want) { + if got, want := tc.lastPacket.dstConnID, cid; !bytes.Equal(got, want) { t.Fatalf("used destination conn id {%x}, want {%x} from preferred address transport parameter", got, want) } } diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go index dc0dc6cd33..bb43030330 100644 --- a/internal/quic/conn_loss_test.go +++ b/internal/quic/conn_loss_test.go @@ -271,12 +271,7 @@ func TestLostStreamPartialLoss(t *testing.T) { data: data[i : i+1], }) if i%2 == 0 { - num := tc.sentFramePacket.num - tc.writeFrames(packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{ - {num, num + 1}, - }, - }) + tc.writeAckForLatest() } } const pto = false diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 2480f9cb0a..2aa38fcf3d 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -137,10 +137,10 @@ type testConn struct { // Datagrams, packets, and frames sent by the conn, // but not yet processed by the test. - sentDatagrams [][]byte - sentPackets []*testPacket - sentFrames []debugFrame - sentFramePacket *testPacket + sentDatagrams [][]byte + sentPackets []*testPacket + sentFrames []debugFrame + lastPacket *testPacket // Frame types to ignore in tests. ignoreFrames map[byte]bool @@ -388,6 +388,28 @@ func (tc *testConn) writeFrames(ptype packetType, frames ...debugFrame) { tc.write(d) } +// writeAckForAll sends the Conn a datagram containing an ack for all packets up to the +// last one received. +func (tc *testConn) writeAckForAll() { + if tc.lastPacket == nil { + return + } + tc.writeFrames(tc.lastPacket.ptype, debugFrameAck{ + ranges: []i64range[packetNumber]{{0, tc.lastPacket.num + 1}}, + }) +} + +// writeAckForLatest sends the Conn a datagram containing an ack for the +// most recent packet received. +func (tc *testConn) writeAckForLatest() { + if tc.lastPacket == nil { + return + } + tc.writeFrames(tc.lastPacket.ptype, debugFrameAck{ + ranges: []i64range[packetNumber]{{tc.lastPacket.num, tc.lastPacket.num + 1}}, + }) +} + // ignoreFrame hides frames of the given type sent by the Conn. func (tc *testConn) ignoreFrame(frameType byte) { tc.ignoreFrames[frameType] = true @@ -423,6 +445,7 @@ func (tc *testConn) readPacket() *testPacket { } p := tc.sentPackets[0] tc.sentPackets = tc.sentPackets[1:] + tc.lastPacket = p return p } @@ -435,12 +458,11 @@ func (tc *testConn) readFrame() (debugFrame, packetType) { if p == nil { return nil, packetTypeInvalid } - tc.sentFramePacket = p tc.sentFrames = p.frames } f := tc.sentFrames[0] tc.sentFrames = tc.sentFrames[1:] - return f, tc.sentFramePacket.ptype + return f, tc.lastPacket.ptype } // wantDatagram indicates that we expect the Conn to send a datagram. diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index 5904a93428..bafd236c9f 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -193,9 +193,7 @@ func TestStreamWriteBlockedByWriteBufferLimit(t *testing.T) { tc.wantIdle("no STREAM_DATA_BLOCKED, we're blocked locally not by flow control") // ACK for previously-sent data allows making more progress. - tc.writeFrames(packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, - }) + tc.writeAckForAll() tc.wantFrame("ACK for previous data allows making progress", packetType1RTT, debugFrameStream{ id: s.id, @@ -968,9 +966,7 @@ func TestStreamCloseWaitsForAcks(t *testing.T) { if _, err := closing.result(); err != errNotDone { t.Fatalf("s.CloseContext() = %v, want it to block waiting for acks", err) } - tc.writeFrames(packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, - }) + tc.writeAckForAll() if _, err := closing.result(); err != nil { t.Fatalf("s.CloseContext() = %v, want nil (all data acked)", err) } @@ -983,9 +979,7 @@ func TestStreamCloseUnblocked(t *testing.T) { }{{ name: "data received", unblock: func(tc *testConn, s *Stream) { - tc.writeFrames(packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, - }) + tc.writeAckForAll() }, }, { name: "stop sending received", @@ -1135,12 +1129,7 @@ func TestStreamPeerStopSendingForActiveStream(t *testing.T) { t.Errorf("s.Write() after STOP_SENDING = %v, %v; want error", n, err) } // This ack will result in some of the previous frames being marked as lost. - tc.writeFrames(packetType1RTT, debugFrameAck{ - ranges: []i64range[packetNumber]{{ - tc.sentFramePacket.num, - tc.sentFramePacket.num + 1, - }}, - }) + tc.writeAckForLatest() tc.wantIdle("lost STREAM frames for reset stream are not resent") }) } diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go index 35cb8bf008..180ea8beec 100644 --- a/internal/quic/tls_test.go +++ b/internal/quic/tls_test.go @@ -266,7 +266,7 @@ func (tc *testConn) uncheckedHandshake() { debugFrameAck{ ackDelay: unscaledAckDelayFromDuration( maxAckDelay, ackDelayExponent), - ranges: []i64range[packetNumber]{{0, tc.sentFramePacket.num + 1}}, + ranges: []i64range[packetNumber]{{0, tc.lastPacket.num + 1}}, }) } else { tc.wantIdle("initial frames are ignored") From 4332436fd1223b60e3127494d5ff771fba2c0adf Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 16 Aug 2023 08:57:37 -0400 Subject: [PATCH 37/46] quic: send more transport parameters Send various transport parameters that we weren't sending yet, but should have been. Add a test for transport parameters sent by us. For golang/go#58547 Change-Id: Id16c46ee39040b091633aca8d4cff4c60562a603 Reviewed-on: https://go-review.googlesource.com/c/net/+/523575 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Jonathan Amsterdam --- internal/quic/config_test.go | 32 ++++++++++++++++++++++++++++++++ internal/quic/conn.go | 14 ++++++++++---- internal/quic/conn_test.go | 10 ++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 internal/quic/config_test.go diff --git a/internal/quic/config_test.go b/internal/quic/config_test.go new file mode 100644 index 0000000000..cec57c5e36 --- /dev/null +++ b/internal/quic/config_test.go @@ -0,0 +1,32 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "testing" + +func TestConfigTransportParameters(t *testing.T) { + const ( + wantInitialMaxStreamData = int64(2) + ) + tc := newTestConn(t, clientSide, func(c *Config) { + c.StreamReadBufferSize = wantInitialMaxStreamData + }) + tc.handshake() + if tc.sentTransportParameters == nil { + t.Fatalf("conn didn't send transport parameters during handshake") + } + p := tc.sentTransportParameters + if got, want := p.initialMaxStreamDataBidiLocal, wantInitialMaxStreamData; got != want { + t.Errorf("initial_max_stream_data_bidi_local = %v, want %v", got, want) + } + if got, want := p.initialMaxStreamDataBidiRemote, wantInitialMaxStreamData; got != want { + t.Errorf("initial_max_stream_data_bidi_remote = %v, want %v", got, want) + } + if got, want := p.initialMaxStreamDataUni, wantInitialMaxStreamData; got != want { + t.Errorf("initial_max_stream_data_uni = %v, want %v", got, want) + } +} diff --git a/internal/quic/conn.go b/internal/quic/conn.go index ee8f011f8a..04dcd7b6bb 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -111,11 +111,17 @@ func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip. c.loss.init(c.side, maxDatagramSize, now) c.streamsInit() + // TODO: initial_source_connection_id, retry_source_connection_id c.startTLS(now, initialConnID, transportParameters{ - initialSrcConnID: c.connIDState.srcConnID(), - ackDelayExponent: ackDelayExponent, - maxUDPPayloadSize: maxUDPPayloadSize, - maxAckDelay: maxAckDelay, + initialSrcConnID: c.connIDState.srcConnID(), + ackDelayExponent: ackDelayExponent, + maxUDPPayloadSize: maxUDPPayloadSize, + maxAckDelay: maxAckDelay, + disableActiveMigration: true, + initialMaxStreamDataBidiLocal: config.streamReadBufferSize(), + initialMaxStreamDataBidiRemote: config.streamReadBufferSize(), + initialMaxStreamDataUni: config.streamReadBufferSize(), + activeConnIDLimit: activeConnIDLimit, }) go c.loop(now) diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 2aa38fcf3d..8ebe49e0e5 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -142,6 +142,9 @@ type testConn struct { sentFrames []debugFrame lastPacket *testPacket + // Transport parameters sent by the conn. + sentTransportParameters *transportParameters + // Frame types to ignore in tests. ignoreFrames map[byte]bool @@ -719,6 +722,13 @@ func (tc *testConnHooks) handleTLSEvent(e tls.QUICEvent) { setKey(&tc.rkeys, e) case tls.QUICWriteData: tc.cryptoDataIn[e.Level] = append(tc.cryptoDataIn[e.Level], e.Data...) + case tls.QUICTransportParameters: + p, err := unmarshalTransportParams(e.Data) + if err != nil { + tc.t.Logf("sent unparseable transport parameters %x %v", e.Data, err) + } else { + tc.sentTransportParameters = &p + } } } } From d1b0a97d84e0fa88851b5a065e23262afed10400 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 15 Aug 2023 11:11:36 -0400 Subject: [PATCH 38/46] quic: avoid sending 1-RTT frames in initial/handshake packets Restructure the send path a little to make it clear that 1-RTT frames go in 1-RTT packets. For golang/go#58547 Change-Id: Id4c2c86c8ccd350bf490f38a8bb01ad9bc2639ee Reviewed-on: https://go-review.googlesource.com/c/net/+/524035 Reviewed-by: Jonathan Amsterdam LUCI-TryBot-Result: Go LUCI --- internal/quic/conn_send.go | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index 6e6fbc5857..9d315fb392 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -224,14 +224,6 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, // TODO: Add all the other frames we can send. - // HANDSHAKE_DONE - if c.handshakeConfirmed.shouldSendPTO(pto) { - if !c.w.appendHandshakeDoneFrame() { - return - } - c.handshakeConfirmed.setSent(pnum) - } - // CRYPTO c.crypto[space].dataToSend(pto, func(off, size int64) int64 { b, _ := c.w.appendCryptoFrame(off, int(size)) @@ -239,13 +231,6 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, return int64(len(b)) }) - // NEW_CONNECTION_ID, RETIRE_CONNECTION_ID - if space == appDataSpace { - if !c.connIDState.appendFrames(&c.w, pnum, pto) { - return - } - } - // Test-only PING frames. if space == c.testSendPingSpace && c.testSendPing.shouldSendPTO(pto) { if !c.w.appendPingFrame() { @@ -254,11 +239,26 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, c.testSendPing.setSent(pnum) } - // All stream-related frames. This should come last in the packet, - // so large amounts of STREAM data don't crowd out other frames - // we may need to send. - if !c.appendStreamFrames(&c.w, pnum, pto) { - return + if space == appDataSpace { + // HANDSHAKE_DONE + if c.handshakeConfirmed.shouldSendPTO(pto) { + if !c.w.appendHandshakeDoneFrame() { + return + } + c.handshakeConfirmed.setSent(pnum) + } + + // NEW_CONNECTION_ID, RETIRE_CONNECTION_ID + if !c.connIDState.appendFrames(&c.w, pnum, pto) { + return + } + + // All stream-related frames. This should come last in the packet, + // so large amounts of STREAM data don't crowd out other frames + // we may need to send. + if !c.appendStreamFrames(&c.w, pnum, pto) { + return + } } // If this is a PTO probe and we haven't added an ack-eliciting frame yet, From fe2abcb6e15f7a34d285220b437d421da4c76775 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 29 Aug 2023 08:21:51 -0700 Subject: [PATCH 39/46] quic: validate stream limits in transport params The maximum number of streams of a given type (bidi/uni) is capped to 2^60, since a larger number would overflow a varint. Validate limits received in transport parameters. RFC 9000, Section 4.6 For golang/go#58547 Change-Id: I7a4a15c569da91ad1b89a5dc71e1c5b213dbda9a Reviewed-on: https://go-review.googlesource.com/c/net/+/524037 LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- internal/quic/packet_parser.go | 2 +- internal/quic/quic.go | 4 ++++ internal/quic/stream_test.go | 4 ++-- internal/quic/transport_params.go | 6 ++++++ internal/quic/transport_params_test.go | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/internal/quic/packet_parser.go b/internal/quic/packet_parser.go index 9a00da7560..ca5b37b2bd 100644 --- a/internal/quic/packet_parser.go +++ b/internal/quic/packet_parser.go @@ -378,7 +378,7 @@ func consumeMaxStreamsFrame(b []byte) (typ streamType, max int64, n int) { return 0, 0, -1 } n += nn - if v > 1<<60 { + if v > maxStreamsLimit { return 0, 0, -1 } return typ, int64(v), n diff --git a/internal/quic/quic.go b/internal/quic/quic.go index 8cd61aed08..71738e129d 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -55,6 +55,10 @@ const timerGranularity = 1 * time.Millisecond // https://www.rfc-editor.org/rfc/rfc9000#section-14.1 const minimumClientInitialDatagramSize = 1200 +// Maximum number of streams of a given type which may be created. +// https://www.rfc-editor.org/rfc/rfc9000.html#section-4.6-2 +const maxStreamsLimit = 1 << 60 + // A connSide distinguishes between the client and server sides of a connection. type connSide int8 diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index bafd236c9f..7b8ba2c54f 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -1165,8 +1165,8 @@ func newTestConnAndRemoteStream(t *testing.T, side connSide, styp streamType, op // permissiveTransportParameters may be passed as an option to newTestConn. func permissiveTransportParameters(p *transportParameters) { - p.initialMaxStreamsBidi = maxVarint - p.initialMaxStreamsUni = maxVarint + p.initialMaxStreamsBidi = maxStreamsLimit + p.initialMaxStreamsUni = maxStreamsLimit p.initialMaxData = maxVarint p.initialMaxStreamDataBidiRemote = maxVarint p.initialMaxStreamDataBidiLocal = maxVarint diff --git a/internal/quic/transport_params.go b/internal/quic/transport_params.go index 89ea69fb97..dc76d16509 100644 --- a/internal/quic/transport_params.go +++ b/internal/quic/transport_params.go @@ -212,8 +212,14 @@ func unmarshalTransportParams(params []byte) (transportParameters, error) { p.initialMaxStreamDataUni, n = consumeVarintInt64(val) case paramInitialMaxStreamsBidi: p.initialMaxStreamsBidi, n = consumeVarintInt64(val) + if p.initialMaxStreamsBidi > maxStreamsLimit { + return p, localTransportError(errTransportParameter) + } case paramInitialMaxStreamsUni: p.initialMaxStreamsUni, n = consumeVarintInt64(val) + if p.initialMaxStreamsUni > maxStreamsLimit { + return p, localTransportError(errTransportParameter) + } case paramAckDelayExponent: var v uint64 v, n = consumeVarint(val) diff --git a/internal/quic/transport_params_test.go b/internal/quic/transport_params_test.go index e1c45ca0e6..cc88e83fd6 100644 --- a/internal/quic/transport_params_test.go +++ b/internal/quic/transport_params_test.go @@ -236,6 +236,20 @@ func TestTransportParametersErrors(t *testing.T) { 15, // length 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, }, + }, { + desc: "initial_max_streams_bidi is too large", + enc: []byte{ + 0x08, // initial_max_streams_bidi, + 8, // length, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + }, { + desc: "initial_max_streams_uni is too large", + enc: []byte{ + 0x08, // initial_max_streams_uni, + 9, // length, + 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, }, { desc: "preferred_address is too short", enc: []byte{ From 8b010a5243b670aca8d2277a3c989d1b6a198a08 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 29 Aug 2023 09:25:04 -0700 Subject: [PATCH 40/46] quic: fix race condition in runAsync test helper asyncTestState.wakeAsync runs on the conn's goroutine and accesses as.blocked, so we need to hold as.mu while initializing as.blocked in runAsync. For golang/go#58547 Change-Id: Idb5921895cee89dfceec2b2439c43f2e380b64ce Reviewed-on: https://go-review.googlesource.com/c/net/+/524095 LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- internal/quic/conn_async_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/quic/conn_async_test.go b/internal/quic/conn_async_test.go index 0da3ddb452..5b419c4e54 100644 --- a/internal/quic/conn_async_test.go +++ b/internal/quic/conn_async_test.go @@ -101,7 +101,9 @@ func runAsync[T any](ts *testConn, f func(context.Context) (T, error)) *asyncOp[ as := &ts.asyncTestState if as.notify == nil { as.notify = make(chan struct{}) + as.mu.Lock() as.blocked = make(map[*blockedAsync]struct{}) + as.mu.Unlock() } _, file, line, _ := runtime.Caller(1) ctx := context.WithValue(context.Background(), asyncContextKey{}, true) From b4d09be75101024ceed6b173b49a5630084174e6 Mon Sep 17 00:00:00 2001 From: Mateusz Poliwczak Date: Mon, 28 Aug 2023 16:40:48 +0000 Subject: [PATCH 41/46] dns/dnsmessage: compress all names while appending to a buffer Change-Id: Iedccbf3e47a63b2239def189ab41bab18a64c398 GitHub-Last-Rev: eb23195734794ab2b211677e5e3616de5f0eb7be GitHub-Pull-Request: golang/net#189 Reviewed-on: https://go-review.googlesource.com/c/net/+/522575 TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Mateusz Poliwczak Reviewed-by: Joedian Reid Auto-Submit: Ian Lance Taylor --- dns/dnsmessage/message.go | 7 ++++--- dns/dnsmessage/message_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/dns/dnsmessage/message.go b/dns/dnsmessage/message.go index 9ddf2c2292..0215a5dde3 100644 --- a/dns/dnsmessage/message.go +++ b/dns/dnsmessage/message.go @@ -2000,14 +2000,15 @@ func (n *Name) pack(msg []byte, compression map[string]int, compressionOff int) } // Miss. Add the suffix to the compression table if the - // offset can be stored in the available 14 bytes. - if len(msg) <= int(^uint16(0)>>2) { + // offset can be stored in the available 14 bits. + newPtr := len(msg) - compressionOff + if newPtr <= int(^uint16(0)>>2) { if nameAsStr == "" { // allocate n.Data on the heap once, to avoid allocating it // multiple times (for next labels). nameAsStr = string(n.Data[:n.Length]) } - compression[nameAsStr[i:]] = len(msg) - compressionOff + compression[nameAsStr[i:]] = newPtr } } } diff --git a/dns/dnsmessage/message_test.go b/dns/dnsmessage/message_test.go index ee42febbc2..23fb3d5748 100644 --- a/dns/dnsmessage/message_test.go +++ b/dns/dnsmessage/message_test.go @@ -1847,3 +1847,30 @@ func TestBuilderNameCompressionWithNonZeroedName(t *testing.T) { t.Fatalf("b.Finish() = %v, want: %v", msg, expect) } } + +func TestBuilderCompressionInAppendMode(t *testing.T) { + maxPtr := int(^uint16(0) >> 2) + b := NewBuilder(make([]byte, maxPtr, maxPtr+512), Header{}) + b.EnableCompression() + if err := b.StartQuestions(); err != nil { + t.Fatalf("b.StartQuestions() unexpected error: %v", err) + } + if err := b.Question(Question{Name: MustNewName("go.dev.")}); err != nil { + t.Fatalf("b.Question() unexpected error: %v", err) + } + if err := b.Question(Question{Name: MustNewName("go.dev.")}); err != nil { + t.Fatalf("b.Question() unexpected error: %v", err) + } + msg, err := b.Finish() + if err != nil { + t.Fatalf("b.Finish() unexpected error: %v", err) + } + expect := []byte{ + 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, // header + 2, 'g', 'o', 3, 'd', 'e', 'v', 0, 0, 0, 0, 0, // question 1 + 0xC0, 12, 0, 0, 0, 0, // question 2 + } + if !bytes.Equal(msg[maxPtr:], expect) { + t.Fatalf("msg[maxPtr:] = %v, want: %v", msg[maxPtr:], expect) + } +} From 7374d342a2c3de79d7b96f3aacdb13498c3fc3b5 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 29 Aug 2023 12:51:05 -0700 Subject: [PATCH 42/46] quic: don't block when closing read-only streams Stream.Close blocks until all data sent on a stream has been acked by the peer. Don't block indefinitely when closing a read-only stream, waiting for an ack of data we never sent. For golang/go#58547 Change-Id: I4087666f739d7388e460b613d211c043626f1c87 Reviewed-on: https://go-review.googlesource.com/c/net/+/524038 LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- internal/quic/stream.go | 7 ++++++- internal/quic/stream_test.go | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/quic/stream.go b/internal/quic/stream.go index 12117dbd39..1033cbb401 100644 --- a/internal/quic/stream.go +++ b/internal/quic/stream.go @@ -66,7 +66,9 @@ func newStream(c *Conn, id streamID) *Stream { inresetcode: -1, // -1 indicates no RESET_STREAM received ingate: newLockedGate(), outgate: newLockedGate(), - outdone: make(chan struct{}), + } + if !s.IsReadOnly() { + s.outdone = make(chan struct{}) } return s } @@ -237,6 +239,9 @@ func (s *Stream) Close() error { // CloseContext discards the buffer and returns the context error. func (s *Stream) CloseContext(ctx context.Context) error { s.CloseRead() + if s.IsReadOnly() { + return nil + } s.CloseWrite() // TODO: Return code from peer's RESET_STREAM frame? return s.conn.waitOnDone(ctx, s.outdone) diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index 7b8ba2c54f..79377c6a4a 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -972,6 +972,17 @@ func TestStreamCloseWaitsForAcks(t *testing.T) { } } +func TestStreamCloseReadOnly(t *testing.T) { + tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, permissiveTransportParameters) + if err := s.CloseContext(canceledContext()); err != nil { + t.Errorf("s.CloseContext() = %v, want nil", err) + } + tc.wantFrame("closed stream sends STOP_SENDING", + packetType1RTT, debugFrameStopSending{ + id: s.id, + }) +} + func TestStreamCloseUnblocked(t *testing.T) { for _, test := range []struct { name string From b82f062c4bc1abcfe993e3750d64c4bdd4a14f87 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 29 Aug 2023 15:35:30 -0700 Subject: [PATCH 43/46] quic: include ignored frames in test log output When looking at a test log, it's a bit confusing to have some of the frames silently omitted. Print ignored frames. Unfortunately, this means we need to do the actual ignoring of frames after printing the packet. We specify frames to ignore by the frame number, but after parsing we don't have a simple way to map from the debugFrame type back to the number. Add a big, ugly mapping function to do this; it's clunky, but isolated to one function in tests. For golang/go#58547 Change-Id: I242f5511dc16be2350fa49030af38588fe92a988 Reviewed-on: https://go-review.googlesource.com/c/net/+/524295 LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- internal/quic/conn_test.go | 77 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 8ebe49e0e5..d8c44558dc 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -431,7 +431,80 @@ func (tc *testConn) readDatagram() *testDatagram { buf := tc.sentDatagrams[0] tc.sentDatagrams = tc.sentDatagrams[1:] d := tc.parseTestDatagram(buf) + // Log the datagram before removing ignored frames. + // When things go wrong, it's useful to see all the frames. tc.logDatagram("-> conn under test sends", d) + typeForFrame := func(f debugFrame) byte { + // This is very clunky, and points at a problem + // in how we specify what frames to ignore in tests. + // + // We mark frames to ignore using the frame type, + // but we've got a debugFrame data structure here. + // Perhaps we should be ignoring frames by debugFrame + // type instead: tc.ignoreFrame[debugFrameAck](). + switch f := f.(type) { + case debugFramePadding: + return frameTypePadding + case debugFramePing: + return frameTypePing + case debugFrameAck: + return frameTypeAck + case debugFrameResetStream: + return frameTypeResetStream + case debugFrameStopSending: + return frameTypeStopSending + case debugFrameCrypto: + return frameTypeCrypto + case debugFrameNewToken: + return frameTypeNewToken + case debugFrameStream: + return frameTypeStreamBase + case debugFrameMaxData: + return frameTypeMaxData + case debugFrameMaxStreamData: + return frameTypeMaxStreamData + case debugFrameMaxStreams: + if f.streamType == bidiStream { + return frameTypeMaxStreamsBidi + } else { + return frameTypeMaxStreamsUni + } + case debugFrameDataBlocked: + return frameTypeDataBlocked + case debugFrameStreamDataBlocked: + return frameTypeStreamDataBlocked + case debugFrameStreamsBlocked: + if f.streamType == bidiStream { + return frameTypeStreamsBlockedBidi + } else { + return frameTypeStreamsBlockedUni + } + case debugFrameNewConnectionID: + return frameTypeNewConnectionID + case debugFrameRetireConnectionID: + return frameTypeRetireConnectionID + case debugFramePathChallenge: + return frameTypePathChallenge + case debugFramePathResponse: + return frameTypePathResponse + case debugFrameConnectionCloseTransport: + return frameTypeConnectionCloseTransport + case debugFrameConnectionCloseApplication: + return frameTypeConnectionCloseApplication + case debugFrameHandshakeDone: + return frameTypeHandshakeDone + } + panic(fmt.Errorf("unhandled frame type %T", f)) + } + for _, p := range d.packets { + var frames []debugFrame + for _, f := range p.frames { + if !tc.ignoreFrames[typeForFrame(f)] { + frames = append(frames, f) + } + } + p.frames = frames + } return d } @@ -632,9 +705,7 @@ func (tc *testConn) parseTestFrames(payload []byte) ([]debugFrame, error) { if n < 0 { return nil, errors.New("error parsing frames") } - if !tc.ignoreFrames[payload[0]] { - frames = append(frames, f) - } + frames = append(frames, f) payload = payload[n:] } return frames, nil From 03d5e623398478fa929c8ba4b8f15de74017d82a Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Tue, 29 Aug 2023 15:48:27 +0200 Subject: [PATCH 44/46] http2: remove unused ClientConn.tconnClosed It was added in CL 429060 but was never used. Change-Id: Ie1bcd44559006082afed319c0db677ff2ca957d7 Reviewed-on: https://go-review.googlesource.com/c/net/+/523935 Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Run-TryBot: Tobias Klauser Reviewed-by: Damien Neil TryBot-Result: Gopher Robot LUCI-TryBot-Result: Go LUCI --- http2/transport.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/http2/transport.go b/http2/transport.go index b0d482f9f4..4515b22c4a 100644 --- a/http2/transport.go +++ b/http2/transport.go @@ -291,8 +291,7 @@ func (t *Transport) initConnPool() { // HTTP/2 server. type ClientConn struct { t *Transport - tconn net.Conn // usually *tls.Conn, except specialized impls - tconnClosed bool + tconn net.Conn // usually *tls.Conn, except specialized impls tlsState *tls.ConnectionState // nil only for specialized impls reused uint32 // whether conn is being reused; atomic singleUse bool // whether being used for a single http.Request From 97384c11dd0db63357820b2cfcb44c40fbc3116a Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 29 Aug 2023 13:18:00 -0700 Subject: [PATCH 45/46] quic: remove streams from the conn when done When a stream has been fully shut down--the peer has closed its end and acked every frame we will send for it--remove it from the Conn's set of active streams. We do the actual removal on the conn's loop, so stream cleanup can access conn state without worrying about locking. For golang/go#58547 Change-Id: Id9715693649929b07d303f0c4b3a782d135f0326 Reviewed-on: https://go-review.googlesource.com/c/net/+/524296 Reviewed-by: Jonathan Amsterdam LUCI-TryBot-Result: Go LUCI --- internal/quic/atomic_bits.go | 33 +++++++ internal/quic/conn_streams.go | 62 +++++++++---- internal/quic/conn_streams_test.go | 89 ++++++++++++++++++ internal/quic/conn_test.go | 2 + internal/quic/stream.go | 140 +++++++++++++++++++++++------ internal/quic/stream_test.go | 33 +++++++ 6 files changed, 315 insertions(+), 44 deletions(-) create mode 100644 internal/quic/atomic_bits.go diff --git a/internal/quic/atomic_bits.go b/internal/quic/atomic_bits.go new file mode 100644 index 0000000000..e1e2594d15 --- /dev/null +++ b/internal/quic/atomic_bits.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import "sync/atomic" + +// atomicBits is an atomic uint32 that supports setting individual bits. +type atomicBits[T ~uint32] struct { + bits atomic.Uint32 +} + +// set sets the bits in mask to the corresponding bits in v. +// It returns the new value. +func (a *atomicBits[T]) set(v, mask T) T { + if v&^mask != 0 { + panic("BUG: bits in v are not in mask") + } + for { + o := a.bits.Load() + n := (o &^ uint32(mask)) | uint32(v) + if a.bits.CompareAndSwap(o, n) { + return T(n) + } + } +} + +func (a *atomicBits[T]) load() T { + return T(a.bits.Load()) +} diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go index dd35e34cf6..0ede284e23 100644 --- a/internal/quic/conn_streams.go +++ b/internal/quic/conn_streams.go @@ -185,24 +185,46 @@ func (c *Conn) appendStreamFrames(w *packetWriter, pnum packetNumber, pto bool) for { s := c.streams.sendHead const pto = false - if !s.appendInFrames(w, pnum, pto) { - return false + + state := s.state.load() + if state&streamInSend != 0 { + s.ingate.lock() + ok := s.appendInFramesLocked(w, pnum, pto) + state = s.inUnlockNoQueue() + if !ok { + return false + } } - avail := w.avail() - if !s.appendOutFrames(w, pnum, pto) { - // We've sent some data for this stream, but it still has more to send. - // If the stream got a reasonable chance to put data in a packet, - // advance sendHead to the next stream in line, to avoid starvation. - // We'll come back to this stream after going through the others. - // - // If the packet was already mostly out of space, leave sendHead alone - // and come back to this stream again on the next packet. - if avail > 512 { - c.streams.sendHead = s.next - c.streams.sendTail = s + + if state&streamOutSend != 0 { + avail := w.avail() + s.outgate.lock() + ok := s.appendOutFramesLocked(w, pnum, pto) + state = s.outUnlockNoQueue() + if !ok { + // We've sent some data for this stream, but it still has more to send. + // If the stream got a reasonable chance to put data in a packet, + // advance sendHead to the next stream in line, to avoid starvation. + // We'll come back to this stream after going through the others. + // + // If the packet was already mostly out of space, leave sendHead alone + // and come back to this stream again on the next packet. + if avail > 512 { + c.streams.sendHead = s.next + c.streams.sendTail = s + } + return false } - return false } + + if state == streamInDone|streamOutDone { + // Stream is finished, remove it from the conn. + s.state.set(streamConnRemoved, streamConnRemoved) + delete(c.streams.streams, s.id) + + // TODO: Provide the peer with additional stream quota (MAX_STREAMS). + } + next := s.next s.next = nil if (next == s) != (s == c.streams.sendTail) { @@ -231,10 +253,16 @@ func (c *Conn) appendStreamFramesPTO(w *packetWriter, pnum packetNumber) bool { defer c.streams.sendMu.Unlock() for _, s := range c.streams.streams { const pto = true - if !s.appendInFrames(w, pnum, pto) { + s.ingate.lock() + inOK := s.appendInFramesLocked(w, pnum, pto) + s.inUnlockNoQueue() + if !inOK { return false } - if !s.appendOutFrames(w, pnum, pto) { + s.outgate.lock() + outOK := s.appendOutFramesLocked(w, pnum, pto) + s.outUnlockNoQueue() + if !outOK { return false } } diff --git a/internal/quic/conn_streams_test.go b/internal/quic/conn_streams_test.go index 877dbb94fc..9bbc994b11 100644 --- a/internal/quic/conn_streams_test.go +++ b/internal/quic/conn_streams_test.go @@ -8,6 +8,8 @@ package quic import ( "context" + "fmt" + "io" "testing" ) @@ -253,3 +255,90 @@ func TestStreamsWriteQueueFairness(t *testing.T) { } } } + +func TestStreamsShutdown(t *testing.T) { + // These tests verify that a stream is removed from the Conn's map of live streams + // after it is fully shut down. + // + // Each case consists of a setup step, after which one stream should exist, + // and a shutdown step, after which no streams should remain in the Conn. + for _, test := range []struct { + name string + side streamSide + styp streamType + setup func(*testing.T, *testConn, *Stream) + shutdown func(*testing.T, *testConn, *Stream) + }{{ + name: "closed", + side: localStream, + styp: uniStream, + setup: func(t *testing.T, tc *testConn, s *Stream) { + s.CloseContext(canceledContext()) + }, + shutdown: func(t *testing.T, tc *testConn, s *Stream) { + tc.writeAckForAll() + }, + }, { + name: "local close", + side: localStream, + styp: bidiStream, + setup: func(t *testing.T, tc *testConn, s *Stream) { + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + }) + s.CloseContext(canceledContext()) + }, + shutdown: func(t *testing.T, tc *testConn, s *Stream) { + tc.writeAckForAll() + }, + }, { + name: "remote reset", + side: localStream, + styp: bidiStream, + setup: func(t *testing.T, tc *testConn, s *Stream) { + s.CloseContext(canceledContext()) + tc.wantIdle("all frames after CloseContext are ignored") + tc.writeAckForAll() + }, + shutdown: func(t *testing.T, tc *testConn, s *Stream) { + tc.writeFrames(packetType1RTT, debugFrameResetStream{ + id: s.id, + }) + }, + }, { + name: "local close", + side: remoteStream, + styp: uniStream, + setup: func(t *testing.T, tc *testConn, s *Stream) { + ctx := canceledContext() + tc.writeFrames(packetType1RTT, debugFrameStream{ + id: s.id, + fin: true, + }) + if n, err := s.ReadContext(ctx, make([]byte, 16)); n != 0 || err != io.EOF { + t.Errorf("ReadContext() = %v, %v; want 0, io.EOF", n, err) + } + }, + shutdown: func(t *testing.T, tc *testConn, s *Stream) { + s.CloseRead() + }, + }} { + name := fmt.Sprintf("%v/%v/%v", test.side, test.styp, test.name) + t.Run(name, func(t *testing.T) { + tc, s := newTestConnAndStream(t, serverSide, test.side, test.styp, + permissiveTransportParameters) + tc.ignoreFrame(frameTypeStreamBase) + tc.ignoreFrame(frameTypeStopSending) + test.setup(t, tc, s) + tc.wantIdle("conn should be idle after setup") + if got, want := len(tc.conn.streams.streams), 1; got != want { + t.Fatalf("after setup: %v streams in Conn's map; want %v", got, want) + } + test.shutdown(t, tc, s) + tc.wantIdle("conn should be idle after shutdown") + if got, want := len(tc.conn.streams.streams), 0; got != want { + t.Fatalf("after shutdown: %v streams in Conn's map; want %v", got, want) + } + }) + } +} diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index d8c44558dc..ea720d5754 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -394,6 +394,7 @@ func (tc *testConn) writeFrames(ptype packetType, frames ...debugFrame) { // writeAckForAll sends the Conn a datagram containing an ack for all packets up to the // last one received. func (tc *testConn) writeAckForAll() { + tc.t.Helper() if tc.lastPacket == nil { return } @@ -405,6 +406,7 @@ func (tc *testConn) writeAckForAll() { // writeAckForLatest sends the Conn a datagram containing an ack for the // most recent packet received. func (tc *testConn) writeAckForLatest() { + tc.t.Helper() if tc.lastPacket == nil { return } diff --git a/internal/quic/stream.go b/internal/quic/stream.go index 1033cbb401..2dbf4461ba 100644 --- a/internal/quic/stream.go +++ b/internal/quic/stream.go @@ -49,9 +49,38 @@ type Stream struct { outresetcode uint64 // reset code to send in RESET_STREAM outdone chan struct{} // closed when all data sent + // Atomic stream state bits. + // + // These bits provide a fast way to coordinate between the + // send and receive sides of the stream, and the conn's loop. + // + // streamIn* bits must be set with ingate held. + // streamOut* bits must be set with outgate held. + // streamConn* bits are set by the conn's loop. + state atomicBits[streamState] + prev, next *Stream // guarded by streamsState.sendMu } +type streamState uint32 + +const ( + // streamInSend and streamOutSend are set when there are + // frames to send for the inbound or outbound sides of the stream. + // For example, MAX_STREAM_DATA or STREAM_DATA_BLOCKED. + streamInSend = streamState(1 << iota) + streamOutSend + + // streamInDone and streamOutDone are set when the inbound or outbound + // sides of the stream are finished. When both are set, the stream + // can be removed from the Conn and forgotten. + streamInDone + streamOutDone + + // streamConnRemoved is set when the stream has been removed from the conn. + streamConnRemoved +) + // newStream returns a new stream. // // The stream's ingate and outgate are locked. @@ -289,15 +318,34 @@ func (s *Stream) CloseWrite() { // that the stream was terminated abruptly. // Any blocked writes will be unblocked and return errors. // -// Reset sends the application protocol error code to the peer. +// Reset sends the application protocol error code, which must be +// less than 2^62, to the peer. // It does not wait for the peer to acknowledge receipt of the error. // Use CloseContext to wait for the peer's acknowledgement. +// +// Reset does not affect reads. +// Use CloseRead to abort reads on the stream. func (s *Stream) Reset(code uint64) { + const userClosed = true + s.resetInternal(code, userClosed) +} + +func (s *Stream) resetInternal(code uint64, userClosed bool) { s.outgate.lock() defer s.outUnlock() + if s.IsReadOnly() { + return + } + if userClosed { + // Mark that the user closed the stream. + s.outclosed.set() + } if s.outreset.isSet() { return } + if code > maxVarint { + code = maxVarint + } // We could check here to see if the stream is closed and the // peer has acked all the data and the FIN, but sending an // extra RESET_STREAM in this case is harmless. @@ -310,44 +358,67 @@ func (s *Stream) Reset(code uint64) { // inUnlock unlocks s.ingate. // It sets the gate condition if reads from s will not block. -// If s has receive-related frames to write, it notifies the Conn. +// If s has receive-related frames to write or if both directions +// are done and the stream should be removed, it notifies the Conn. func (s *Stream) inUnlock() { - if s.inUnlockNoQueue() { + state := s.inUnlockNoQueue() + if state&streamInSend != 0 || state == streamInDone|streamOutDone { s.conn.queueStreamForSend(s) } } // inUnlockNoQueue is inUnlock, // but reports whether s has frames to write rather than notifying the Conn. -func (s *Stream) inUnlockNoQueue() (shouldSend bool) { +func (s *Stream) inUnlockNoQueue() streamState { canRead := s.inset.contains(s.in.start) || // data available to read s.insize == s.in.start || // at EOF s.inresetcode != -1 || // reset by peer s.inclosed.isSet() // closed locally defer s.ingate.unlock(canRead) - return s.insendmax.shouldSend() || // STREAM_MAX_DATA - s.inclosed.shouldSend() // STOP_SENDING + var state streamState + switch { + case s.IsWriteOnly(): + state = streamInDone + case s.inresetcode != -1: // reset by peer + fallthrough + case s.in.start == s.insize: // all data received and read + // We don't increase MAX_STREAMS until the user calls ReadClose or Close, + // so the receive side is not finished until inclosed is set. + if s.inclosed.isSet() { + state = streamInDone + } + case s.insendmax.shouldSend(): // STREAM_MAX_DATA + state = streamInSend + case s.inclosed.shouldSend(): // STOP_SENDING + state = streamInSend + } + const mask = streamInDone | streamInSend + return s.state.set(state, mask) } // outUnlock unlocks s.outgate. // It sets the gate condition if writes to s will not block. -// If s has send-related frames to write, it notifies the Conn. +// If s has send-related frames to write or if both directions +// are done and the stream should be removed, it notifies the Conn. func (s *Stream) outUnlock() { - if s.outUnlockNoQueue() { + state := s.outUnlockNoQueue() + if state&streamOutSend != 0 || state == streamInDone|streamOutDone { s.conn.queueStreamForSend(s) } } // outUnlockNoQueue is outUnlock, // but reports whether s has frames to write rather than notifying the Conn. -func (s *Stream) outUnlockNoQueue() (shouldSend bool) { +func (s *Stream) outUnlockNoQueue() streamState { isDone := s.outclosed.isReceived() && s.outacked.isrange(0, s.out.end) || // all data acked s.outreset.isSet() // reset locally if isDone { select { case <-s.outdone: default: - close(s.outdone) + if !s.IsReadOnly() { + close(s.outdone) + } } } lim := min(s.out.start+s.outmaxbuf, s.outwin) @@ -355,14 +426,32 @@ func (s *Stream) outUnlockNoQueue() (shouldSend bool) { s.outclosed.isSet() || // closed locally s.outreset.isSet() // reset locally defer s.outgate.unlock(canWrite) - if s.outreset.isSet() { - // If the stream is reset locally, the only frame we'll send is RESET_STREAM. - return s.outreset.shouldSend() - } - return len(s.outunsent) > 0 || // STREAM frame with data - s.outclosed.shouldSend() || // STREAM frame with FIN bit - s.outopened.shouldSend() || // STREAM frame with no data - s.outblocked.shouldSend() // STREAM_DATA_BLOCKED + var state streamState + switch { + case s.IsReadOnly(): + state = streamOutDone + case s.outclosed.isReceived() && s.outacked.isrange(0, s.out.end): // all data sent and acked + fallthrough + case s.outreset.isReceived(): // RESET_STREAM sent and acked + // We don't increase MAX_STREAMS until the user calls WriteClose or Close, + // so the send side is not finished until outclosed is set. + if s.outclosed.isSet() { + state = streamOutDone + } + case s.outreset.shouldSend(): // RESET_STREAM + state = streamOutSend + case s.outreset.isSet(): // RESET_STREAM sent but not acknowledged + case len(s.outunsent) > 0: // STREAM frame with data + state = streamOutSend + case s.outclosed.shouldSend(): // STREAM frame with FIN bit + state = streamOutSend + case s.outopened.shouldSend(): // STREAM frame with no data + state = streamOutSend + case s.outblocked.shouldSend(): // STREAM_DATA_BLOCKED + state = streamOutSend + } + const mask = streamOutDone | streamOutSend + return s.state.set(state, mask) } // handleData handles data received in a STREAM frame. @@ -431,7 +520,8 @@ func (s *Stream) checkStreamBounds(end int64, fin bool) error { func (s *Stream) handleStopSending(code uint64) error { // Peer requests that we reset this stream. // https://www.rfc-editor.org/rfc/rfc9000#section-3.5-4 - s.Reset(code) + const userReset = false + s.resetInternal(code, userReset) return nil } @@ -504,14 +594,12 @@ func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fa } } -// appendInFrames appends STOP_SENDING and MAX_STREAM_DATA frames +// appendInFramesLocked appends STOP_SENDING and MAX_STREAM_DATA frames // to the current packet. // // It returns true if no more frames need appending, // false if not everything fit in the current packet. -func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bool { - s.ingate.lock() - defer s.inUnlockNoQueue() +func (s *Stream) appendInFramesLocked(w *packetWriter, pnum packetNumber, pto bool) bool { if s.inclosed.shouldSendPTO(pto) { // We don't currently have an API for setting the error code. // Just send zero. @@ -534,14 +622,12 @@ func (s *Stream) appendInFrames(w *packetWriter, pnum packetNumber, pto bool) bo return true } -// appendOutFrames appends RESET_STREAM, STREAM_DATA_BLOCKED, and STREAM frames +// appendOutFramesLocked appends RESET_STREAM, STREAM_DATA_BLOCKED, and STREAM frames // to the current packet. // // It returns true if no more frames need appending, // false if not everything fit in the current packet. -func (s *Stream) appendOutFrames(w *packetWriter, pnum packetNumber, pto bool) bool { - s.outgate.lock() - defer s.outUnlockNoQueue() +func (s *Stream) appendOutFramesLocked(w *packetWriter, pnum packetNumber, pto bool) bool { if s.outreset.isSet() { // RESET_STREAM if s.outreset.shouldSendPTO(pto) { diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go index 79377c6a4a..e22e0432ef 100644 --- a/internal/quic/stream_test.go +++ b/internal/quic/stream_test.go @@ -1111,6 +1111,24 @@ func TestStreamPeerResetFollowedByData(t *testing.T) { }) } +func TestStreamResetInvalidCode(t *testing.T) { + tc, s := newTestConnAndLocalStream(t, serverSide, uniStream) + s.Reset(1 << 62) + tc.wantFrame("reset with invalid code sends a RESET_STREAM anyway", + packetType1RTT, debugFrameResetStream{ + id: s.id, + // The code we send here isn't specified, + // so this could really be any value. + code: (1 << 62) - 1, + }) +} + +func TestStreamResetReceiveOnly(t *testing.T) { + tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream) + s.Reset(0) + tc.wantIdle("resetting a receive-only stream has no effect") +} + func TestStreamPeerStopSendingForActiveStream(t *testing.T) { // "An endpoint that receives a STOP_SENDING frame MUST send a RESET_STREAM frame if // the stream is in the "Ready" or "Send" state." @@ -1145,6 +1163,21 @@ func TestStreamPeerStopSendingForActiveStream(t *testing.T) { }) } +type streamSide string + +const ( + localStream = streamSide("local") + remoteStream = streamSide("remote") +) + +func newTestConnAndStream(t *testing.T, side connSide, sside streamSide, styp streamType, opts ...any) (*testConn, *Stream) { + if sside == localStream { + return newTestConnAndLocalStream(t, side, styp, opts...) + } else { + return newTestConnAndRemoteStream(t, side, styp, opts...) + } +} + func newTestConnAndLocalStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) { t.Helper() ctx := canceledContext() From 2a0da8be5a758b33fa896384d689071e832b4aa2 Mon Sep 17 00:00:00 2001 From: Gopher Robot Date: Tue, 5 Sep 2023 15:01:14 +0000 Subject: [PATCH 46/46] go.mod: update golang.org/x dependencies Update golang.org/x dependencies to their latest tagged versions. Change-Id: I2011e2fc11608c371c3145c95a4cf98609010f99 Reviewed-on: https://go-review.googlesource.com/c/net/+/525635 Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot Run-TryBot: Gopher Robot Auto-Submit: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: Carlos Amedee --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 90f428f40d..b16f4e5e68 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module golang.org/x/net go 1.17 require ( - golang.org/x/crypto v0.12.0 - golang.org/x/sys v0.11.0 - golang.org/x/term v0.11.0 - golang.org/x/text v0.12.0 + golang.org/x/crypto v0.13.0 + golang.org/x/sys v0.12.0 + golang.org/x/term v0.12.0 + golang.org/x/text v0.13.0 ) diff --git a/go.sum b/go.sum index c39d831315..0fd3311f48 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -20,21 +20,21 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=