Thanks to visit codestin.com
Credit goes to github.com

Skip to content

fix: Avoid panic in ServerSentEventSender by keeping handler alive #4821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 1, 2022
Merged

fix: Avoid panic in ServerSentEventSender by keeping handler alive #4821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed0189c
fix: Avoid panic in `ServerSentEventSender` by keeping handler alive
mafredri Nov 1, 2022
87a10aa
Improve cancellation in sendEvent
mafredri Nov 1, 2022
6a6232b
Merge branch 'main' into mafredri/fix-sse-flush-after-handler-end
mafredri Nov 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 53 additions & 26 deletions coderd/httpapi/httpapi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,13 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"reflect"
"strings"
"sync"
"time"

"github.com/go-playground/validator/v10"
"golang.org/x/xerrors"

"github.com/coder/coder/coderd/tracing"
"github.com/coder/coder/codersdk"
Expand Down Expand Up @@ -174,8 +173,7 @@ func WebsocketCloseSprintf(format string, vars ...any) string {
return msg
}

func ServerSentEventSender(rw http.ResponseWriter, r *http.Request) (func(ctx context.Context, sse codersdk.ServerSentEvent) error, error) {
var mu sync.Mutex
func ServerSentEventSender(rw http.ResponseWriter, r *http.Request) (sendEvent func(ctx context.Context, sse codersdk.ServerSentEvent) error, closed chan struct{}, err error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like we can eliminate the error return from this as it's never set to a non-nil value

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really relevant for this PR, but that's a good point. I'd like to see it kept and remove the panic in favor of an error, though.

h := rw.Header()
h.Set("Content-Type", "text/event-stream")
h.Set("Cache-Control", "no-cache")
Expand All @@ -187,37 +185,50 @@ func ServerSentEventSender(rw http.ResponseWriter, r *http.Request) (func(ctx co
panic("http.ResponseWriter is not http.Flusher")
}

// Send a heartbeat every 15 seconds to avoid the connection being killed.
closed = make(chan struct{})
type sseEvent struct {
payload []byte
errC chan error
}
eventC := make(chan sseEvent)

// Synchronized handling of events (no guarantee of order).
go func() {
defer close(closed)

// Send a heartbeat every 15 seconds to avoid the connection being killed.
ticker := time.NewTicker(time.Second * 15)
defer ticker.Stop()

for {
var event sseEvent

select {
case <-r.Context().Done():
return
Comment on lines 207 to 208
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why doesn't this prevent the write/flush after finish? We're not hijacking the connection so the context should still be cancelled on finish right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because this code had the following race condition:

  1. Say watchWorkspace encountered an error and exited (this begins the process of all handlers and middleware exiting (r.Context() is not yet cancelled)
  2. Timer is triggered
  3. We write to rw which still succeeds as the teardown is still happening
  4. Teardown finishes, context is cancelled, but we're currently not listening to this signal
  5. We hit Flush after teardown so we get a nil pointer deref

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome explanation thank you

case event = <-eventC:
case <-ticker.C:
mu.Lock()
_, err := io.WriteString(rw, fmt.Sprintf("event: %s\n\n", codersdk.ServerSentEventTypePing))
if err != nil {
mu.Unlock()
return
event = sseEvent{
payload: []byte(fmt.Sprintf("event: %s\n\n", codersdk.ServerSentEventTypePing)),
}
f.Flush()
mu.Unlock()
}
}
}()

sendEvent := func(ctx context.Context, sse codersdk.ServerSentEvent) error {
if ctx.Err() != nil {
return ctx.Err()
_, err := rw.Write(event.payload)
if event.errC != nil {
event.errC <- err
}
if err != nil {
return
}
f.Flush()
}
}()

sendEvent = func(ctx context.Context, sse codersdk.ServerSentEvent) error {
buf := &bytes.Buffer{}
enc := json.NewEncoder(buf)

_, err := buf.Write([]byte(fmt.Sprintf("event: %s\ndata: ", sse.Type)))
_, err := buf.WriteString(fmt.Sprintf("event: %s\ndata: ", sse.Type))
if err != nil {
return err
}
Expand All @@ -232,16 +243,32 @@ func ServerSentEventSender(rw http.ResponseWriter, r *http.Request) (func(ctx co
return err
}

mu.Lock()
defer mu.Unlock()
_, err = rw.Write(buf.Bytes())
if err != nil {
return err
event := sseEvent{
payload: buf.Bytes(),
errC: make(chan error, 1), // Buffered to prevent deadlock.
}
f.Flush()

return nil
select {
case <-r.Context().Done():
return r.Context().Err()
case <-ctx.Done():
return ctx.Err()
case <-closed:
return xerrors.New("server sent event sender closed")
case eventC <- event:
// Re-check closure signals after sending the event to allow
// for early exit. We don't check closed here because it
// can't happen while processing the event.
select {
case <-r.Context().Done():
return r.Context().Err()
case <-ctx.Done():
return ctx.Err()
case err := <-event.errC:
return err
}
}
}

return sendEvent, nil
return sendEvent, closed, nil
}
8 changes: 7 additions & 1 deletion coderd/workspaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -867,14 +867,18 @@ func (api *API) watchWorkspace(rw http.ResponseWriter, r *http.Request) {
return
}

sendEvent, err := httpapi.ServerSentEventSender(rw, r)
sendEvent, senderClosed, err := httpapi.ServerSentEventSender(rw, r)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error setting up server-sent events.",
Detail: err.Error(),
})
return
}
// Prevent handler from returning until the sender is closed.
defer func() {
<-senderClosed
}()

// Ignore all trace spans after this, they're not too useful.
ctx = trace.ContextWithSpan(ctx, tracing.NoopSpan)
Expand All @@ -885,6 +889,8 @@ func (api *API) watchWorkspace(rw http.ResponseWriter, r *http.Request) {
select {
case <-ctx.Done():
return
case <-senderClosed:
return
case <-t.C:
workspace, err := api.Database.GetWorkspaceByID(ctx, workspace.ID)
if err != nil {
Expand Down