Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 812549d

Browse files
fix: pin workspace agent API client to intended agent (#26600) (#26612)
Backport of #26600 to `release/2.32`. Original PR: #26600 - fix: pin workspace agent API client to intended agent Merge commit: eeb2624 Requested by: @ethanndickson ## What this fixes The workspace agent API client followed HTTP redirects and trusted the redirected host, letting a malicious agent bounce a coderd request onto a different agent's unauthenticated port-4 API (cross-tenant file read/write and RCE, Cure53 CODAGT-668). `apiClient` now refuses redirects and pins every dial to the intended agent address, and the task-app / scaletest clients share `AppHTTPClient`, which blocks redirects too. ## Conflict resolution The automatic cherry-pick conflicted because #26600 was built on a separate request-context refactor of `apiClient` that is not present on this release branch. The redirect block and agent-address pinning are applied to the existing `apiClient()` here, and the request-context-bounded dial test (which depends on that refactor) is omitted. This branch had no `agentconn_test.go`, so the redirect tests are added as a new `agentconn_test.go`. The two redirect regression tests added by #26600 are included. Co-authored-by: Ethan Dickson <[email protected]>
1 parent cf72d85 commit 812549d

7 files changed

Lines changed: 268 additions & 40 deletions

File tree

coderd/aitasks.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"encoding/json"
77
"errors"
88
"fmt"
9-
"net"
109
"net/http"
1110
"net/url"
1211
"slices"
@@ -1086,13 +1085,7 @@ func (api *API) authAndDoWithTaskAppClient(
10861085
}
10871086
defer release()
10881087

1089-
client := &http.Client{
1090-
Transport: &http.Transport{
1091-
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
1092-
return agentConn.DialContext(ctx, network, addr)
1093-
},
1094-
},
1095-
}
1088+
client := agentConn.AppHTTPClient()
10961089
return do(ctx, client, parsedURL)
10971090
}
10981091

coderd/tailnet.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,7 @@ func (s *ServerTailnet) AgentConn(ctx context.Context, agentID uuid.UUID) (works
298298
conn = workspacesdk.NewAgentConn(s.conn, workspacesdk.AgentConnOptions{
299299
AgentID: agentID,
300300
CloseFunc: func() error { return workspacesdk.ErrSkipClose },
301+
Logger: s.logger,
301302
})
302303

303304
// Since we now have an open conn, be careful to close it if we error

codersdk/workspacesdk/agentconn.go

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ type AgentConn interface {
6666
DebugMagicsock(ctx context.Context) ([]byte, error)
6767
DebugManifest(ctx context.Context) ([]byte, error)
6868
DialContext(ctx context.Context, network string, addr string) (net.Conn, error)
69+
AppHTTPClient() *http.Client
6970
GetPeerDiagnostics() tailnet.PeerDiagnostics
7071
ListContainers(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error)
7172
ListMCPTools(ctx context.Context) (ListMCPToolsResponse, error)
@@ -121,6 +122,7 @@ func (c *agentConn) SetExtraHeaders(h http.Header) {
121122
type AgentConnOptions struct {
122123
AgentID uuid.UUID
123124
CloseFunc func() error
125+
Logger slog.Logger
124126
}
125127

126128
func (c *agentConn) agentAddress() netip.Addr {
@@ -333,6 +335,24 @@ func (c *agentConn) DialContext(ctx context.Context, network string, addr string
333335
}
334336
}
335337

338+
// AppHTTPClient returns an HTTP client for reaching HTTP apps served by this
339+
// workspace agent. Redirects are blocked to prevent misuse.
340+
func (c *agentConn) AppHTTPClient() *http.Client {
341+
return &http.Client{
342+
CheckRedirect: func(*http.Request, []*http.Request) error {
343+
return http.ErrUseLastResponse
344+
},
345+
Transport: &http.Transport{
346+
// Disable keep-alives so these short-lived clients don't leave
347+
// idle connections (and their goroutines) lingering after they're
348+
// discarded.
349+
DisableKeepAlives: true,
350+
// Host locked to agent, port from URL.
351+
DialContext: c.DialContext,
352+
},
353+
}
354+
}
355+
336356
// ListeningPorts lists the ports that are currently in use by the workspace.
337357
func (c *agentConn) ListeningPorts(ctx context.Context) (codersdk.WorkspaceAgentListeningPortsResponse, error) {
338358
ctx, span := tracing.StartSpan(ctx)
@@ -1224,7 +1244,12 @@ func (c *agentConn) apiRequest(ctx context.Context, method, path string, body in
12241244
// apiClient returns an HTTP client that can be used to make
12251245
// requests to the workspace agent's HTTP API server.
12261246
func (c *agentConn) apiClient() *http.Client {
1247+
agentAddr := netip.AddrPortFrom(c.agentAddress(), AgentHTTPAPIServerPort)
12271248
return &http.Client{
1249+
// Redirects are blocked to prevent misuse.
1250+
CheckRedirect: func(*http.Request, []*http.Request) error {
1251+
return http.ErrUseLastResponse
1252+
},
12281253
Transport: &http.Transport{
12291254
// Disable keep alives as we're usually only making a single
12301255
// request, and this triggers goleak in tests
@@ -1244,16 +1269,21 @@ func (c *agentConn) apiClient() *http.Client {
12441269
return nil, xerrors.Errorf("request %q does not appear to be for http api", addr)
12451270
}
12461271

1247-
if !c.AwaitReachable(ctx) {
1248-
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
1272+
if reqAddr, err := netip.ParseAddr(host); err != nil || reqAddr != agentAddr.Addr() {
1273+
c.opts.Logger.Warn(ctx, "blocked workspace agent API request to unintended host",
1274+
slog.F("agent_id", c.opts.AgentID),
1275+
slog.F("request_host", host),
1276+
slog.F("intended_agent_addr", agentAddr.Addr()),
1277+
)
1278+
return nil, xerrors.Errorf("request host %q does not match intended agent %q", host, agentAddr.Addr())
12491279
}
12501280

1251-
ipAddr, err := netip.ParseAddr(host)
1252-
if err != nil {
1253-
return nil, xerrors.Errorf("parse host addr: %w", err)
1281+
if !c.AwaitReachable(ctx) {
1282+
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
12541283
}
12551284

1256-
conn, err := c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(ipAddr, AgentHTTPAPIServerPort))
1285+
// Always dial the pinned agent address, never the request host.
1286+
conn, err := c.Conn.DialContextTCP(ctx, agentAddr)
12571287
if err != nil {
12581288
return nil, xerrors.Errorf("dial http api: %w", err)
12591289
}
Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
package workspacesdk_test
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"net"
8+
"net/http"
9+
"net/netip"
10+
"strings"
11+
"sync/atomic"
12+
"testing"
13+
14+
"github.com/google/uuid"
15+
"github.com/stretchr/testify/assert"
16+
"github.com/stretchr/testify/require"
17+
"tailscale.com/tailcfg"
18+
19+
"github.com/coder/coder/v2/codersdk/workspacesdk"
20+
"github.com/coder/coder/v2/tailnet"
21+
"github.com/coder/coder/v2/tailnet/proto"
22+
"github.com/coder/coder/v2/tailnet/tailnettest"
23+
"github.com/coder/coder/v2/testutil"
24+
)
25+
26+
func TestAgentConnRejectsCrossAgentRedirects(t *testing.T) {
27+
t.Parallel()
28+
29+
derpMap, _ := tailnettest.RunDERPAndSTUN(t)
30+
cases := []struct {
31+
name string
32+
status int
33+
invoke func(context.Context, workspacesdk.AgentConn) error
34+
}{
35+
{
36+
name: "get 302",
37+
status: http.StatusFound,
38+
invoke: func(ctx context.Context, conn workspacesdk.AgentConn) error {
39+
_, err := conn.ListeningPorts(ctx)
40+
return err
41+
},
42+
},
43+
{
44+
name: "post 307",
45+
status: http.StatusTemporaryRedirect,
46+
invoke: func(ctx context.Context, conn workspacesdk.AgentConn) error {
47+
return conn.WriteFile(ctx, "/tmp/attacker", strings.NewReader("redirect-body"))
48+
},
49+
},
50+
{
51+
name: "post 308",
52+
status: http.StatusPermanentRedirect,
53+
invoke: func(ctx context.Context, conn workspacesdk.AgentConn) error {
54+
return conn.WriteFile(ctx, "/tmp/attacker", strings.NewReader("redirect-body"))
55+
},
56+
},
57+
}
58+
59+
for _, tc := range cases {
60+
t.Run(tc.name, func(t *testing.T) {
61+
t.Parallel()
62+
63+
ctx := testutil.Context(t, testutil.WaitMedium)
64+
65+
clientID := uuid.New()
66+
attackerID := uuid.New()
67+
victimID := uuid.New()
68+
clientConn, _ := newTailnetConn(t, derpMap, clientID, "client")
69+
attackerConn, attackerIP := newTailnetConn(t, derpMap, attackerID, "attacker")
70+
victimConn, victimIP := newTailnetConn(t, derpMap, victimID, "victim")
71+
stitchTailnet(t, map[uuid.UUID]*tailnet.Conn{
72+
clientID: clientConn,
73+
attackerID: attackerConn,
74+
victimID: victimConn,
75+
})
76+
77+
var victimHit atomic.Bool
78+
victimRouter := http.NewServeMux()
79+
victimRouter.HandleFunc("/api/v0/listening-ports", func(rw http.ResponseWriter, _ *http.Request) {
80+
victimHit.Store(true)
81+
rw.Header().Set("Content-Type", "application/json")
82+
_, _ = rw.Write([]byte(`{"ports":[]}`))
83+
})
84+
victimRouter.HandleFunc("/api/v0/write-file", func(rw http.ResponseWriter, _ *http.Request) {
85+
victimHit.Store(true)
86+
rw.WriteHeader(http.StatusOK)
87+
})
88+
serveTailnetHTTP(t, victimConn, victimRouter)
89+
90+
victimBaseURL := fmt.Sprintf("http://[%s]:%d", victimIP, workspacesdk.AgentHTTPAPIServerPort)
91+
attackerRouter := http.NewServeMux()
92+
attackerRouter.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
93+
http.Redirect(rw, r, victimBaseURL+r.URL.RequestURI(), tc.status)
94+
})
95+
serveTailnetHTTP(t, attackerConn, attackerRouter)
96+
97+
require.True(t, clientConn.AwaitReachable(ctx, attackerIP))
98+
require.True(t, clientConn.AwaitReachable(ctx, victimIP))
99+
100+
conn := workspacesdk.NewAgentConn(clientConn, workspacesdk.AgentConnOptions{
101+
AgentID: attackerID,
102+
})
103+
104+
err := tc.invoke(ctx, conn)
105+
require.Error(t, err)
106+
require.False(t, victimHit.Load())
107+
})
108+
}
109+
}
110+
111+
// TestAgentConnAppHTTPClientRefusesRedirects verifies the app HTTP client does
112+
// not follow redirects.
113+
func TestAgentConnAppHTTPClientRefusesRedirects(t *testing.T) {
114+
t.Parallel()
115+
116+
tailnetConn, err := tailnet.NewConn(&tailnet.Options{
117+
Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.RandomPrefix()},
118+
Logger: testutil.Logger(t),
119+
})
120+
require.NoError(t, err)
121+
t.Cleanup(func() {
122+
_ = tailnetConn.Close()
123+
})
124+
125+
conn := workspacesdk.NewAgentConn(tailnetConn, workspacesdk.AgentConnOptions{
126+
AgentID: uuid.New(),
127+
})
128+
129+
client := conn.AppHTTPClient()
130+
require.NotNil(t, client.CheckRedirect)
131+
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://example.invalid/", nil)
132+
require.NoError(t, err)
133+
require.ErrorIs(t, client.CheckRedirect(req, nil), http.ErrUseLastResponse)
134+
}
135+
136+
func newTailnetConn(t *testing.T, derpMap *tailcfg.DERPMap, id uuid.UUID, name string) (*tailnet.Conn, netip.Addr) {
137+
t.Helper()
138+
139+
addr := tailnet.TailscaleServicePrefix.AddrFromUUID(id)
140+
conn, err := tailnet.NewConn(&tailnet.Options{
141+
ID: id,
142+
Addresses: []netip.Prefix{netip.PrefixFrom(addr, 128)},
143+
Logger: testutil.Logger(t).Named(name),
144+
DERPMap: derpMap,
145+
})
146+
require.NoError(t, err)
147+
t.Cleanup(func() {
148+
assert.NoError(t, conn.Close())
149+
})
150+
151+
return conn, addr
152+
}
153+
154+
func serveTailnetHTTP(t *testing.T, conn *tailnet.Conn, handler http.Handler) {
155+
t.Helper()
156+
157+
ln, err := conn.Listen("tcp", fmt.Sprintf(":%d", workspacesdk.AgentHTTPAPIServerPort))
158+
require.NoError(t, err)
159+
160+
server := &http.Server{Handler: handler, ReadHeaderTimeout: testutil.WaitShort}
161+
t.Cleanup(func() {
162+
assert.NoError(t, server.Close())
163+
assert.NoError(t, ln.Close())
164+
})
165+
166+
go func() {
167+
err := server.Serve(ln)
168+
if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) {
169+
assert.NoError(t, err)
170+
}
171+
}()
172+
}
173+
174+
// stitchTailnet cross-programs every conn's node into every other conn, the
175+
// N-peer analog of tailnet's stitch test helper, so the peers can reach each
176+
// other without a coordinator.
177+
func stitchTailnet(t *testing.T, conns map[uuid.UUID]*tailnet.Conn) {
178+
t.Helper()
179+
180+
sendNode := func(srcID uuid.UUID, node *tailnet.Node) {
181+
protoNode, err := tailnet.NodeToProto(node)
182+
if !assert.NoError(t, err) {
183+
return
184+
}
185+
for dstID, dst := range conns {
186+
if dstID == srcID {
187+
continue
188+
}
189+
err = dst.UpdatePeers([]*proto.CoordinateResponse_PeerUpdate{{
190+
Id: srcID[:],
191+
Node: protoNode,
192+
Kind: proto.CoordinateResponse_PeerUpdate_NODE,
193+
}})
194+
assert.NoError(t, err)
195+
}
196+
}
197+
198+
for srcID, src := range conns {
199+
src.SetNodeCallback(func(node *tailnet.Node) {
200+
sendNode(srcID, node)
201+
})
202+
if node := src.Node(); node != nil {
203+
sendNode(srcID, node)
204+
}
205+
}
206+
207+
t.Cleanup(func() {
208+
for _, conn := range conns {
209+
conn.SetNodeCallback(nil)
210+
}
211+
})
212+
}

codersdk/workspacesdk/agentconnmock/agentconnmock.go

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

codersdk/workspacesdk/workspacesdk.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@ func (c *Client) DialAgent(dialCtx context.Context, agentID uuid.UUID, options *
295295
<-controller.Closed()
296296
return conn.Close()
297297
},
298+
Logger: options.Logger,
298299
})
299300

300301
if !agentConn.AwaitReachable(dialCtx) {

0 commit comments

Comments
 (0)