coder · Emyrk · May 9, 2025 · May 9, 2025 · May 9, 2025 · May 9, 2025
diff --git a/.gitignore b/.gitignore
@@ -50,6 +50,7 @@ site/stats/
 *.tfplan
 *.lock.hcl
 .terraform/
+!provisioner/terraform/testdata/modules-source-caching/.terraform/
 
 **/.coderv2/*
 **/__debug_bin

diff --git a/agent/agenttest/client.go b/agent/agenttest/client.go
@@ -16,6 +16,7 @@ import (
 	"golang.org/x/xerrors"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/emptypb"
+	"storj.io/drpc/drpcmanager"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 	"tailscale.com/tailcfg"
@@ -60,6 +61,7 @@ func NewClient(t testing.TB,
 	err = agentproto.DRPCRegisterAgent(mux, fakeAAPI)
 	require.NoError(t, err)
 	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Manager: drpcmanager.Options{WriterBufferSize: drpcsdk.MaxMessageSize},
 		Log: func(err error) {
 			if xerrors.Is(err, io.EOF) {
 				return

diff --git a/cli/testdata/coder_provisioner_list_--output_json.golden b/cli/testdata/coder_provisioner_list_--output_json.golden
@@ -7,7 +7,7 @@
     "last_seen_at": "====[timestamp]=====",
     "name": "test",
     "version": "v0.0.0-devel",
-    "api_version": "1.4",
+    "api_version": "1.5",
     "provisioners": [
       "echo"
     ],

diff --git a/coderd/agentapi/api.go b/coderd/agentapi/api.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/google/uuid"
 	"golang.org/x/xerrors"
+	"storj.io/drpc/drpcmanager"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 	"tailscale.com/tailcfg"
@@ -30,6 +31,7 @@ import (
 	"github.com/coder/coder/v2/coderd/wspubsub"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/tailnet"
 	tailnetproto "github.com/coder/coder/v2/tailnet/proto"
 	"github.com/coder/quartz"
@@ -209,6 +211,7 @@ func (a *API) Server(ctx context.Context) (*drpcserver.Server, error) {
 
 	return drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
 		drpcserver.Options{
+			Manager: drpcmanager.Options{WriterBufferSize: drpcsdk.MaxMessageSize},
 			Log: func(err error) {
 				if xerrors.Is(err, io.EOF) {
 					return

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -29,6 +29,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 	"google.golang.org/api/idtoken"
+	"storj.io/drpc/drpcmanager"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 	"tailscale.com/derp"
@@ -84,7 +85,7 @@ import (
 	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/coderd/workspacestats"
 	"github.com/coder/coder/v2/codersdk"
-	"github.com/coder/coder/v2/codersdk/drpc"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/codersdk/healthsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
@@ -1725,7 +1726,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(dialCtx context.Context, name st
 
 func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, name string, provisionerTypes []codersdk.ProvisionerType, provisionerTags map[string]string) (client proto.DRPCProvisionerDaemonClient, err error) {
 	tracer := api.TracerProvider.Tracer(tracing.TracerName)
-	clientSession, serverSession := drpc.MemTransportPipe()
+	clientSession, serverSession := drpcsdk.MemTransportPipe()
 	defer func() {
 		if err != nil {
 			_ = clientSession.Close()
@@ -1803,6 +1804,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 	}
 	server := drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
 		drpcserver.Options{
+			Manager: drpcmanager.Options{WriterBufferSize: drpcsdk.MaxMessageSize},
 			Log: func(err error) {
 				if xerrors.Is(err, io.EOF) {
 					return

diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go
@@ -12,21 +12,19 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"golang.org/x/xerrors"
-
 	"github.com/open-policy-agent/opa/topdown"
+	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
 
-	"github.com/coder/coder/v2/coderd/prebuilds"
-	"github.com/coder/coder/v2/coderd/rbac/policy"
-	"github.com/coder/coder/v2/coderd/rbac/rolestore"
-
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi/httpapiconstraints"
 	"github.com/coder/coder/v2/coderd/httpmw/loggermw"
+	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/rbac"
+	"github.com/coder/coder/v2/coderd/rbac/policy"
+	"github.com/coder/coder/v2/coderd/rbac/rolestore"
 	"github.com/coder/coder/v2/coderd/util/slice"
 	"github.com/coder/coder/v2/provisionersdk"
 )
@@ -347,6 +345,7 @@ var (
 					rbac.ResourceNotificationPreference.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceNotificationTemplate.Type:   {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
 					rbac.ResourceCryptoKey.Type:              {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceFile.Type:                   {policy.ActionCreate, policy.ActionRead},
 				}),
 				Org:  map[string][]rbac.Permission{},
 				User: []rbac.Permission{},

diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go
@@ -999,9 +999,10 @@ func TemplateVersionTerraformValues(t testing.TB, db database.Store, orig databa
 	t.Helper()
 
 	params := database.InsertTemplateVersionTerraformValuesByJobIDParams{
-		JobID:      takeFirst(orig.JobID, uuid.New()),
-		CachedPlan: takeFirstSlice(orig.CachedPlan, []byte("{}")),
-		UpdatedAt:  takeFirst(orig.UpdatedAt, dbtime.Now()),
+		JobID:             takeFirst(orig.JobID, uuid.New()),
+		CachedPlan:        takeFirstSlice(orig.CachedPlan, []byte("{}")),
+		CachedModuleFiles: orig.CachedModuleFiles,
+		UpdatedAt:         takeFirst(orig.UpdatedAt, dbtime.Now()),
 	}
 
 	err := db.InsertTemplateVersionTerraformValuesByJobID(genCtx, params)

diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go
@@ -9315,6 +9315,7 @@ func (q *FakeQuerier) InsertTemplateVersionTerraformValuesByJobID(_ context.Cont
 	row := database.TemplateVersionTerraformValue{
 		TemplateVersionID: templateVersion.ID,
 		CachedPlan:        arg.CachedPlan,
+		CachedModuleFiles: arg.CachedModuleFiles,
 		UpdatedAt:         arg.UpdatedAt,
 	}
 	q.templateVersionTerraformValues = append(q.templateVersionTerraformValues, row)

diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql
diff --git a/coderd/database/foreign_key_constraint.go b/coderd/database/foreign_key_constraint.go
diff --git a/coderd/database/migrations/000320_terraform_cached_modules.down.sql b/coderd/database/migrations/000320_terraform_cached_modules.down.sql
@@ -0,0 +1 @@
+ALTER TABLE template_version_terraform_values DROP COLUMN cached_module_files;
diff --git a/coderd/database/migrations/000320_terraform_cached_modules.up.sql b/coderd/database/migrations/000320_terraform_cached_modules.up.sql
@@ -0,0 +1 @@
+ALTER TABLE template_version_terraform_values ADD COLUMN cached_module_files uuid references files(id);
diff --git a/coderd/database/models.go b/coderd/database/models.go
diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go
diff --git a/coderd/database/queries/templateversionterraformvalues.sql b/coderd/database/queries/templateversionterraformvalues.sql
@@ -11,11 +11,13 @@ INSERT INTO
 	template_version_terraform_values (
 		template_version_id,
 		cached_plan,
+		cached_module_files,
 		updated_at
 	)
 VALUES
 	(
 		(select id from template_versions where job_id = @job_id),
 		@cached_plan,
+		@cached_module_files,
 		@updated_at
 	);
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -2,7 +2,9 @@ package provisionerdserver
 
 import (
 	"context"
+	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -50,6 +52,10 @@ import (
 	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )
 
+const (
+	tarMimeType = "application/x-tar"
+)
+
 const (
 	// DefaultAcquireJobLongPollDur is the time the (deprecated) AcquireJob rpc waits to try to obtain a job before
 	// canceling and returning an empty job.
@@ -1426,11 +1432,59 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
 			return nil, xerrors.Errorf("update template version external auth providers: %w", err)
 		}
 
-		if len(jobType.TemplateImport.Plan) > 0 {
-			err := s.Database.InsertTemplateVersionTerraformValuesByJobID(ctx, database.InsertTemplateVersionTerraformValuesByJobIDParams{
-				JobID:      jobID,
-				CachedPlan: jobType.TemplateImport.Plan,
-				UpdatedAt:  now,
+		plan := jobType.TemplateImport.Plan
+		moduleFiles := jobType.TemplateImport.ModuleFiles
+		// If there is a plan, or a module files archive we need to insert a
+		// template_version_terraform_values row.
+		if len(plan) > 0 || len(moduleFiles) > 0 {
+			// ...but the plan and the module files archive are both optional! So
+			// we need to fallback to a valid JSON object if the plan was omitted.
+			if len(plan) == 0 {
+				plan = []byte("{}")
+			}
+
+			// ...and we only want to insert a files row if an archive was provided.
+			var fileID uuid.NullUUID
+			if len(moduleFiles) > 0 {
+				hashBytes := sha256.Sum256(moduleFiles)
+				hash := hex.EncodeToString(hashBytes[:])
+
+				// nolint:gocritic // Requires reading "system" files
+				file, err := s.Database.GetFileByHashAndCreator(dbauthz.AsSystemRestricted(ctx), database.GetFileByHashAndCreatorParams{Hash: hash, CreatedBy: uuid.Nil})
+				switch {
+				case err == nil:
+					// This set of modules is already cached, which means we can reuse them
+					fileID = uuid.NullUUID{
+						Valid: true,
+						UUID:  file.ID,
+					}
+				case !xerrors.Is(err, sql.ErrNoRows):
+					return nil, xerrors.Errorf("check for cached modules: %w", err)
+				default:
+					// nolint:gocritic // Requires creating a "system" file
+					file, err = s.Database.InsertFile(dbauthz.AsSystemRestricted(ctx), database.InsertFileParams{
+						ID:        uuid.New(),
+						Hash:      hash,
+						CreatedBy: uuid.Nil,
+						CreatedAt: dbtime.Now(),
+						Mimetype:  tarMimeType,
+						Data:      moduleFiles,
+					})
+					if err != nil {
+						return nil, xerrors.Errorf("insert template version terraform modules: %w", err)
+					}
+					fileID = uuid.NullUUID{
+						Valid: true,
+						UUID:  file.ID,
+					}
+				}
+			}
+
+			err = s.Database.InsertTemplateVersionTerraformValuesByJobID(ctx, database.InsertTemplateVersionTerraformValuesByJobIDParams{
+				JobID:             jobID,
+				UpdatedAt:         now,
+				CachedPlan:        plan,
+				CachedModuleFiles: fileID,
 			})
 			if err != nil {
 				return nil, xerrors.Errorf("insert template version terraform data: %w", err)

diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go
@@ -17,7 +17,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/buildinfo"
-	"github.com/coder/coder/v2/codersdk/drpc"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/codersdk/wsjson"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionerd/runner"
@@ -332,7 +332,7 @@ func (c *Client) ServeProvisionerDaemon(ctx context.Context, req ServeProvisione
 		_ = wsNetConn.Close()
 		return nil, xerrors.Errorf("multiplex client: %w", err)
 	}
-	return proto.NewDRPCProvisionerDaemonClient(drpc.MultiplexedConn(session)), nil
+	return proto.NewDRPCProvisionerDaemonClient(drpcsdk.MultiplexedConn(session)), nil
 }
 
 type ProvisionerKeyTags map[string]string

diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go
@@ -15,6 +15,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/exp/maps"
 	"golang.org/x/xerrors"
+	"storj.io/drpc/drpcmanager"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 
@@ -31,6 +32,7 @@ import (
 	"github.com/coder/coder/v2/coderd/telemetry"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/codersdk"
+	drpcsdk "github.com/coder/coder/v2/codersdk/drpc"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
 	"github.com/coder/websocket"
@@ -370,6 +372,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 		return
 	}
 	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
+		Manager: drpcmanager.Options{WriterBufferSize: drpcsdk.MaxMessageSize},
 		Log: func(err error) {
 			if xerrors.Is(err, io.EOF) {
 				return
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE template_version_terraform_values DROP COLUMN cached_module_files;