From be08606413a4392d04d3e388ccc1edbe64439c14 Mon Sep 17 00:00:00 2001 From: Kemal <223029+disq@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:19:09 +0100 Subject: [PATCH 1/7] feat: Show plugin version in plugin server logs (#2124) This could make certain things easier to debug. --- plugin/plugin.go | 4 ++++ serve/info.go | 4 +--- serve/plugin.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugin/plugin.go b/plugin/plugin.go index 2d0771d42f..9a482041ea 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -166,6 +166,10 @@ func (p *Plugin) Meta() Meta { } } +func (p *Plugin) PackageAndVersion() string { + return fmt.Sprintf("%s/%s/%s@%s", p.team, p.kind, p.name, p.version) +} + // SetSkipUsageClient sets whether the usage client should be skipped func (p *Plugin) SetSkipUsageClient(v bool) { p.skipUsageClient = v diff --git a/serve/info.go b/serve/info.go index a087f61e1c..53cc5e0d9f 100644 --- a/serve/info.go +++ b/serve/info.go @@ -1,8 +1,6 @@ package serve import ( - "fmt" - "github.com/spf13/cobra" ) @@ -18,7 +16,7 @@ func (s *PluginServe) newCmdPluginInfo() *cobra.Command { Long: pluginInfoLong, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, _ []string) error { - cmd.Println(fmt.Sprintf("Package and version: %s/%s/%s@%s", s.plugin.Team(), s.plugin.Kind(), s.plugin.Name(), s.plugin.Version())) + cmd.Println("Package and version:", s.plugin.PackageAndVersion()) return nil }, } diff --git a/serve/plugin.go b/serve/plugin.go index 9f32995190..b38d44c8a5 100644 --- a/serve/plugin.go +++ b/serve/plugin.go @@ -222,7 +222,7 @@ func (s *PluginServe) newCmdPluginServe() *cobra.Command { } }() - logger.Info().Str("address", listener.Addr().String()).Msg("Plugin server listening") + logger.Info().Str("address", listener.Addr().String()).Str("plugin", s.plugin.PackageAndVersion()).Msg("Plugin server listening") if err := grpcServer.Serve(listener); err != nil { return fmt.Errorf("failed to serve: %w", err) } From fa285f05cf65dc3ff09566d80553437b806cdd77 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 22 Apr 2025 19:44:52 +0100 Subject: [PATCH 2/7] chore(deps): Update module github.com/cloudquery/plugin-sdk/v4 to v4.78.0 (#2127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-sdk/v4](https://redirect.github.com/cloudquery/plugin-sdk) | require | minor | `v4.77.0` -> `v4.78.0` | --- ### Release Notes

cloudquery/plugin-sdk (github.com/cloudquery/plugin-sdk/v4)

### [`v4.78.0`](https://redirect.github.com/cloudquery/plugin-sdk/releases/tag/v4.78.0) [Compare Source](https://redirect.github.com/cloudquery/plugin-sdk/compare/v4.77.0...v4.78.0) ##### Features - Add logger to context ([#2125](https://redirect.github.com/cloudquery/plugin-sdk/issues/2125)) ([718e8ed](https://redirect.github.com/cloudquery/plugin-sdk/commit/718e8ed781fb27130636a87c76bfeb6c00348383)) ##### Bug Fixes - **deps:** Update aws-sdk-go-v2 monorepo ([#2119](https://redirect.github.com/cloudquery/plugin-sdk/issues/2119)) ([5554039](https://redirect.github.com/cloudquery/plugin-sdk/commit/5554039d4358a66f21e765b8dc7c3203b7437f04)) - **deps:** Update aws-sdk-go-v2 monorepo ([#2121](https://redirect.github.com/cloudquery/plugin-sdk/issues/2121)) ([7b54577](https://redirect.github.com/cloudquery/plugin-sdk/commit/7b54577964b523aba6ca93497d65c8bad6132149)) - **deps:** Update aws-sdk-go-v2 monorepo ([#2123](https://redirect.github.com/cloudquery/plugin-sdk/issues/2123)) ([8f370f8](https://redirect.github.com/cloudquery/plugin-sdk/commit/8f370f80da7ba9f8c48896aa31414bf16c57fbf1)) - **deps:** Update Google Golang modules ([#2118](https://redirect.github.com/cloudquery/plugin-sdk/issues/2118)) ([93d9203](https://redirect.github.com/cloudquery/plugin-sdk/commit/93d9203936fb499ab516fe9e847e078e758afb36)) - **deps:** Update module golang.org/x/net to v0.38.0 \[SECURITY] ([#2122](https://redirect.github.com/cloudquery/plugin-sdk/issues/2122)) ([0b0e187](https://redirect.github.com/cloudquery/plugin-sdk/commit/0b0e18763cccad1d01cf61c7a8b4c6c5e5ec343c))

--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://redirect.github.com/renovatebot/renovate). --- examples/simple_plugin/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index 3412ba762d..8a300963f3 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.1 require ( github.com/apache/arrow-go/v18 v18.2.0 - github.com/cloudquery/plugin-sdk/v4 v4.77.0 + github.com/cloudquery/plugin-sdk/v4 v4.78.0 github.com/rs/zerolog v1.33.0 ) From 2387b5765133a03e202f332290bfd94c2ac50eab Mon Sep 17 00:00:00 2001 From: Marcel <144216124+maaarcelino@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:58:42 +0200 Subject: [PATCH 3/7] feat: Add transformer to update table description with its table options (#2128) Adds code we commonly reuse in plugins to add the table options to the table description --- docs/table_options.go | 52 +++++++++++++++++++++++++++++ docs/table_options_test.go | 63 +++++++++++++++++++++++++++++++++++ docs/testdata/schema.json | 44 ++++++++++++++++++++++++ docs/testdata/tableoptions.go | 5 +++ examples/simple_plugin/go.mod | 3 +- examples/simple_plugin/go.sum | 2 ++ go.mod | 5 ++- go.sum | 6 ++-- 8 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 docs/table_options.go create mode 100644 docs/table_options_test.go create mode 100644 docs/testdata/schema.json create mode 100644 docs/testdata/tableoptions.go diff --git a/docs/table_options.go b/docs/table_options.go new file mode 100644 index 0000000000..d9b3f45d7a --- /dev/null +++ b/docs/table_options.go @@ -0,0 +1,52 @@ +package docs + +import ( + "encoding/json" + "errors" + "reflect" + "regexp" + "strings" + + schemaDocs "github.com/cloudquery/codegen/jsonschema/docs" + "github.com/cloudquery/plugin-sdk/v4/schema" + invoschema "github.com/invopop/jsonschema" +) + +func TableOptionsDescriptionTransformer(tableOptions any, jsonSchema string) (schema.Transform, error) { + var sc invoschema.Schema + if err := json.Unmarshal([]byte(jsonSchema), &sc); err != nil { + return nil, err + } + tableNamesToOptionsDocs := make(map[string]string) + tableOptionsType := reflect.ValueOf(tableOptions).Elem().Type() + for i := range tableOptionsType.NumField() { + field := tableOptionsType.Field(i) + fieldType := field.Type.String() + if strings.Contains(fieldType, ".") { + fieldType = strings.Split(fieldType, ".")[1] + } + defValue, ok := sc.Definitions[fieldType] + if !ok { + return nil, errors.New("definition not found for " + field.Name) + } + tableName := strings.Split(field.Tag.Get("json"), ",")[0] + if tableName == "" { + return nil, errors.New("json tag not found for table " + field.Name) + } + newRoot := sc + newRoot.ID = "Table Options" + newRoot.Ref = "#/$defs/" + "Table Options" + newRoot.Definitions["Table Options"] = defValue + sch, _ := json.Marshal(newRoot) + doc, _ := schemaDocs.Generate(sch, 1) + tocRegex := regexp.MustCompile(`# Table of contents[\s\S]+?##`) + tableNamesToOptionsDocs[tableName] = tocRegex.ReplaceAllString(doc, "##") + } + + return func(table *schema.Table) error { + if tableNamesToOptionsDocs[table.Name] != "" { + table.Description = table.Description + "\n\n" + tableNamesToOptionsDocs[table.Name] + } + return nil + }, nil +} diff --git a/docs/table_options_test.go b/docs/table_options_test.go new file mode 100644 index 0000000000..e7a34cbf4b --- /dev/null +++ b/docs/table_options_test.go @@ -0,0 +1,63 @@ +package docs + +import ( + _ "embed" + "testing" + + "github.com/cloudquery/plugin-sdk/v4/docs/testdata" + "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/stretchr/testify/require" +) + +//go:embed testdata/schema.json +var testSchema string + +type testTableOptions struct { + Dummy testdata.DummyTableOptions `json:"dummy,omitempty"` +} + +var testTable = &schema.Table{ + Name: "dummy", + Description: "This is a dummy table", +} + +func TestTableOptionsDescriptionTransformer(t *testing.T) { + type args struct { + tableOptions any + jsonSchema string + table *schema.Table + } + tests := []struct { + name string + args args + wantDesc string + wantErr bool + }{ + { + name: "adds table options to description", + args: args{tableOptions: &testTableOptions{}, jsonSchema: testSchema, table: testTable}, + wantDesc: "This is a dummy table\n\n## Table Options\n\n DummyTableOptions contains configuration for the dummy table\n\n* `filter` (`string`)\n", + }, + { + name: "leaves description unchanged when table doesn't have options", + args: args{tableOptions: &testTableOptions{}, jsonSchema: testSchema, table: &schema.Table{Description: "Foobar"}}, + wantDesc: "Foobar", + }, + { + name: "errors out when table options don't match schema", + wantErr: true, + args: args{tableOptions: &testTableOptions{}, jsonSchema: "", table: testTable}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + transformer, err := TableOptionsDescriptionTransformer(tt.args.tableOptions, tt.args.jsonSchema) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, transformer(tt.args.table)) + require.Equal(t, tt.wantDesc, tt.args.table.Description) + }) + } +} diff --git a/docs/testdata/schema.json b/docs/testdata/schema.json new file mode 100644 index 0000000000..b595918ff1 --- /dev/null +++ b/docs/testdata/schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "test/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "table_options": { + "$ref": "#/$defs/TableOptions", + "description": "TableOptions is a set of options to override the defaults for certain tables." + } + }, + "additionalProperties": false, + "type": "object" + }, + "TableOptions": { + "properties": { + "dummy": { + "oneOf": [ + { + "$ref": "#/$defs/DummyTableOptions", + "description": "Options for the dummy table." + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DummyTableOptions": { + "properties": { + "filter": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "description": "DummyTableOptions contains configuration for the dummy table" + } + } +} diff --git a/docs/testdata/tableoptions.go b/docs/testdata/tableoptions.go new file mode 100644 index 0000000000..cc6a7a8f4d --- /dev/null +++ b/docs/testdata/tableoptions.go @@ -0,0 +1,5 @@ +package testdata + +type DummyTableOptions struct { + Filter string `json:"filter,omitempty"` +} diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index 8a300963f3..490d1957fc 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -1,6 +1,6 @@ module github.com/cloudquery/plugin-sdk/examples/simple_plugin -go 1.23.0 +go 1.23.4 toolchain go1.24.1 @@ -33,6 +33,7 @@ require ( github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudquery/cloudquery-api-go v1.13.8 // indirect + github.com/cloudquery/codegen v0.3.26 // indirect github.com/cloudquery/plugin-pb-go v1.26.9 // indirect github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/examples/simple_plugin/go.sum b/examples/simple_plugin/go.sum index 540762d4a4..fcb8f1a9d9 100644 --- a/examples/simple_plugin/go.sum +++ b/examples/simple_plugin/go.sum @@ -52,6 +52,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0= github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA= +github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY= +github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs= github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA= github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= diff --git a/go.mod b/go.mod index dc44116a86..24dbe742a7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cloudquery/plugin-sdk/v4 -go 1.23.0 +go 1.23.4 toolchain go1.24.1 @@ -65,6 +65,7 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cloudquery/codegen v0.3.26 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -97,3 +98,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 diff --git a/go.sum b/go.sum index 2c7cf6a0e8..6b280bc069 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,10 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0= github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA= +github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY= +github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs= +github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0= +github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA= github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -99,8 +103,6 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= -github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= From a65b101d05ee43bc1f1f0033736ede756ee55604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20St=C4=99pie=C5=84?= Date: Mon, 28 Apr 2025 11:18:29 +0200 Subject: [PATCH 4/7] fix: Prevent deadlock in transformer (#2130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Summary Currently sync method has 3 gorutines running in single errorGroup let's call them G1,G2 and G3 G1- ``` eg.Go(func() error { if err := s.Plugin.Transform(gctx, recvRecords, sendRecords); err != nil { return status.Error(codes.Internal, err.Error()) } return nil }) ``` runs the transformer logic which consumes records from recvRecords and pushes them to sendRecords both of which are unbuffered channels. Gorutine g2 consumes record produced by G1 ``` record := range sendRecords (..) ``` when gorutine g2 returns an error, code executing in G1 may hang indefinietly trying to write into channel that has no associated error. --- Use the following steps to ensure your PR is ready to be reviewed - [ ] Read the [contribution guidelines](../blob/main/CONTRIBUTING.md) 🧑‍🎓 - [ ] Run `go fmt` to format your code 🖊 - [ ] Lint your changes via `golangci-lint run` 🚨 (install golangci-lint [here](https://golangci-lint.run/usage/install/#local-installation)) - [ ] Update or add tests 🧪 - [ ] Ensure the status checks below are successful ✅ --- internal/servers/plugin/v3/plugin.go | 13 +- internal/servers/plugin/v3/plugin_test.go | 144 ++++++++++++++++++++++ 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/internal/servers/plugin/v3/plugin.go b/internal/servers/plugin/v3/plugin.go index c84771597b..83dcc9a0c6 100644 --- a/internal/servers/plugin/v3/plugin.go +++ b/internal/servers/plugin/v3/plugin.go @@ -427,16 +427,23 @@ func (s *Server) Transform(stream pb.Plugin_TransformServer) error { // The reading never closes the writer, because it's up to the Plugin to decide when to finish // writing, regardless of if the reading finished. eg.Go(func() error { + var sendErr error for record := range sendRecords { + // We cannot terminate the stream here, because the plugin may still be sending records. So if error was returned channel has to be drained + if sendErr != nil { + continue + } recordBytes, err := pb.RecordToBytes(record) if err != nil { - return status.Errorf(codes.Internal, "failed to convert record to bytes: %v", err) + sendErr = status.Errorf(codes.Internal, "failed to convert record to bytes: %v", err) + continue } if err := stream.Send(&pb.Transform_Response{Record: recordBytes}); err != nil { - return status.Errorf(codes.Internal, "error sending response: %v", err) + sendErr = status.Errorf(codes.Internal, "error sending response: %v", err) + continue } } - return nil + return sendErr }) // Read records from source to transformer diff --git a/internal/servers/plugin/v3/plugin_test.go b/internal/servers/plugin/v3/plugin_test.go index b6cf41302a..5ad3c47e13 100644 --- a/internal/servers/plugin/v3/plugin_test.go +++ b/internal/servers/plugin/v3/plugin_test.go @@ -2,8 +2,12 @@ package plugin import ( "context" + "errors" "io" + "strings" + "sync/atomic" "testing" + "time" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" @@ -248,3 +252,143 @@ func (*mockSourceColumnAdderPluginClient) TransformSchema(_ context.Context, old return old.AddField(1, arrow.Field{Name: "source", Type: arrow.BinaryTypes.String}) } func (*mockSourceColumnAdderPluginClient) Close(context.Context) error { return nil } + +type testTransformPluginClient struct { + plugin.UnimplementedDestination + plugin.UnimplementedSource + recordsSent int32 +} + +func (c *testTransformPluginClient) Transform(ctx context.Context, recvRecords <-chan arrow.Record, sendRecords chan<- arrow.Record) error { + for record := range recvRecords { + select { + default: + time.Sleep(1 * time.Second) + sendRecords <- record + atomic.AddInt32(&c.recordsSent, 1) + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + +func (*testTransformPluginClient) TransformSchema(_ context.Context, old *arrow.Schema) (*arrow.Schema, error) { + return old, nil +} + +func (*testTransformPluginClient) Close(context.Context) error { + return nil +} + +func TestTransformNoDeadlockOnSendError(t *testing.T) { + client := &testTransformPluginClient{} + p := plugin.NewPlugin("test", "development", func(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions) (plugin.Client, error) { + return client, nil + }) + s := Server{ + Plugin: p, + } + _, err := s.Init(context.Background(), &pb.Init_Request{}) + require.NoError(t, err) + + // Create a channel to signal when Send was called + sendCalled := make(chan struct{}) + // Create a channel to signal when we should return from the test + done := make(chan struct{}) + defer close(done) + + stream := &mockTransformServerWithBlockingSend{ + incomingMessages: makeRequests(3), // Multiple messages to ensure Transform tries to keep sending + sendCalled: sendCalled, + done: done, + } + + // Run Transform in a goroutine with a timeout + errCh := make(chan error) + go func() { + errCh <- s.Transform(stream) + }() + + // Wait for the first Send to be called + select { + case <-sendCalled: + // Send was called, good + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting for Send to be called") + } + + // Now wait for Transform to complete or timeout + select { + case err := <-errCh: + require.Error(t, err) + // Check for either the simulated error or context cancellation + if !strings.Contains(err.Error(), "simulated stream send error") && + !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("unexpected error: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Transform got deadlocked") + } +} + +type mockTransformServerWithBlockingSend struct { + grpc.ServerStream + incomingMessages []*pb.Transform_Request + sendCalled chan struct{} + done chan struct{} + sendCount int32 +} + +func (s *mockTransformServerWithBlockingSend) Recv() (*pb.Transform_Request, error) { + if len(s.incomingMessages) > 0 { + msg := s.incomingMessages[0] + s.incomingMessages = s.incomingMessages[1:] + return msg, nil + } + return nil, io.EOF +} + +func (s *mockTransformServerWithBlockingSend) Send(*pb.Transform_Response) error { + // Signal that Send was called + select { + case s.sendCalled <- struct{}{}: + default: + } + + // Return error on first send + if atomic.AddInt32(&s.sendCount, 1) == 1 { + return errors.New("simulated stream send error") + } + + // Block until test is done + <-s.done + return nil +} + +func (*mockTransformServerWithBlockingSend) Context() context.Context { + return context.Background() +} + +func makeRequests(i int) []*pb.Transform_Request { + requests := make([]*pb.Transform_Request, i) + for i := range i { + requests[i] = makeRequestFromString("test") + } + return requests +} + +func makeRequestFromString(s string) *pb.Transform_Request { + record := makeRecordFromString(s) + bs, _ := pb.RecordToBytes(record) + return &pb.Transform_Request{Record: bs} +} + +func makeRecordFromString(s string) arrow.Record { + str := array.NewStringBuilder(memory.DefaultAllocator) + str.AppendString(s) + arr := str.NewStringArray() + sch := arrow.NewSchema([]arrow.Field{{Name: "col1", Type: arrow.BinaryTypes.String}}, nil) + + return array.NewRecord(sch, []arrow.Array{arr}, 1) +} From 12beabc8c58b1f191bf6db78d52d5b6d83e8e6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20St=C4=99pie=C5=84?= Date: Mon, 28 Apr 2025 14:38:59 +0200 Subject: [PATCH 5/7] chore: Double the limit of grpc messages (#2131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Summary Following up on the transformer hanging issue reported by a customer and addressed in PR [#2130](https://github.com/cloudquery/plugin-sdk/pull/2130): Currently, the transformer plugin can increase the message size beyond the limit enforced in the sync method of plugin-sdk [1](https://github.com/cloudquery/plugin-sdk/blob/bc50fd3d2be414edba8f8ad5bb7739a012840bf1/internal/servers/plugin/v3/plugin.go#L267). To accommodate scenarios where transformers or destination plugins need to handle larger messages than the source plugin, we should consider increasing this limit to provide more flexibility. --- Use the following steps to ensure your PR is ready to be reviewed - [ ] Read the [contribution guidelines](../blob/main/CONTRIBUTING.md) 🧑‍🎓 - [ ] Run `go fmt` to format your code 🖊 - [ ] Lint your changes via `golangci-lint run` 🚨 (install golangci-lint [here](https://golangci-lint.run/usage/install/#local-installation)) - [ ] Update or add tests 🧪 - [ ] Ensure the status checks below are successful ✅ --------- Co-authored-by: Bartosz Lesniewski --- examples/simple_plugin/go.mod | 2 +- examples/simple_plugin/go.sum | 4 ++-- go.mod | 4 ++-- go.sum | 4 ++-- serve/constants.go | 6 +++--- serve/plugin.go | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index 490d1957fc..2f1b4cde23 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -34,7 +34,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudquery/cloudquery-api-go v1.13.8 // indirect github.com/cloudquery/codegen v0.3.26 // indirect - github.com/cloudquery/plugin-pb-go v1.26.9 // indirect + github.com/cloudquery/plugin-pb-go v1.26.10 // indirect github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect diff --git a/examples/simple_plugin/go.sum b/examples/simple_plugin/go.sum index fcb8f1a9d9..8ec16f7f29 100644 --- a/examples/simple_plugin/go.sum +++ b/examples/simple_plugin/go.sum @@ -54,8 +54,8 @@ github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA= github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY= github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs= -github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA= -github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA= +github.com/cloudquery/plugin-pb-go v1.26.10 h1:VNRk3JMLR7+pCXGCk4729I8r3vTrn64qonCs+4KY7+M= +github.com/cloudquery/plugin-pb-go v1.26.10/go.mod h1:OVk5GXAlz8u6+sNlBgYxzpP+pYU+TqyyQV6FqMBBcIM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/go.mod b/go.mod index 24dbe742a7..254993412e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.29.0 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/cloudquery/cloudquery-api-go v1.13.8 - github.com/cloudquery/plugin-pb-go v1.26.9 + github.com/cloudquery/codegen v0.3.26 + github.com/cloudquery/plugin-pb-go v1.26.10 github.com/cloudquery/plugin-sdk/v2 v2.7.0 github.com/goccy/go-json v0.10.5 github.com/golang/mock v1.6.0 @@ -65,7 +66,6 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cloudquery/codegen v0.3.26 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum index 6b280bc069..265557a509 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMp github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs= github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0= github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= -github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA= -github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA= +github.com/cloudquery/plugin-pb-go v1.26.10 h1:VNRk3JMLR7+pCXGCk4729I8r3vTrn64qonCs+4KY7+M= +github.com/cloudquery/plugin-pb-go v1.26.10/go.mod h1:OVk5GXAlz8u6+sNlBgYxzpP+pYU+TqyyQV6FqMBBcIM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/serve/constants.go b/serve/constants.go index 95971f7dd1..04bf214283 100644 --- a/serve/constants.go +++ b/serve/constants.go @@ -6,7 +6,7 @@ import ( const ( // bufSize used for unit testing grpc server and client - testBufSize = 1024 * 1024 - flushTimeout = 5 * time.Second - MaxMsgSize = 100 * 1024 * 1024 // 100 MiB + testBufSize = 1024 * 1024 + flushTimeout = 5 * time.Second + MaxGrpcMsgSize = 200 * 1024 * 1024 // 200 MiB ) diff --git a/serve/plugin.go b/serve/plugin.go index b38d44c8a5..3038e46b78 100644 --- a/serve/plugin.go +++ b/serve/plugin.go @@ -179,8 +179,8 @@ func (s *PluginServe) newCmdPluginServe() *cobra.Command { grpc.ChainStreamInterceptor( logging.StreamServerInterceptor(grpczerolog.InterceptorLogger(logger)), ), - grpc.MaxRecvMsgSize(MaxMsgSize), - grpc.MaxSendMsgSize(MaxMsgSize), + grpc.MaxRecvMsgSize(MaxGrpcMsgSize), + grpc.MaxSendMsgSize(MaxGrpcMsgSize), ) s.plugin.SetLogger(logger) pbv3.RegisterPluginServer(grpcServer, &serversv3.Server{ From 775d537cc9ee69ee548d69be67e7172baa03fcab Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:46:41 +0100 Subject: [PATCH 6/7] fix(deps): Update module github.com/cloudquery/plugin-pb-go to v1.26.10 (#2132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-pb-go](https://redirect.github.com/cloudquery/plugin-pb-go) | require | patch | `v1.26.9` -> `v1.26.10` | --- ### Release Notes

cloudquery/plugin-pb-go (github.com/cloudquery/plugin-pb-go)

### [`v1.26.10`](https://redirect.github.com/cloudquery/plugin-pb-go/releases/tag/v1.26.10) [Compare Source](https://redirect.github.com/cloudquery/plugin-pb-go/compare/v1.26.9...v1.26.10) ##### Bug Fixes - **deps:** Update Google Golang modules ([#491](https://redirect.github.com/cloudquery/plugin-pb-go/issues/491)) ([7d5e53b](https://redirect.github.com/cloudquery/plugin-pb-go/commit/7d5e53bcbaf6dd5dab2d6dafc53bbb425520a528)) - **deps:** Update module golang.org/x/net to v0.38.0 \[SECURITY] ([#493](https://redirect.github.com/cloudquery/plugin-pb-go/issues/493)) ([9c3349a](https://redirect.github.com/cloudquery/plugin-pb-go/commit/9c3349afc4276be5149be4d232d7f0a6a8b8d8ec))