From d1b52da44faf204b56f88133e9b6b1c24b006a16 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 30 May 2023 16:23:15 +0100 Subject: [PATCH 01/10] Start migration --- plugins/source/postgresql/client/cdc.go | 2 +- plugins/source/postgresql/client/client.go | 4 +-- .../source/postgresql/client/list_tables.go | 12 +++---- plugins/source/postgresql/client/sync.go | 4 +-- plugins/source/postgresql/client/types.go | 7 ++-- plugins/source/postgresql/go.mod | 18 +++++++--- plugins/source/postgresql/go.sum | 34 +++++++++++++++---- plugins/source/postgresql/main.go | 2 +- .../postgresql/resources/plugin/plugin.go | 2 +- .../resources/plugin/plugin_test.go | 2 +- .../postgresql/resources/plugin/tables.go | 2 +- 11 files changed, 59 insertions(+), 30 deletions(-) diff --git a/plugins/source/postgresql/client/cdc.go b/plugins/source/postgresql/client/cdc.go index 1cc2ee68c72f3e..b98dbd844281b3 100644 --- a/plugins/source/postgresql/client/cdc.go +++ b/plugins/source/postgresql/client/cdc.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/schema" "github.com/jackc/pglogrepl" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" diff --git a/plugins/source/postgresql/client/client.go b/plugins/source/postgresql/client/client.go index f82c8af10ab7a8..0fa2bc16fea43e 100644 --- a/plugins/source/postgresql/client/client.go +++ b/plugins/source/postgresql/client/client.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/cloudquery/plugin-pb-go/specs" - "github.com/cloudquery/plugin-sdk/v2/plugins/source" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/plugins/source" + "github.com/cloudquery/plugin-sdk/v3/schema" pgx_zero_log "github.com/jackc/pgx-zerolog" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" diff --git a/plugins/source/postgresql/client/list_tables.go b/plugins/source/postgresql/client/list_tables.go index da3c4910fca5a7..5d84a99746a5ce 100644 --- a/plugins/source/postgresql/client/list_tables.go +++ b/plugins/source/postgresql/client/list_tables.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/schema" ) // this returns the following table in sorted manner: @@ -76,12 +76,10 @@ func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { table.PkConstraintName = pkName } table.Columns = append(table.Columns, schema.Column{ - Name: columnName, - CreationOptions: schema.ColumnCreationOptions{ - PrimaryKey: isPrimaryKey, - NotNull: notNull, - }, - Type: c.PgToSchemaType(columnType), + Name: columnName, + PrimaryKey: isPrimaryKey, + NotNull: notNull, + Type: c.PgToSchemaType(columnType), }) } return tables, nil diff --git a/plugins/source/postgresql/client/sync.go b/plugins/source/postgresql/client/sync.go index 9afea794095b2c..adb210b2f645b3 100644 --- a/plugins/source/postgresql/client/sync.go +++ b/plugins/source/postgresql/client/sync.go @@ -6,8 +6,8 @@ import ( "fmt" "strings" - "github.com/cloudquery/plugin-sdk/v2/plugins/source" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/plugins/source" + "github.com/cloudquery/plugin-sdk/v3/schema" "github.com/jackc/pgx/v5" ) diff --git a/plugins/source/postgresql/client/types.go b/plugins/source/postgresql/client/types.go index a764e107c3c0c1..27fad1c770bab2 100644 --- a/plugins/source/postgresql/client/types.go +++ b/plugins/source/postgresql/client/types.go @@ -3,6 +3,7 @@ package client import ( "strings" + "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/plugin-sdk/v2/schema" ) @@ -97,7 +98,7 @@ func (*Client) SchemaTypeToCockroach(t schema.ValueType) string { } } -func (c *Client) PgToSchemaType(t string) schema.ValueType { +func (c *Client) PgToSchemaType(t string) arrow.DataType { switch c.pgType { case pgTypeCockroachDB: return c.CockroachToSchemaType(t) @@ -106,7 +107,7 @@ func (c *Client) PgToSchemaType(t string) schema.ValueType { } } -func (*Client) Pg10ToSchemaType(t string) schema.ValueType { +func (*Client) Pg10ToSchemaType(t string) arrow.DataType { if strings.HasPrefix(t, "timestamp") { return schema.TypeTimestamp } @@ -147,7 +148,7 @@ func (*Client) Pg10ToSchemaType(t string) schema.ValueType { } } -func (*Client) CockroachToSchemaType(t string) schema.ValueType { +func (*Client) CockroachToSchemaType(t string) arrow.DataType { if strings.HasPrefix(t, "timestamp") { return schema.TypeTimestamp } diff --git a/plugins/source/postgresql/go.mod b/plugins/source/postgresql/go.mod index 99c6d6b7e35839..a53aee3d930a86 100644 --- a/plugins/source/postgresql/go.mod +++ b/plugins/source/postgresql/go.mod @@ -3,10 +3,12 @@ module github.com/cloudquery/cloudquery/plugins/source/postgresql go 1.19 require ( + github.com/apache/arrow/go/v13 v13.0.0-20230531201200-cbc17a98dfd9 github.com/cloudquery/plugin-pb-go v1.1.0 github.com/cloudquery/plugin-sdk/v2 v2.7.0 + github.com/cloudquery/plugin-sdk/v3 v3.10.6 github.com/google/uuid v1.3.0 - github.com/jackc/pglogrepl v0.0.0-20230428004623-0c5b98f52784 + github.com/jackc/pglogrepl v0.0.0-20230318140337-5ef673a9d169 github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb github.com/jackc/pgx/v5 v5.3.1 github.com/rs/zerolog v1.29.1 @@ -14,15 +16,18 @@ require ( ) // TODO: remove once all updates are merged -replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853 +replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 require ( - github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apache/thrift v0.16.0 // indirect + github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/goccy/go-json v0.10.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.1.21+incompatible // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect @@ -31,10 +36,15 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.0 // indirect + github.com/klauspost/asmfmt v1.3.2 // indirect + github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.6.1 // indirect @@ -50,7 +60,7 @@ require ( golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/plugins/source/postgresql/go.sum b/plugins/source/postgresql/go.sum index 454bd3be67e1fe..5043bd09d76267 100644 --- a/plugins/source/postgresql/go.sum +++ b/plugins/source/postgresql/go.sum @@ -33,18 +33,27 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853 h1:MeXRH104spmnswFe4dQoYsWRD0fS9HzhQ69GkSkF7a8= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853 h1:HvZJa27CHTtd4TQa26ANVdckiYAoTZbt8gndfVsBEfw= +github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853/go.mod h1:GZvXlnoq/C1N1EDil2jmVHUijhWiMb77VihjHChz3wA= +github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg= +github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po= github.com/cloudquery/plugin-pb-go v1.1.0 h1:F1r/x4aF5aO1hmgBk8rqAp2oejeYMMughTPaYosQLDk= github.com/cloudquery/plugin-pb-go v1.1.0/go.mod h1:327Dd56bQ357KNIbhZNGDoJ7jPYXsCZWZ4Tj955gU7M= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= +github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= +github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -83,6 +92,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -100,6 +110,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v23.1.21+incompatible h1:bUqzx/MXCDxuS0hRJL2EfjyZL3uQrPbMocUa8zGqsTA= @@ -141,8 +153,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pglogrepl v0.0.0-20230428004623-0c5b98f52784 h1:qk5+0FP+HrGuqOcrQB2ISaLduvBaPnXBUoJ2FsQE9wg= -github.com/jackc/pglogrepl v0.0.0-20230428004623-0c5b98f52784/go.mod h1:P5+MSYwllwjij1PDNGA4NF6hpomKWs0CmuagKUW9s0c= +github.com/jackc/pglogrepl v0.0.0-20230318140337-5ef673a9d169 h1:r3eRvbo5j+OfO8NFYRUK7q162ZVHRb5sVT/A865RpuY= +github.com/jackc/pglogrepl v0.0.0-20230318140337-5ef673a9d169/go.mod h1:P5+MSYwllwjij1PDNGA4NF6hpomKWs0CmuagKUW9s0c= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= @@ -160,7 +172,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -179,8 +194,13 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -414,7 +434,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -467,8 +487,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/plugins/source/postgresql/main.go b/plugins/source/postgresql/main.go index 5246759ff815e5..aea32666447bb4 100644 --- a/plugins/source/postgresql/main.go +++ b/plugins/source/postgresql/main.go @@ -2,7 +2,7 @@ package main import ( "github.com/cloudquery/cloudquery/plugins/source/postgresql/resources/plugin" - "github.com/cloudquery/plugin-sdk/v2/serve" + "github.com/cloudquery/plugin-sdk/v3/serve" ) const sentryDSN = "https://995c68a7e67541338e22dd8120e81c42@o1396617.ingest.sentry.io/4504316028452864" diff --git a/plugins/source/postgresql/resources/plugin/plugin.go b/plugins/source/postgresql/resources/plugin/plugin.go index 3ee86fe5f2440b..2aa88cc4f3e2c7 100644 --- a/plugins/source/postgresql/resources/plugin/plugin.go +++ b/plugins/source/postgresql/resources/plugin/plugin.go @@ -2,7 +2,7 @@ package plugin import ( "github.com/cloudquery/cloudquery/plugins/source/postgresql/client" - "github.com/cloudquery/plugin-sdk/v2/plugins/source" + "github.com/cloudquery/plugin-sdk/v3/plugins/source" ) var Version = "Development" diff --git a/plugins/source/postgresql/resources/plugin/plugin_test.go b/plugins/source/postgresql/resources/plugin/plugin_test.go index e52a0dd8de985d..04ecd02de4f752 100644 --- a/plugins/source/postgresql/resources/plugin/plugin_test.go +++ b/plugins/source/postgresql/resources/plugin/plugin_test.go @@ -13,7 +13,7 @@ import ( "github.com/cloudquery/cloudquery/plugins/source/postgresql/client" "github.com/cloudquery/plugin-pb-go/specs" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/schema" "github.com/google/uuid" pgx_zero_log "github.com/jackc/pgx-zerolog" "github.com/jackc/pgx/v5" diff --git a/plugins/source/postgresql/resources/plugin/tables.go b/plugins/source/postgresql/resources/plugin/tables.go index f696ba82d46476..343c1b8e28892b 100644 --- a/plugins/source/postgresql/resources/plugin/tables.go +++ b/plugins/source/postgresql/resources/plugin/tables.go @@ -4,7 +4,7 @@ import ( "context" "github.com/cloudquery/cloudquery/plugins/source/postgresql/client" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/plugin-sdk/v3/schema" ) func getDynamicTables(ctx context.Context, c schema.ClientMeta) (schema.Tables, error) { From 60163569cef91901742370ff3787dc4d547ae565 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 1 Jun 2023 11:02:11 +0100 Subject: [PATCH 02/10] Sync tests passing --- plugins/source/postgresql/client/cdc.go | 6 +- plugins/source/postgresql/client/sync.go | 60 +++- plugins/source/postgresql/client/types.go | 170 +---------- plugins/source/postgresql/go.mod | 1 + plugins/source/postgresql/pgarrow/to_arrow.go | 123 ++++++++ .../postgresql/pgarrow/to_arrow_test.go | 141 +++++++++ plugins/source/postgresql/pgarrow/to_pg.go | 65 ++++ .../resources/plugin/plugin_test.go | 279 +++++++++--------- 8 files changed, 536 insertions(+), 309 deletions(-) create mode 100644 plugins/source/postgresql/pgarrow/to_arrow.go create mode 100644 plugins/source/postgresql/pgarrow/to_arrow_test.go create mode 100644 plugins/source/postgresql/pgarrow/to_pg.go diff --git a/plugins/source/postgresql/client/cdc.go b/plugins/source/postgresql/client/cdc.go index b98dbd844281b3..e07fe61d558eef 100644 --- a/plugins/source/postgresql/client/cdc.go +++ b/plugins/source/postgresql/client/cdc.go @@ -282,7 +282,11 @@ func (c *Client) resourceFromCDCValues(tableName string, values map[string]any) table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) for _, col := range table.Columns { - if err := resource.Set(col.Name, values[col.Name]); err != nil { + v, err := prepareValueForResourceSet(col, values[col.Name]) + if err != nil { + return nil, err + } + if err := resource.Set(col.Name, v); err != nil { return nil, err } } diff --git a/plugins/source/postgresql/client/sync.go b/plugins/source/postgresql/client/sync.go index adb210b2f645b3..fc908c7962170e 100644 --- a/plugins/source/postgresql/client/sync.go +++ b/plugins/source/postgresql/client/sync.go @@ -2,13 +2,16 @@ package client import ( "context" + "database/sql/driver" "errors" "fmt" "strings" + "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/plugin-sdk/v3/plugins/source" "github.com/cloudquery/plugin-sdk/v3/schema" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" ) func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- *schema.Resource) error { @@ -127,9 +130,64 @@ func (c *Client) resourceFromValues(tableName string, values []any) (*schema.Res table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) for i, col := range table.Columns { - if err := resource.Set(col.Name, values[i]); err != nil { + v, err := prepareValueForResourceSet(col, values[i]) + if err != nil { + return nil, err + } + if err := resource.Set(col.Name, v); err != nil { return nil, err } } return resource, nil } + +func prepareValueForResourceSet(col schema.Column, v any) (any, error) { + switch tp := col.Type.(type) { + case *arrow.StringType: + if value, ok := v.(driver.Valuer); ok { + if value == driver.Valuer(nil) { + v = nil + } else { + val, err := value.Value() + if err != nil { + return nil, err + } + if s, ok := val.(string); ok { + v = s + } + } + } + case *arrow.Time32Type: + t, err := v.(pgtype.Time).TimeValue() + if err != nil { + return nil, err + } + v = stringForTime(t, tp.Unit) + case *arrow.Time64Type: + t, err := v.(pgtype.Time).TimeValue() + if err != nil { + return nil, err + } + v = stringForTime(t, tp.Unit) + } + return v, nil +} + +func stringForTime(t pgtype.Time, unit arrow.TimeUnit) string { + extra := "" + hour := t.Microseconds / 1e6 / 60 / 60 + minute := t.Microseconds / 1e6 / 60 % 60 + second := t.Microseconds / 1e6 % 60 + micros := t.Microseconds % 1e6 + switch unit { + case arrow.Millisecond: + extra = fmt.Sprintf(".%03d", (micros)/1e3) + case arrow.Microsecond: + extra = fmt.Sprintf(".%06d", micros) + case arrow.Nanosecond: + // postgres doesn't support nanosecond precision + extra = fmt.Sprintf(".%06d", micros) + } + + return fmt.Sprintf("%02d:%02d:%02d"+extra, hour, minute, second) +} diff --git a/plugins/source/postgresql/client/types.go b/plugins/source/postgresql/client/types.go index 27fad1c770bab2..8b943e5708cd8b 100644 --- a/plugins/source/postgresql/client/types.go +++ b/plugins/source/postgresql/client/types.go @@ -1,182 +1,24 @@ package client import ( - "strings" - "github.com/apache/arrow/go/v13/arrow" - "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/cloudquery/cloudquery/plugins/source/postgresql/pgarrow" ) -func (c *Client) SchemaTypeToPg(t schema.ValueType) string { +func (c *Client) SchemaTypeToPg(t arrow.DataType) string { switch c.pgType { case pgTypeCockroachDB: - return c.SchemaTypeToCockroach(t) - default: - return c.SchemaTypeToPg10(t) - } -} - -func (*Client) SchemaTypeToPg10(t schema.ValueType) string { - switch t { - case schema.TypeBool: - return "boolean" - case schema.TypeInt: - return "bigint" - case schema.TypeFloat: - return "double precision" - case schema.TypeUUID: - return "uuid" - case schema.TypeString: - return "text" - case schema.TypeByteArray: - return "bytea" - case schema.TypeStringArray: - return "text[]" - case schema.TypeTimestamp: - return "timestamp without time zone" - case schema.TypeJSON: - return "jsonb" - case schema.TypeUUIDArray: - return "uuid[]" - case schema.TypeCIDR: - return "cidr" - case schema.TypeCIDRArray: - return "cidr[]" - case schema.TypeMacAddr: - return "macaddr" - case schema.TypeMacAddrArray: - return "macaddr[]" - case schema.TypeInet: - return "inet" - case schema.TypeInetArray: - return "inet[]" - case schema.TypeIntArray: - return "bigint[]" + return pgarrow.ArrowToCockroach(t) default: - panic("unknown type " + t.String()) - } -} - -func (*Client) SchemaTypeToCockroach(t schema.ValueType) string { - switch t { - case schema.TypeBool: - return "boolean" - case schema.TypeInt: - return "bigint" - case schema.TypeFloat: - return "double precision" - case schema.TypeUUID: - return "uuid" - case schema.TypeString: - return "text" - case schema.TypeByteArray: - return "bytea" - case schema.TypeStringArray: - return "text[]" - case schema.TypeTimestamp: - return "timestamp without time zone" - case schema.TypeJSON: - return "jsonb" - case schema.TypeUUIDArray: - return "uuid[]" - case schema.TypeCIDR: - return "inet" - case schema.TypeCIDRArray: - return "inet[]" - case schema.TypeMacAddr: - return "text" - case schema.TypeMacAddrArray: - return "text[]" - case schema.TypeInet: - return "inet" - case schema.TypeInetArray: - return "inet[]" - case schema.TypeIntArray: - return "bigint[]" - default: - panic("unknown type " + t.String()) + return pgarrow.ArrowToPg10(t) } } func (c *Client) PgToSchemaType(t string) arrow.DataType { switch c.pgType { case pgTypeCockroachDB: - return c.CockroachToSchemaType(t) - default: - return c.Pg10ToSchemaType(t) - } -} - -func (*Client) Pg10ToSchemaType(t string) arrow.DataType { - if strings.HasPrefix(t, "timestamp") { - return schema.TypeTimestamp - } - - switch t { - case "boolean": - return schema.TypeBool - case "bigint", "integer", "bigserial", "smallint", "smallserial", "serial": - return schema.TypeInt - case "double precision", "float", "real", "numeric": - return schema.TypeFloat - case "uuid": - return schema.TypeUUID - case "bytea": - return schema.TypeByteArray - case "text[]": - return schema.TypeStringArray - case "json", "jsonb": - return schema.TypeJSON - case "uuid[]": - return schema.TypeUUIDArray - case "cidr": - return schema.TypeCIDR - case "cidr[]": - return schema.TypeCIDRArray - case "macaddr", "macaddr8": - return schema.TypeMacAddr - case "macaddr[]", "macaddr8[]": - return schema.TypeMacAddrArray - case "inet": - return schema.TypeInet - case "inet[]": - return schema.TypeInetArray - case "bigint[]", "integer[]", "smallint[]", "bigserial[]", "smallserial[]", "serial[]": - return schema.TypeIntArray - default: - return schema.TypeString - } -} - -func (*Client) CockroachToSchemaType(t string) arrow.DataType { - if strings.HasPrefix(t, "timestamp") { - return schema.TypeTimestamp - } - - switch t { - case "boolean": - return schema.TypeBool - case "bigint", "int", "oid", "serial": - return schema.TypeInt - case "decimal", "float": - return schema.TypeFloat - case "uuid": - return schema.TypeUUID - case "bytea": - return schema.TypeByteArray - case "text[]": - return schema.TypeStringArray - case "jsonb": - return schema.TypeJSON - case "uuid[]": - return schema.TypeUUIDArray - case "inet": - return schema.TypeInet - case "inet[]": - return schema.TypeInetArray - case "bigint[]": - return schema.TypeIntArray + return pgarrow.Pg10ToCockroach(t) default: - return schema.TypeString + return pgarrow.Pg10ToArrow(t) } } diff --git a/plugins/source/postgresql/go.mod b/plugins/source/postgresql/go.mod index a53aee3d930a86..3427e14d246706 100644 --- a/plugins/source/postgresql/go.mod +++ b/plugins/source/postgresql/go.mod @@ -17,6 +17,7 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 +replace github.com/cloudquery/plugin-sdk/v3 => ../../../../plugin-sdk require ( github.com/andybalholm/brotli v1.0.5 // indirect diff --git a/plugins/source/postgresql/pgarrow/to_arrow.go b/plugins/source/postgresql/pgarrow/to_arrow.go new file mode 100644 index 00000000000000..27283d420ce7c6 --- /dev/null +++ b/plugins/source/postgresql/pgarrow/to_arrow.go @@ -0,0 +1,123 @@ +package pgarrow + +import ( + "regexp" + "strings" + + "github.com/apache/arrow/go/v13/arrow" + cqtypes "github.com/cloudquery/plugin-sdk/v3/types" +) + +var ( + reTimestamp = regexp.MustCompile(`timestamp\s*(?:\(([0-6])\))?(?: with(?:out)? time zone)?`) + reTime = regexp.MustCompile(`time\s*(?:\(([0-6])\))?(?: with(?:out)? time zone)?`) +) + +func Pg10ToArrow(t string) arrow.DataType { + t = normalize(t) + if strings.HasSuffix(t, "[]") { + return arrow.ListOf(Pg10ToArrow(t[:len(t)-2])) + } + + parsers := []func(string) (arrow.DataType, bool){ + parseTimestamp, + parseTime, + } + for _, parser := range parsers { + got, matched := parser(t) + if matched { + return got + } + } + + switch t { + case "boolean": + return arrow.FixedWidthTypes.Boolean + case "smallint": + return arrow.PrimitiveTypes.Int16 + case "smallserial": + return arrow.PrimitiveTypes.Int16 + case "serial": + return arrow.PrimitiveTypes.Int32 + case "integer", "int", "int4": + return arrow.PrimitiveTypes.Int32 + case "bigint", "int8": + return arrow.PrimitiveTypes.Int64 + case "bigserial", "serial8": + return arrow.PrimitiveTypes.Int64 + case "double precision", "float8": + return arrow.PrimitiveTypes.Float64 + case "real", "float4": + return arrow.PrimitiveTypes.Float32 + case "uuid": + return cqtypes.ExtensionTypes.UUID + case "bytea": + return arrow.BinaryTypes.Binary + case "json", "jsonb": + return cqtypes.ExtensionTypes.JSON + case "cidr": + return cqtypes.ExtensionTypes.Inet + case "macaddr", "macaddr8": + return cqtypes.ExtensionTypes.MAC + case "inet": + return cqtypes.ExtensionTypes.Inet + case "date": + return arrow.FixedWidthTypes.Date32 + default: + return arrow.BinaryTypes.String + } +} + +func Pg10ToCockroach(t string) arrow.DataType { + return Pg10ToArrow(t) +} + +func normalize(t string) string { + return strings.ToLower(strings.TrimSpace(t)) +} + +func parseTimestamp(t string) (arrow.DataType, bool) { + timestamptzPrefix := "timestamptz using" + if strings.HasPrefix(t, timestamptzPrefix) { + t = strings.TrimPrefix(t, timestamptzPrefix) + } + if t == "timestamptz" { + return arrow.FixedWidthTypes.Timestamp_us, true + } + + matches := reTimestamp.FindAllStringSubmatch(t, -1) + if len(matches) == 0 { + return nil, false + } + switch matches[0][1] { + case "0": + return arrow.FixedWidthTypes.Timestamp_s, true + case "1": + return arrow.FixedWidthTypes.Timestamp_ms, true + case "2": + return arrow.FixedWidthTypes.Timestamp_ms, true + case "3": + return arrow.FixedWidthTypes.Timestamp_ms, true + default: + return arrow.FixedWidthTypes.Timestamp_us, true + } +} + +func parseTime(t string) (arrow.DataType, bool) { + matches := reTime.FindAllStringSubmatch(t, -1) + if len(matches) == 0 { + return nil, false + } + switch matches[0][1] { + case "0": + return arrow.FixedWidthTypes.Time32s, true + case "1": + return arrow.FixedWidthTypes.Time32ms, true + case "2": + return arrow.FixedWidthTypes.Time32ms, true + case "3": + return arrow.FixedWidthTypes.Time32ms, true + default: + return arrow.FixedWidthTypes.Time64us, true + } +} diff --git a/plugins/source/postgresql/pgarrow/to_arrow_test.go b/plugins/source/postgresql/pgarrow/to_arrow_test.go new file mode 100644 index 00000000000000..f9946bcc1703e3 --- /dev/null +++ b/plugins/source/postgresql/pgarrow/to_arrow_test.go @@ -0,0 +1,141 @@ +package pgarrow + +import ( + "testing" + + "github.com/apache/arrow/go/v13/arrow" + "github.com/cloudquery/plugin-sdk/v3/types" +) + +func TestPg10ToSchemaType(t *testing.T) { + cases := []struct { + pgType string + want arrow.DataType + }{ + {"boolean", arrow.FixedWidthTypes.Boolean}, + {"bigint", arrow.PrimitiveTypes.Int64}, + {"double precision", arrow.PrimitiveTypes.Float64}, + {"uuid", types.ExtensionTypes.UUID}, + {"bytea", arrow.BinaryTypes.Binary}, + {"text[]", arrow.ListOf(arrow.BinaryTypes.String)}, + {"json", types.ExtensionTypes.JSON}, + {"jsonb", types.ExtensionTypes.JSON}, + {"uuid[]", arrow.ListOf(types.ExtensionTypes.UUID)}, + {"cidr", types.ExtensionTypes.Inet}, + {"cidr[]", arrow.ListOf(types.ExtensionTypes.Inet)}, + {"macaddr", types.ExtensionTypes.MAC}, + {"macaddr8", types.ExtensionTypes.MAC}, + {"macaddr[]", arrow.ListOf(types.ExtensionTypes.MAC)}, + {"macaddr8[]", arrow.ListOf(types.ExtensionTypes.MAC)}, + {"inet", types.ExtensionTypes.Inet}, + {"inet[]", arrow.ListOf(types.ExtensionTypes.Inet)}, + {"timestamp", arrow.FixedWidthTypes.Timestamp_us}, + {"text[][]", arrow.ListOf(arrow.ListOf(arrow.BinaryTypes.String))}, + {"varchar(50)", arrow.BinaryTypes.String}, + {"varchar(50)[][]", arrow.ListOf(arrow.ListOf(arrow.BinaryTypes.String))}, + {"character(10)", arrow.BinaryTypes.String}, + {"integer[]", arrow.ListOf(arrow.PrimitiveTypes.Int32)}, + {"timestamp", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp with time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamptz", arrow.FixedWidthTypes.Timestamp_us}, + {"TIMESTAMPTZ USING timestamp(0)", arrow.FixedWidthTypes.Timestamp_s}, + {"TIMESTAMPTZ USING timestamp(3)", arrow.FixedWidthTypes.Timestamp_ms}, + {"TIMESTAMPTZ USING timestamp(6)", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(0)", arrow.FixedWidthTypes.Timestamp_s}, + {"timestamp(1)", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(2)", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(3)", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(4)", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(5)", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(6)", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(0) without time zone", arrow.FixedWidthTypes.Timestamp_s}, + {"timestamp(1) without time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(2) without time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(3) without time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(4) without time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(5) without time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(6) without time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(0) with time zone", arrow.FixedWidthTypes.Timestamp_s}, + {"timestamp(1) with time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(2) with time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(3) with time zone", arrow.FixedWidthTypes.Timestamp_ms}, + {"timestamp(4) with time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(5) with time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"timestamp(6) with time zone", arrow.FixedWidthTypes.Timestamp_us}, + {"date", arrow.FixedWidthTypes.Date32}, + {"date[]", arrow.ListOf(arrow.FixedWidthTypes.Date32)}, + {"time", arrow.FixedWidthTypes.Time64us}, + {"time[]", arrow.ListOf(arrow.FixedWidthTypes.Time64us)}, + {"time with time zone", arrow.FixedWidthTypes.Time64us}, + {"time with time zone[]", arrow.ListOf(arrow.FixedWidthTypes.Time64us)}, + {"time without time zone", arrow.FixedWidthTypes.Time64us}, + {"time without time zone[]", arrow.ListOf(arrow.FixedWidthTypes.Time64us)}, + {"time(0)", arrow.FixedWidthTypes.Time32s}, + {"time(1)", arrow.FixedWidthTypes.Time32ms}, + {"time(2)", arrow.FixedWidthTypes.Time32ms}, + {"time(3)", arrow.FixedWidthTypes.Time32ms}, + {"time(4)", arrow.FixedWidthTypes.Time64us}, + {"time(5)", arrow.FixedWidthTypes.Time64us}, + {"time(6)", arrow.FixedWidthTypes.Time64us}, + {"time(0) without time zone", arrow.FixedWidthTypes.Time32s}, + {"time(1) without time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(2) without time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(3) without time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(4) without time zone", arrow.FixedWidthTypes.Time64us}, + {"time(5) without time zone", arrow.FixedWidthTypes.Time64us}, + {"time(6) without time zone", arrow.FixedWidthTypes.Time64us}, + {"time(0) with time zone", arrow.FixedWidthTypes.Time32s}, + {"time(1) with time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(2) with time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(3) with time zone", arrow.FixedWidthTypes.Time32ms}, + {"time(4) with time zone", arrow.FixedWidthTypes.Time64us}, + {"time(5) with time zone", arrow.FixedWidthTypes.Time64us}, + {"time(6) with time zone", arrow.FixedWidthTypes.Time64us}, + + // types that are converted to string for now - more specific support for these types + // may be added in the future + {"numeric", arrow.BinaryTypes.String}, + {"numeric (1, 0)", arrow.BinaryTypes.String}, + {"numeric (1000, 1000)", arrow.BinaryTypes.String}, + {"interval", arrow.BinaryTypes.String}, + {"interval YEAR", arrow.BinaryTypes.String}, + {"interval YEAR TO MONTH", arrow.BinaryTypes.String}, + {"interval SECOND", arrow.BinaryTypes.String}, + {"interval MINUTE TO SECOND", arrow.BinaryTypes.String}, + {"interval MINUTE TO SECOND (0)", arrow.BinaryTypes.String}, + {"interval MINUTE TO SECOND (3)", arrow.BinaryTypes.String}, + {"interval SECOND (6)", arrow.BinaryTypes.String}, + {"interval HOUR TO MINUTE", arrow.BinaryTypes.String}, + {"interval YEAR", arrow.BinaryTypes.String}, + {"interval MONTH", arrow.BinaryTypes.String}, + {"interval DAY", arrow.BinaryTypes.String}, + {"interval HOUR", arrow.BinaryTypes.String}, + {"interval MINUTE", arrow.BinaryTypes.String}, + {"interval SECOND", arrow.BinaryTypes.String}, + {"interval YEAR TO MONTH", arrow.BinaryTypes.String}, + {"interval DAY TO HOUR", arrow.BinaryTypes.String}, + {"interval DAY TO MINUTE", arrow.BinaryTypes.String}, + {"interval DAY TO SECOND", arrow.BinaryTypes.String}, + {"interval HOUR TO MINUTE", arrow.BinaryTypes.String}, + {"interval HOUR TO SECOND", arrow.BinaryTypes.String}, + {"interval MINUTE TO SECOND", arrow.BinaryTypes.String}, + {"money", arrow.BinaryTypes.String}, + {"box", arrow.BinaryTypes.String}, + {"bit", arrow.BinaryTypes.String}, + {"bit varying(10)", arrow.BinaryTypes.String}, + {"circle", arrow.BinaryTypes.String}, + {"line", arrow.BinaryTypes.String}, + {"point", arrow.BinaryTypes.String}, + {"path", arrow.BinaryTypes.String}, + {"polygon", arrow.BinaryTypes.String}, + } + + for _, c := range cases { + t.Run(c.pgType, func(t *testing.T) { + got := Pg10ToArrow(c.pgType) + if !arrow.TypeEqual(got, c.want) { + t.Errorf("Pg10ToArrow(%q) = %v, want %v", c.pgType, got, c.want) + } + }) + } +} diff --git a/plugins/source/postgresql/pgarrow/to_pg.go b/plugins/source/postgresql/pgarrow/to_pg.go new file mode 100644 index 00000000000000..9210d38f2dce9d --- /dev/null +++ b/plugins/source/postgresql/pgarrow/to_pg.go @@ -0,0 +1,65 @@ +package pgarrow + +import ( + "strconv" + + "github.com/apache/arrow/go/v13/arrow" + cqtypes "github.com/cloudquery/plugin-sdk/v3/types" +) + +func ArrowToPg10(t arrow.DataType) string { + switch dt := t.(type) { + case *cqtypes.UUIDType: + return "uuid" + case *cqtypes.JSONType: + return "jsonb" + case *cqtypes.MACType: + return "macaddr" + case *cqtypes.InetType: + return "inet" + case *arrow.ListType: + return ArrowToPg10(dt.Elem()) + "[]" + case *arrow.FixedSizeListType: + return ArrowToPg10(dt.Elem()) + "[]" + case *arrow.LargeListType: + return ArrowToPg10(dt.Elem()) + "[]" + case *arrow.MapType: + return "text" + case *arrow.BooleanType: + return "boolean" + case *arrow.Int8Type: + return "smallint" + case *arrow.Int16Type: + return "smallint" + case *arrow.Int32Type: + return "int" + case *arrow.Int64Type: + return "bigint" + case *arrow.Uint8Type: + return "smallint" + case *arrow.Uint16Type: + return "int" + case *arrow.Uint32Type: + return "bigint" + case *arrow.Uint64Type: + return "numeric(20,0)" + case *arrow.Float32Type: + return "real" + case *arrow.Float64Type: + return "double precision" + case arrow.DecimalType: + return "numeric(" + strconv.Itoa(int(dt.GetPrecision())) + "," + strconv.Itoa(int(dt.GetScale())) + ")" + case *arrow.StringType: + return "text" + case *arrow.BinaryType: + return "bytea" + case *arrow.TimestampType: + return "timestamp without time zone" + default: + panic("unknown type " + t.String()) + } +} + +func ArrowToCockroach(t arrow.DataType) string { + return ArrowToPg10(t) +} diff --git a/plugins/source/postgresql/resources/plugin/plugin_test.go b/plugins/source/postgresql/resources/plugin/plugin_test.go index 04ecd02de4f752..2a1a12391f3feb 100644 --- a/plugins/source/postgresql/resources/plugin/plugin_test.go +++ b/plugins/source/postgresql/resources/plugin/plugin_test.go @@ -5,14 +5,15 @@ import ( "errors" "fmt" "os" - "reflect" "strings" "sync" "testing" "time" + "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/cloudquery/plugins/source/postgresql/client" "github.com/cloudquery/plugin-pb-go/specs" + "github.com/cloudquery/plugin-sdk/v3/scalar" "github.com/cloudquery/plugin-sdk/v3/schema" "github.com/google/uuid" pgx_zero_log "github.com/jackc/pgx-zerolog" @@ -53,62 +54,119 @@ func getTestConnectionString() string { return testConn } -type pgTypeToValue struct { +type testCase struct { typeName string value any + expect scalar.Scalar } -func getPgTypesToData() []pgTypeToValue { - var pgTypesToData = []pgTypeToValue{ - {"bigint", 1}, - {"bigserial", nil}, - {"bit", "1"}, - {"bit(5)", "11111"}, - {"bit varying", "1"}, - {"bit varying(5)", "11111"}, - {"boolean", true}, - {"box", "((1,2),(3,4))"}, - {"bytea", []byte("test")}, - {"character", "a"}, - {"character(5)", "aaaaa"}, - {"character varying", "a"}, - {"character varying(5)", "aaaaa"}, - {"cidr", "10.1.2.3/32"}, - {"circle", "<(1,2),3>"}, - {"date", "1999-01-08"}, - {"double precision", 1.1}, - {"inet", "192.168.0.1/24"}, - {"integer", 1}, - {"interval", "1-2"}, - {"json", `{"a":1}`}, - {"jsonb", `{"a":1}`}, - {"line", "{1,2,3}"}, - {"lseg", "[(1,2),(3,4)]"}, - {"macaddr", "08:00:2b:01:02:03"}, - {"macaddr8", "08:00:2b:01:02:03:04:05"}, - {"money", "$1,000.00"}, - {"path", `[(1,2),(3,4)]`}, - {"point", "(1,2)"}, - {"polygon", `((1,2),(3,4))`}, - {"real", 1.1}, - {"smallint", 1}, - {"smallserial", nil}, - {"serial", nil}, - {"text", "test"}, - {"time without time zone", "04:05:06.789"}, - {"time(3)", "04:05:06.789"}, - {"time(3) without time zone", "04:05:06.789"}, - {"timestamp", "1999-01-08 04:05:06.789"}, - {"timestamp without time zone", "1999-01-08 04:05:06.789"}, - {"timestamp(3)", "1999-01-08 04:05:06.789"}, - {"timestamp(3) without time zone", "1999-01-08 04:05:06.789"}, - {"tsquery", "a & b"}, - {"tsvector", "'a':1 'b':2"}, - {"uuid", uuid.New().String()}, - {"xml", "1"}, +func getTestCases(serialValue int64) []testCase { + cidr := scalar.Inet{} + err := cidr.Set("10.1.2.3/32") + if err != nil { + panic(err) + } + mac1 := scalar.Mac{} + err = mac1.Set("08:00:2b:01:02:03") + if err != nil { + panic(err) + } + mac2 := scalar.Mac{} + err = mac2.Set("08:00:2b:01:02:03:04:05") + if err != nil { + panic(err) + } + inet := scalar.Inet{} + err = inet.Set("192.168.0.1/24") + if err != nil { + panic(err) + } + timeMicrosecond := scalar.Time{ + Unit: arrow.Microsecond, + } + err = timeMicrosecond.Set("04:05:06.789000") + if err != nil { + panic(err) + } + timeMillisecond := scalar.Time{ + Int: scalar.Int{ + BitWidth: 32, + }, + Unit: arrow.Millisecond, + } + err = timeMillisecond.Set("04:05:06.789") + if err != nil { + panic(err) + } + timestamp := scalar.Timestamp{} + err = timestamp.Set("1999-01-08 04:05:06.789") + if err != nil { + panic(err) + } + timestampMillisecond := scalar.Timestamp{ + Type: &arrow.TimestampType{ + Unit: arrow.Millisecond, + TimeZone: "UTC", + }, + } + err = timestampMillisecond.Set("1999-01-08 04:05:06.789") + if err != nil { + panic(err) + } + uuidData := scalar.UUID{} + err = uuidData.Set(uuid.New()) + if err != nil { + panic(err) + } + return []testCase{ + {"int", 1, &scalar.Int{Value: 1, Valid: true, BitWidth: 32}}, + {"bigint", 1, &scalar.Int{Value: 1, Valid: true, BitWidth: 64}}, + {"bigserial", nil, &scalar.Int{Value: serialValue, Valid: true, BitWidth: 64}}, + {"bit", "1", &scalar.String{Value: "1", Valid: true}}, + {"bit(5)", "11111", &scalar.String{Value: "11111", Valid: true}}, + {"bit varying", "1", &scalar.String{Value: "1", Valid: true}}, + {"bit varying(5)", "11111", &scalar.String{Value: "11111", Valid: true}}, + {"boolean", true, &scalar.Bool{Value: true, Valid: true}}, + {"box", "((1,2),(3,4))", &scalar.String{Value: "(3,4),(1,2)", Valid: true}}, + {"bytea", []byte("test"), &scalar.Binary{Value: []byte("test"), Valid: true}}, + {"character", "a", &scalar.String{Value: "a", Valid: true}}, + {"character(5)", "aaaaa", &scalar.String{Value: "aaaaa", Valid: true}}, + {"character varying", "a", &scalar.String{Value: "a", Valid: true}}, + {"character varying(5)", "aaaaa", &scalar.String{Value: "aaaaa", Valid: true}}, + {"cidr", "10.1.2.3/32", &cidr}, + {"circle", "<(1,2),3>", &scalar.String{Value: "<(1,2),3>", Valid: true}}, + {"date", "1999-01-08", &scalar.Date32{Value: 10599, Valid: true}}, + {"double precision", 1.1, &scalar.Float{Value: 1.1, Valid: true, BitWidth: 64}}, + {"inet", "192.168.0.1/24", &inet}, + {"integer", 1, &scalar.Int{Value: 1, Valid: true, BitWidth: 32}}, + {"interval", "1-2", &scalar.String{Value: "14 mon 00:00:00.000000", Valid: true}}, + {"json", `{"a":1}`, &scalar.JSON{Value: []byte(`{"a":1}`), Valid: true}}, + {"jsonb", `{"a":1}`, &scalar.JSON{Value: []byte(`{"a":1}`), Valid: true}}, + {"line", "{1,2,3}", &scalar.String{Value: "{1,2,3}", Valid: true}}, + {"lseg", "[(1,2),(3,4)]", &scalar.String{Value: "[(1,2),(3,4)]", Valid: true}}, + {"macaddr", "08:00:2b:01:02:03", &mac1}, + {"macaddr8", "08:00:2b:01:02:03:04:05", &mac2}, + {"money", "$1,000.00", &scalar.String{Value: "$1,000.00", Valid: true}}, + {"path", `[(1,2),(3,4)]`, &scalar.String{Value: "[(1,2),(3,4)]", Valid: true}}, + {"point", "(1,2)", &scalar.String{Value: "(1,2)", Valid: true}}, + {"polygon", `((1,2),(3,4))`, &scalar.String{Value: "((1,2),(3,4))", Valid: true}}, + {"real", 1.1, &scalar.Float{Value: 1.100000023841858, Valid: true, BitWidth: 32}}, + {"smallint", 1, &scalar.Int{Value: 1, Valid: true, BitWidth: 16}}, + {"smallserial", nil, &scalar.Int{Value: serialValue, Valid: true, BitWidth: 16}}, + {"serial", nil, &scalar.Int{Value: serialValue, Valid: true, BitWidth: 32}}, + {"text", "test", &scalar.String{Value: "test", Valid: true}}, + {"time without time zone", "04:05:06.789", &timeMicrosecond}, + {"time(3)", "04:05:06.789", &timeMillisecond}, + {"time(3) without time zone", "04:05:06.789", &timeMillisecond}, + {"timestamp", "1999-01-08 04:05:06.789", ×tamp}, + {"timestamp without time zone", "1999-01-08 04:05:06.789", ×tamp}, + {"timestamp(3)", "1999-01-08 04:05:06.789", ×tampMillisecond}, + {"timestamp(3) without time zone", "1999-01-08 04:05:06.789", ×tampMillisecond}, + {"tsquery", "a & b", &scalar.String{Value: "'a' & 'b'", Valid: true}}, + {"tsvector", "'a':1 'b':2", &scalar.String{Value: "'a':1 'b':2", Valid: true}}, + {"uuid", &uuidData, &uuidData}, + {"xml", "1", &scalar.String{Value: "1", Valid: true}}, } - - return pgTypesToData } func createTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string) error { @@ -116,7 +174,7 @@ func createTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string) sb.WriteString("CREATE TABLE ") sb.WriteString(pgx.Identifier{tableName}.Sanitize()) sb.WriteString(" (") - columns := getPgTypesToData() + columns := getTestCases(0) for i, col := range columns { sb.WriteString(pgx.Identifier{col.typeName + "_type"}.Sanitize()) sb.WriteString(" ") @@ -135,10 +193,10 @@ func createTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string) return nil } -func insertTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string, columns []pgTypeToValue) error { +func insertTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string, testCases []testCase) error { var query = "" query += "INSERT INTO " + pgx.Identifier{tableName}.Sanitize() + " (" - for _, col := range columns { + for _, col := range testCases { if col.value == nil { continue } @@ -146,7 +204,7 @@ func insertTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string, } query = query[:len(query)-2] + ") VALUES (" dataIndex := 0 - for _, col := range columns { + for _, col := range testCases { if col.value == nil { continue } @@ -156,7 +214,7 @@ func insertTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string, query = query[:len(query)-2] + ")" pgData := make([]any, dataIndex) i := 0 - for _, col := range columns { + for _, col := range testCases { if col.value == nil { continue } @@ -170,70 +228,6 @@ func insertTestTable(ctx context.Context, conn *pgxpool.Pool, tableName string, return nil } -func getExpectedData(uuidValue any, serialValue int64) schema.CQTypes { - cidr := schema.CIDR{} - _ = cidr.Set("10.1.2.3/32") - mac1 := schema.Macaddr{} - _ = mac1.Set("08:00:2b:01:02:03") - mac2 := schema.Macaddr{} - _ = mac2.Set("08:00:2b:01:02:03:04:05") - inet := schema.Inet{} - _ = inet.Set("192.168.0.1/24") - timestamp := schema.Timestamptz{} - _ = timestamp.Set("1999-01-08 04:05:06.789") - uuidData := schema.UUID{} - _ = uuidData.Set(uuidValue) - expectedData := schema.CQTypes{ - &schema.Int8{Int: 1, Status: schema.Present}, - &schema.Int8{Int: serialValue, Status: schema.Present}, - &schema.Text{Str: "1", Status: schema.Present}, - &schema.Text{Str: "11111", Status: schema.Present}, - &schema.Text{Str: "1", Status: schema.Present}, - &schema.Text{Str: "11111", Status: schema.Present}, - &schema.Bool{Bool: true, Status: schema.Present}, - &schema.Text{Str: "(3,4),(1,2)", Status: schema.Present}, - &schema.Bytea{Bytes: []byte("test"), Status: schema.Present}, - &schema.Text{Str: "a", Status: schema.Present}, - &schema.Text{Str: "aaaaa", Status: schema.Present}, - &schema.Text{Str: "a", Status: schema.Present}, - &schema.Text{Str: "aaaaa", Status: schema.Present}, - &cidr, - &schema.Text{Str: "<(1,2),3>", Status: schema.Present}, - &schema.Text{Str: "1999-01-08 00:00:00 +0000 UTC", Status: schema.Present}, - &schema.Float8{Float: 1.1, Status: schema.Present}, - &inet, - &schema.Int8{Int: 1, Status: schema.Present}, - &schema.Text{Str: "14 mon 00:00:00.000000", Status: schema.Present}, - &schema.JSON{Bytes: []byte(`{"a":1}`), Status: schema.Present}, - &schema.JSON{Bytes: []byte(`{"a":1}`), Status: schema.Present}, - &schema.Text{Str: "{1,2,3}", Status: schema.Present}, - &schema.Text{Str: "[(1,2),(3,4)]", Status: schema.Present}, - &mac1, - &mac2, - &schema.Text{Str: "$1,000.00", Status: schema.Present}, - &schema.Text{Str: "[(1,2),(3,4)]", Status: schema.Present}, - &schema.Text{Str: "(1,2)", Status: schema.Present}, - &schema.Text{Str: "((1,2),(3,4))", Status: schema.Present}, - &schema.Float8{Float: 1.100000023841858, Status: schema.Present}, - &schema.Int8{Int: 1, Status: schema.Present}, - &schema.Int8{Int: serialValue, Status: schema.Present}, - &schema.Int8{Int: serialValue, Status: schema.Present}, - &schema.Text{Str: "test", Status: schema.Present}, - &schema.Text{Str: "04:05:06.789000", Status: schema.Present}, - &schema.Text{Str: "04:05:06.789000", Status: schema.Present}, - &schema.Text{Str: "04:05:06.789000", Status: schema.Present}, - ×tamp, - ×tamp, - ×tamp, - ×tamp, - &schema.Text{Str: "'a' & 'b'", Status: schema.Present}, - &schema.Text{Str: "'a':1 'b':2", Status: schema.Present}, - &uuidData, - &schema.Text{Str: "1", Status: schema.Present}, - } - return expectedData -} - func TestPlugin(t *testing.T) { p := Plugin() ctx := context.Background() @@ -265,7 +259,7 @@ func TestPlugin(t *testing.T) { if err := createTestTable(ctx, conn, testTable); err != nil { t.Fatal(err) } - data := getPgTypesToData() + data := getTestCases(1) err = insertTestTable(ctx, conn, testTable, data) if err != nil { t.Fatal(err) @@ -278,7 +272,7 @@ func TestPlugin(t *testing.T) { if err := createTestTable(ctx, conn, otherTable); err != nil { t.Fatal(err) } - err = insertTestTable(ctx, conn, otherTable, getPgTypesToData()) + err = insertTestTable(ctx, conn, otherTable, getTestCases(2)) if err != nil { t.Fatal(err) } @@ -289,9 +283,10 @@ func TestPlugin(t *testing.T) { } res := make(chan *schema.Resource, 1) g := errgroup.Group{} + syncTime := time.Now() g.Go(func() error { defer close(res) - return p.Sync(ctx, res) + return p.Sync(ctx, syncTime, res) }) var resource *schema.Resource totalResources := 0 @@ -307,12 +302,11 @@ func TestPlugin(t *testing.T) { t.Fatalf("expected 1 resource, got %d", totalResources) } gotData := resource.GetValues() - expectedData := getExpectedData(data[44].value, 1) - for i, v := range gotData { - expected := expectedData[i] - if !reflect.DeepEqual(v, expected) { - t.Fatalf("expected %v, got %v", expected, v) + for i, got := range gotData { + expected := data[i].expect + if !got.Equal(expected) { + t.Fatalf("type %v: expected %v, got %v", data[i].typeName, expected, got) } } } @@ -322,7 +316,7 @@ func TestPluginCDC(t *testing.T) { ctx := context.Background() l := zerolog.New(zerolog.NewTestWriter(t)).Output( zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro}, - ).Level(zerolog.DebugLevel).With().Timestamp().Logger() + ).Level(zerolog.WarnLevel).With().Timestamp().Logger() p.SetLogger(l) spec := specs.Source{ Name: "test_pg_source", @@ -356,7 +350,7 @@ func TestPluginCDC(t *testing.T) { if err := createTestTable(ctx, conn, testTable); err != nil { t.Fatal(err) } - data := getPgTypesToData() + data := getTestCases(1) err = insertTestTable(ctx, conn, testTable, data) if err != nil { t.Fatal(err) @@ -374,12 +368,13 @@ func TestPluginCDC(t *testing.T) { defer cancel() wg.Add(1) + syncTime := time.Now() go func() { defer wg.Done() defer close(res) - syncErr = p.Sync(syncCtx, res) + syncErr = p.Sync(syncCtx, syncTime, res) }() - data2 := getPgTypesToData() + data2 := getTestCases(2) time.AfterFunc(2*time.Second, func() { err = insertTestTable(ctx, conn, testTable, data2) if err != nil { @@ -391,19 +386,17 @@ func TestPluginCDC(t *testing.T) { for r := range res { gotData := r.GetValues() if totalResources == 0 { - expectedData := getExpectedData(data[44].value, 1) - for i, v := range gotData { - expected := expectedData[i] - if !reflect.DeepEqual(v, expected) { - t.Fatalf("expected %v, got %v", expected, v) + for i, got := range gotData { + expected := data[i].expect + if !got.Equal(expected) { + t.Fatalf("type %v: expected %v, got %v", data[i].typeName, expected, got) } } } else { - expectedData := getExpectedData(data2[44].value, 2) - for i, v := range gotData { - expected := expectedData[i] - if !reflect.DeepEqual(v, expected) { - t.Fatalf("expected %v, got %v", expected, v) + for i, got := range gotData { + expected := data2[i].expect + if !got.Equal(expected) { + t.Fatalf("type %v: expected %v, got %v", data[i].typeName, expected, got) } } } From 0eb6fc889db9c50182d9dd6c04642411e9827c1e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 1 Jun 2023 11:48:54 +0100 Subject: [PATCH 03/10] All tests passing after fixing testing bug with cdc --- plugins/source/postgresql/resources/plugin/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/postgresql/resources/plugin/plugin_test.go b/plugins/source/postgresql/resources/plugin/plugin_test.go index 2a1a12391f3feb..6826d9f713aec4 100644 --- a/plugins/source/postgresql/resources/plugin/plugin_test.go +++ b/plugins/source/postgresql/resources/plugin/plugin_test.go @@ -339,7 +339,7 @@ func TestPluginCDC(t *testing.T) { t.Fatal(err) } var pgErr *pgconn.PgError - if _, err := conn.Exec(ctx, "SELECT pg_drop_replication_slot('\"user\"')"); err != nil { + if _, err := conn.Exec(ctx, "SELECT pg_drop_replication_slot('test_pg_source')"); err != nil { if !(errors.As(err, &pgErr) && pgErr.Code == "42704") { t.Fatal(err) } From a545168ece144dc2460f2fff22fa045f49a14519 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 1 Jun 2023 15:10:40 +0100 Subject: [PATCH 04/10] Save --- .../source/postgresql/client/list_tables.go | 31 ++-- .../source/postgresql/client/transformer.go | 141 ------------------ 2 files changed, 20 insertions(+), 152 deletions(-) delete mode 100644 plugins/source/postgresql/client/transformer.go diff --git a/plugins/source/postgresql/client/list_tables.go b/plugins/source/postgresql/client/list_tables.go index 5d84a99746a5ce..cf877ee3afd7e3 100644 --- a/plugins/source/postgresql/client/list_tables.go +++ b/plugins/source/postgresql/client/list_tables.go @@ -8,14 +8,14 @@ import ( ) // this returns the following table in sorted manner: -// +----------------+-------------+-------------+------------+---------------+-----------+---------------------+ -// | ordinal_position | table_name | column_name | data_type | is_primary_key| not_null | pk_constraint_name | -// +----------------+-------------+-------------+------------+---------------+-----------+---------------------+ -// | 1 | users | id | bigint | YES | true | cq_users_pk | -// | 2 | users | name | text | NO | false | | -// | 3 | users | email | text | NO | true | cq_users_pk | -// | 1 | posts | id | bigint | YES | true | cq_posts_pk | -// | 2 | posts | title | text | NO | false | | +// +----------------+-------------+-------------+------------+----------------+-----------+-----------+---------------------+ +// | ordinal_position | table_name | column_name | data_type | is_primary_key | not_null | is_unique | pk_constraint_name | +// +----------------+-------------+-------------+------------+----------------+-----------+-----------+---------------------+ +// | 1 | users | id | bigint | YES | true | true | cq_users_pk | +// | 2 | users | name | text | NO | false | false | | +// | 3 | users | email | text | NO | true | false | cq_users_pk | +// | 1 | posts | id | bigint | YES | true | true | cq_posts_pk | +// | 2 | posts | title | text | NO | false | false | | const selectTables = ` SELECT columns.ordinal_position AS ordinal_position, @@ -30,6 +30,10 @@ SELECT WHEN pg_attribute.attnotnull THEN true ELSE false END AS not_null, + CASE + WHEN pg_index.indisunique THEN true + ELSE false + END AS is_unique, COALESCE(pg_constraint.conname, '') AS primary_key_constraint_name FROM pg_catalog.pg_attribute @@ -41,6 +45,8 @@ FROM pg_catalog.pg_constraint ON pg_constraint.conrelid = pg_attribute.attrelid AND conkey IS NOT NULL AND array_position(conkey, pg_attribute.attnum) > 0 AND contype = 'p' + LEFT JOIN pg_catalog.pg_index ON pg_index.indrelid = pg_attribute.attrelid + AND pg_index.indkey::text LIKE '%%' || pg_attribute.attnum || '%%' INNER JOIN information_schema.columns ON columns.table_name = pg_class.relname AND columns.column_name = pg_attribute.attname AND columns.table_schema = pg_catalog.pg_namespace.nspname WHERE @@ -53,7 +59,9 @@ ORDER BY func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { var tables schema.Tables - rows, err := c.Conn.Query(ctx, fmt.Sprintf(selectTables, c.currentSchemaName)) + q := fmt.Sprintf(selectTables, c.currentSchemaName) + fmt.Println(q) + rows, err := c.Conn.Query(ctx, q) if err != nil { return nil, err } @@ -61,8 +69,8 @@ func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { for rows.Next() { var ordinalPosition int var tableName, columnName, columnType, pkName string - var isPrimaryKey, notNull bool - if err := rows.Scan(&ordinalPosition, &tableName, &columnName, &columnType, &isPrimaryKey, ¬Null, &pkName); err != nil { + var isPrimaryKey, notNull, isUnique bool + if err := rows.Scan(&ordinalPosition, &tableName, &columnName, &columnType, &isPrimaryKey, ¬Null, &isUnique, &pkName); err != nil { return nil, err } if ordinalPosition == 1 { @@ -79,6 +87,7 @@ func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { Name: columnName, PrimaryKey: isPrimaryKey, NotNull: notNull, + Unique: isUnique, Type: c.PgToSchemaType(columnType), }) } diff --git a/plugins/source/postgresql/client/transformer.go b/plugins/source/postgresql/client/transformer.go deleted file mode 100644 index 1d05cef880a5e1..00000000000000 --- a/plugins/source/postgresql/client/transformer.go +++ /dev/null @@ -1,141 +0,0 @@ -package client - -import ( - "net" - "strings" - - "github.com/cloudquery/plugin-sdk/v2/schema" - "github.com/jackc/pgx/v5/pgtype" -) - -// this is used for tests -type Transformer struct{} - -func (*Transformer) TransformBool(v *schema.Bool) any { - return &pgtype.Bool{ - Bool: v.Bool, - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformBytea(v *schema.Bytea) any { - if v.Status == schema.Present { - return v.Bytes - } - return nil -} - -func (*Transformer) TransformFloat8(v *schema.Float8) any { - return &pgtype.Float8{ - Float64: v.Float, - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformInt8(v *schema.Int8) any { - return &pgtype.Int8{ - Int64: v.Int, - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformInt8Array(v *schema.Int8Array) any { - r := pgtype.FlatArray[pgtype.Int8]{} - for _, e := range v.Elements { - r = append(r, pgtype.Int8{Int64: e.Int, Valid: e.Status == schema.Present}) - } - return &r -} - -func (*Transformer) TransformJSON(v *schema.JSON) any { - if v.Status == schema.Present { - return v.Bytes - } - return nil -} - -func (*Transformer) TransformText(v *schema.Text) any { - return &pgtype.Text{ - String: stripNulls(v.Str), - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformTextArray(v *schema.TextArray) any { - r := pgtype.FlatArray[pgtype.Text]{} - for _, e := range v.Elements { - r = append(r, pgtype.Text{String: stripNulls(e.Str), Valid: e.Status == schema.Present}) - } - return &r -} - -func (*Transformer) TransformTimestamptz(v *schema.Timestamptz) any { - return &pgtype.Timestamptz{ - Time: v.Time, - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformUUID(v *schema.UUID) any { - return pgtype.UUID{ - Bytes: v.Bytes, - Valid: v.Status == schema.Present, - } -} - -func (*Transformer) TransformUUIDArray(v *schema.UUIDArray) any { - r := pgtype.FlatArray[pgtype.UUID]{} - for _, e := range v.Elements { - r = append(r, pgtype.UUID{Bytes: e.Bytes, Valid: e.Status == schema.Present}) - } - return &r -} - -func (*Transformer) TransformCIDR(v *schema.CIDR) any { - if v.Status == schema.Present { - return v.IPNet - } - return nil -} - -func (*Transformer) TransformCIDRArray(v *schema.CIDRArray) any { - r := pgtype.FlatArray[*net.IPNet]{} - for _, e := range v.Elements { - r = append(r, e.IPNet) - } - return &r -} - -func (*Transformer) TransformInet(v *schema.Inet) any { - if v.Status == schema.Present { - return v.IPNet - } - return nil -} - -func (*Transformer) TransformInetArray(v *schema.InetArray) any { - r := pgtype.FlatArray[*net.IPNet]{} - for _, e := range v.Elements { - r = append(r, e.IPNet) - } - return &r -} - -func (*Transformer) TransformMacaddr(v *schema.Macaddr) any { - if v.Status == schema.Present { - return v.Addr - } - return nil -} - -func (*Transformer) TransformMacaddrArray(v *schema.MacaddrArray) any { - r := pgtype.FlatArray[net.HardwareAddr]{} - for _, e := range v.Elements { - r = append(r, e.Addr) - } - return &r -} - -func stripNulls(s string) string { - return strings.ReplaceAll(s, "\x00", "") -} From 9d688d55a2b390efccc284d06a1b8f6e30206169 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 1 Jun 2023 20:27:44 +0100 Subject: [PATCH 05/10] Add _cq_sync_time and _cq_source_name columns, only to realize that sync time doesn't get passed in to the Sync call, so we'll need to update the SDK --- plugins/source/postgresql/client/cdc.go | 20 ++++- plugins/source/postgresql/client/client.go | 4 + .../source/postgresql/client/list_tables.go | 10 +++ plugins/source/postgresql/client/sync.go | 88 ++++++++++++------- plugins/source/postgresql/go.mod | 6 +- plugins/source/postgresql/go.sum | 2 - .../resources/plugin/plugin_test.go | 8 +- 7 files changed, 93 insertions(+), 45 deletions(-) diff --git a/plugins/source/postgresql/client/cdc.go b/plugins/source/postgresql/client/cdc.go index e07fe61d558eef..97600350fe497d 100644 --- a/plugins/source/postgresql/client/cdc.go +++ b/plugins/source/postgresql/client/cdc.go @@ -78,6 +78,7 @@ func getReplicationName(specName string) string { } func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) error { + syncTime := time.Now().UTC() connPool, err := c.Conn.Acquire(ctx) if err != nil { return fmt.Errorf("failed to acquire connection: %w", err) @@ -186,7 +187,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values) + resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) if err != nil { return err } @@ -213,7 +214,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values) + resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) if err != nil { return err } @@ -239,7 +240,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values) + resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) if err != nil { return err } @@ -278,10 +279,13 @@ func decodeTextColumnData(mi *pgtype.Map, data []byte, dataType uint32) (any, er return string(data), nil } -func (c *Client) resourceFromCDCValues(tableName string, values map[string]any) (*schema.Resource, error) { +func (c *Client) resourceFromCDCValues(tableName string, values map[string]any, syncTime time.Time) (*schema.Resource, error) { table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) for _, col := range table.Columns { + if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { + continue + } v, err := prepareValueForResourceSet(col, values[col.Name]) if err != nil { return nil, err @@ -290,5 +294,13 @@ func (c *Client) resourceFromCDCValues(tableName string, values map[string]any) return nil, err } } + err := resource.Set(schema.CqSourceNameColumn.Name, c.spec.Name) + if err != nil { + return nil, err + } + err = resource.Set(schema.CqSyncTimeColumn.Name, syncTime) + if err != nil { + return nil, err + } return resource, nil } diff --git a/plugins/source/postgresql/client/client.go b/plugins/source/postgresql/client/client.go index 0fa2bc16fea43e..2ee9500fbbfc56 100644 --- a/plugins/source/postgresql/client/client.go +++ b/plugins/source/postgresql/client/client.go @@ -104,6 +104,10 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, _ if err != nil { return nil, fmt.Errorf("failed to apply config to tables: %w", err) } + err = c.addInternalColumns(c.Tables) + if err != nil { + return nil, fmt.Errorf("failed to add internal columns: %w", err) + } if c.pluginSpec.CDC { if len(c.tablesWithPks()) == 0 { diff --git a/plugins/source/postgresql/client/list_tables.go b/plugins/source/postgresql/client/list_tables.go index cf877ee3afd7e3..31ca9a3865a5aa 100644 --- a/plugins/source/postgresql/client/list_tables.go +++ b/plugins/source/postgresql/client/list_tables.go @@ -93,3 +93,13 @@ func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { } return tables, nil } + +// Add internal columns +func (c *Client) addInternalColumns(tables []*schema.Table) error { + for _, table := range tables { + cqSourceName := schema.CqSourceNameColumn + cqSyncTime := schema.CqSyncTimeColumn + table.Columns = append([]schema.Column{cqSourceName, cqSyncTime}, table.Columns...) + } + return nil +} diff --git a/plugins/source/postgresql/client/sync.go b/plugins/source/postgresql/client/sync.go index fc908c7962170e..2f0db5dbc148a1 100644 --- a/plugins/source/postgresql/client/sync.go +++ b/plugins/source/postgresql/client/sync.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/plugin-sdk/v3/plugins/source" @@ -15,7 +16,6 @@ import ( ) func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- *schema.Resource) error { - // var conn *pgconn.PgConn var err error var snapshotName string c.metrics = metrics @@ -42,10 +42,12 @@ func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- * } } + syncTime := time.Now() + if c.pluginSpec.CDC && snapshotName == "" { c.logger.Info().Msg("cdc is enabled but replication slot already exists, skipping initial sync") } else { - if err := c.syncTables(ctx, snapshotName, res); err != nil { + if err := c.syncTables(ctx, snapshotName, res, syncTime); err != nil { return err } } @@ -60,36 +62,7 @@ func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- * return nil } -func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, res chan<- *schema.Resource) error { - colNames := make([]string, len(table.Columns)) - for i, col := range table.Columns { - colNames[i] = pgx.Identifier{col.Name}.Sanitize() - } - query := "SELECT " + strings.Join(colNames, ",") + " FROM " + pgx.Identifier{table.Name}.Sanitize() - rows, err := tx.Query(ctx, query) - if err != nil { - c.metrics.TableClient[table.Name][c.ID()].Errors++ - return err - } - defer rows.Close() - for rows.Next() { - values, err := rows.Values() - if err != nil { - c.metrics.TableClient[table.Name][c.ID()].Errors++ - return err - } - resource, err := c.resourceFromValues(table.Name, values) - if err != nil { - c.metrics.TableClient[table.Name][c.ID()].Errors++ - return err - } - c.metrics.TableClient[table.Name][c.ID()].Resources++ - res <- resource - } - return nil -} - -func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- *schema.Resource) error { +func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- *schema.Resource, syncTime time.Time) error { tx, err := c.Conn.BeginTx(ctx, pgx.TxOptions{ // this transaction is needed for us to take a snapshot and we need to close it only at the end of the initial sync // https://www.postgresql.org/docs/current/transaction-iso.html @@ -116,7 +89,7 @@ func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- } for _, table := range c.Tables { - if err := c.syncTable(ctx, tx, table, res); err != nil { + if err := c.syncTable(ctx, tx, table, res, syncTime); err != nil { return err } } @@ -126,10 +99,56 @@ func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- return nil } +func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, res chan<- *schema.Resource, syncTime time.Time) error { + colNames := make([]string, 0, len(table.Columns)-2) + for _, col := range table.Columns { + if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { + continue + } + colNames = append(colNames, pgx.Identifier{col.Name}.Sanitize()) + } + query := "SELECT " + strings.Join(colNames, ",") + " FROM " + pgx.Identifier{table.Name}.Sanitize() + rows, err := tx.Query(ctx, query) + if err != nil { + c.metrics.TableClient[table.Name][c.ID()].Errors++ + return err + } + defer rows.Close() + for rows.Next() { + values, err := rows.Values() + if err != nil { + c.metrics.TableClient[table.Name][c.ID()].Errors++ + return err + } + resource, err := c.resourceFromValues(table.Name, values) + if err != nil { + c.metrics.TableClient[table.Name][c.ID()].Errors++ + return err + } + err = resource.Set(schema.CqSourceNameColumn.Name, c.spec.Name) + if err != nil { + return fmt.Errorf("failed to set CQ source name column: %w", err) + } + err = resource.Set(schema.CqSyncTimeColumn.Name, syncTime) + if err != nil { + return fmt.Errorf("failed to set CQ sync time column: %w", err) + } + + c.metrics.TableClient[table.Name][c.ID()].Resources++ + + res <- resource + } + return nil +} + func (c *Client) resourceFromValues(tableName string, values []any) (*schema.Resource, error) { table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) - for i, col := range table.Columns { + var i int + for _, col := range table.Columns { + if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { + continue + } v, err := prepareValueForResourceSet(col, values[i]) if err != nil { return nil, err @@ -137,6 +156,7 @@ func (c *Client) resourceFromValues(tableName string, values []any) (*schema.Res if err := resource.Set(col.Name, v); err != nil { return nil, err } + i++ } return resource, nil } diff --git a/plugins/source/postgresql/go.mod b/plugins/source/postgresql/go.mod index 3427e14d246706..8035f59efe7e0d 100644 --- a/plugins/source/postgresql/go.mod +++ b/plugins/source/postgresql/go.mod @@ -3,9 +3,8 @@ module github.com/cloudquery/cloudquery/plugins/source/postgresql go 1.19 require ( - github.com/apache/arrow/go/v13 v13.0.0-20230531201200-cbc17a98dfd9 + github.com/apache/arrow/go/v13 v13.0.0-20230601214540-018e7d3f9c4b github.com/cloudquery/plugin-pb-go v1.1.0 - github.com/cloudquery/plugin-sdk/v2 v2.7.0 github.com/cloudquery/plugin-sdk/v3 v3.10.6 github.com/google/uuid v1.3.0 github.com/jackc/pglogrepl v0.0.0-20230318140337-5ef673a9d169 @@ -17,12 +16,11 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 -replace github.com/cloudquery/plugin-sdk/v3 => ../../../../plugin-sdk require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.16.0 // indirect - github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853 // indirect + github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect diff --git a/plugins/source/postgresql/go.sum b/plugins/source/postgresql/go.sum index 5043bd09d76267..d70caa60cb2477 100644 --- a/plugins/source/postgresql/go.sum +++ b/plugins/source/postgresql/go.sum @@ -44,8 +44,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853 h1:HvZJa27CHTtd4TQa26ANVdckiYAoTZbt8gndfVsBEfw= -github.com/cloudquery/arrow v0.0.0-20230621001250-f0dffc612853/go.mod h1:GZvXlnoq/C1N1EDil2jmVHUijhWiMb77VihjHChz3wA= github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg= github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po= github.com/cloudquery/plugin-pb-go v1.1.0 h1:F1r/x4aF5aO1hmgBk8rqAp2oejeYMMughTPaYosQLDk= diff --git a/plugins/source/postgresql/resources/plugin/plugin_test.go b/plugins/source/postgresql/resources/plugin/plugin_test.go index 6826d9f713aec4..75c0204916a74e 100644 --- a/plugins/source/postgresql/resources/plugin/plugin_test.go +++ b/plugins/source/postgresql/resources/plugin/plugin_test.go @@ -302,7 +302,13 @@ func TestPlugin(t *testing.T) { t.Fatalf("expected 1 resource, got %d", totalResources) } gotData := resource.GetValues() - + if gotData[0].Get().(string) != "test_pg_source" { + t.Fatalf("expected source name to be test_pg_source, got %s", gotData[0]) + } + if !gotData[1].Get().(time.Time).Equal(syncTime.UTC()) { + t.Fatalf("expected sync time to be %s, but got %s", syncTime.UTC(), gotData[1]) + } + gotData = gotData[2:] // ignore the first two columns (_cq_source_name and _cq_sync_time) for i, got := range gotData { expected := data[i].expect if !got.Equal(expected) { From f1a0bebf09d564d79f874193711b464446e6c573 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 21 Jun 2023 15:20:47 +0200 Subject: [PATCH 06/10] fix: Update arrow --- plugins/source/postgresql/go.mod | 10 ++-------- plugins/source/postgresql/go.sum | 20 +++----------------- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/plugins/source/postgresql/go.mod b/plugins/source/postgresql/go.mod index 8035f59efe7e0d..12545a407449a2 100644 --- a/plugins/source/postgresql/go.mod +++ b/plugins/source/postgresql/go.mod @@ -3,7 +3,7 @@ module github.com/cloudquery/cloudquery/plugins/source/postgresql go 1.19 require ( - github.com/apache/arrow/go/v13 v13.0.0-20230601214540-018e7d3f9c4b + github.com/apache/arrow/go/v13 v13.0.0-20230601214540-e07e22c5580a github.com/cloudquery/plugin-pb-go v1.1.0 github.com/cloudquery/plugin-sdk/v3 v3.10.6 github.com/google/uuid v1.3.0 @@ -15,18 +15,15 @@ require ( ) // TODO: remove once all updates are merged -replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 +replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853 require ( - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/thrift v0.16.0 // indirect github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/goccy/go-json v0.10.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.1.21+incompatible // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect @@ -35,14 +32,11 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.0 // indirect - github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.5.0 // indirect diff --git a/plugins/source/postgresql/go.sum b/plugins/source/postgresql/go.sum index d70caa60cb2477..4c2740cba5f3c7 100644 --- a/plugins/source/postgresql/go.sum +++ b/plugins/source/postgresql/go.sum @@ -33,19 +33,14 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po= +github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853 h1:MeXRH104spmnswFe4dQoYsWRD0fS9HzhQ69GkSkF7a8= +github.com/cloudquery/arrow/go/v13 v13.0.0-20230621001250-f0dffc612853/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= github.com/cloudquery/plugin-pb-go v1.1.0 h1:F1r/x4aF5aO1hmgBk8rqAp2oejeYMMughTPaYosQLDk= github.com/cloudquery/plugin-pb-go v1.1.0/go.mod h1:327Dd56bQ357KNIbhZNGDoJ7jPYXsCZWZ4Tj955gU7M= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -90,7 +85,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -108,8 +102,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v23.1.21+incompatible h1:bUqzx/MXCDxuS0hRJL2EfjyZL3uQrPbMocUa8zGqsTA= @@ -170,8 +162,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= @@ -192,10 +182,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -432,7 +418,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From d1c01619c73594e5ce9e346679b42b232a94806c Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 21 Jun 2023 15:36:03 +0200 Subject: [PATCH 07/10] fix: Remove addInternal columns, fix linting --- plugins/source/postgresql/client/client.go | 4 ---- plugins/source/postgresql/client/list_tables.go | 10 ---------- plugins/source/postgresql/pgarrow/to_arrow.go | 4 +--- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/plugins/source/postgresql/client/client.go b/plugins/source/postgresql/client/client.go index 2ee9500fbbfc56..0fa2bc16fea43e 100644 --- a/plugins/source/postgresql/client/client.go +++ b/plugins/source/postgresql/client/client.go @@ -104,10 +104,6 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, _ if err != nil { return nil, fmt.Errorf("failed to apply config to tables: %w", err) } - err = c.addInternalColumns(c.Tables) - if err != nil { - return nil, fmt.Errorf("failed to add internal columns: %w", err) - } if c.pluginSpec.CDC { if len(c.tablesWithPks()) == 0 { diff --git a/plugins/source/postgresql/client/list_tables.go b/plugins/source/postgresql/client/list_tables.go index 31ca9a3865a5aa..cf877ee3afd7e3 100644 --- a/plugins/source/postgresql/client/list_tables.go +++ b/plugins/source/postgresql/client/list_tables.go @@ -93,13 +93,3 @@ func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { } return tables, nil } - -// Add internal columns -func (c *Client) addInternalColumns(tables []*schema.Table) error { - for _, table := range tables { - cqSourceName := schema.CqSourceNameColumn - cqSyncTime := schema.CqSyncTimeColumn - table.Columns = append([]schema.Column{cqSourceName, cqSyncTime}, table.Columns...) - } - return nil -} diff --git a/plugins/source/postgresql/pgarrow/to_arrow.go b/plugins/source/postgresql/pgarrow/to_arrow.go index 27283d420ce7c6..27980903345407 100644 --- a/plugins/source/postgresql/pgarrow/to_arrow.go +++ b/plugins/source/postgresql/pgarrow/to_arrow.go @@ -78,9 +78,7 @@ func normalize(t string) string { func parseTimestamp(t string) (arrow.DataType, bool) { timestamptzPrefix := "timestamptz using" - if strings.HasPrefix(t, timestamptzPrefix) { - t = strings.TrimPrefix(t, timestamptzPrefix) - } + t = strings.TrimPrefix(t, timestamptzPrefix) if t == "timestamptz" { return arrow.FixedWidthTypes.Timestamp_us, true } From cc4388fa69123d86a9b1c05d044b78d8b752b5c3 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 21 Jun 2023 15:52:55 +0200 Subject: [PATCH 08/10] fix: Don't handle internal columns --- plugins/source/postgresql/client/cdc.go | 11 ----------- plugins/source/postgresql/client/sync.go | 14 -------------- .../postgresql/resources/plugin/plugin_test.go | 7 ------- 3 files changed, 32 deletions(-) diff --git a/plugins/source/postgresql/client/cdc.go b/plugins/source/postgresql/client/cdc.go index 97600350fe497d..1418da894a3149 100644 --- a/plugins/source/postgresql/client/cdc.go +++ b/plugins/source/postgresql/client/cdc.go @@ -283,9 +283,6 @@ func (c *Client) resourceFromCDCValues(tableName string, values map[string]any, table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) for _, col := range table.Columns { - if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { - continue - } v, err := prepareValueForResourceSet(col, values[col.Name]) if err != nil { return nil, err @@ -294,13 +291,5 @@ func (c *Client) resourceFromCDCValues(tableName string, values map[string]any, return nil, err } } - err := resource.Set(schema.CqSourceNameColumn.Name, c.spec.Name) - if err != nil { - return nil, err - } - err = resource.Set(schema.CqSyncTimeColumn.Name, syncTime) - if err != nil { - return nil, err - } return resource, nil } diff --git a/plugins/source/postgresql/client/sync.go b/plugins/source/postgresql/client/sync.go index 2f0db5dbc148a1..6d0ccb78f22d93 100644 --- a/plugins/source/postgresql/client/sync.go +++ b/plugins/source/postgresql/client/sync.go @@ -102,9 +102,6 @@ func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, res chan<- *schema.Resource, syncTime time.Time) error { colNames := make([]string, 0, len(table.Columns)-2) for _, col := range table.Columns { - if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { - continue - } colNames = append(colNames, pgx.Identifier{col.Name}.Sanitize()) } query := "SELECT " + strings.Join(colNames, ",") + " FROM " + pgx.Identifier{table.Name}.Sanitize() @@ -125,14 +122,6 @@ func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, c.metrics.TableClient[table.Name][c.ID()].Errors++ return err } - err = resource.Set(schema.CqSourceNameColumn.Name, c.spec.Name) - if err != nil { - return fmt.Errorf("failed to set CQ source name column: %w", err) - } - err = resource.Set(schema.CqSyncTimeColumn.Name, syncTime) - if err != nil { - return fmt.Errorf("failed to set CQ sync time column: %w", err) - } c.metrics.TableClient[table.Name][c.ID()].Resources++ @@ -146,9 +135,6 @@ func (c *Client) resourceFromValues(tableName string, values []any) (*schema.Res resource := schema.NewResourceData(table, nil, values) var i int for _, col := range table.Columns { - if col.Name == schema.CqSourceNameColumn.Name || col.Name == schema.CqSyncTimeColumn.Name { - continue - } v, err := prepareValueForResourceSet(col, values[i]) if err != nil { return nil, err diff --git a/plugins/source/postgresql/resources/plugin/plugin_test.go b/plugins/source/postgresql/resources/plugin/plugin_test.go index 75c0204916a74e..e8fc264a5e25ed 100644 --- a/plugins/source/postgresql/resources/plugin/plugin_test.go +++ b/plugins/source/postgresql/resources/plugin/plugin_test.go @@ -302,13 +302,6 @@ func TestPlugin(t *testing.T) { t.Fatalf("expected 1 resource, got %d", totalResources) } gotData := resource.GetValues() - if gotData[0].Get().(string) != "test_pg_source" { - t.Fatalf("expected source name to be test_pg_source, got %s", gotData[0]) - } - if !gotData[1].Get().(time.Time).Equal(syncTime.UTC()) { - t.Fatalf("expected sync time to be %s, but got %s", syncTime.UTC(), gotData[1]) - } - gotData = gotData[2:] // ignore the first two columns (_cq_source_name and _cq_sync_time) for i, got := range gotData { expected := data[i].expect if !got.Equal(expected) { From 88310daf0745c587c8a2d5bc5d1f254873da1a85 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 21 Jun 2023 15:57:00 +0200 Subject: [PATCH 09/10] refactor: Cleanup --- plugins/source/postgresql/client/list_tables.go | 1 - plugins/source/postgresql/client/sync.go | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/source/postgresql/client/list_tables.go b/plugins/source/postgresql/client/list_tables.go index cf877ee3afd7e3..e3fe341bed522b 100644 --- a/plugins/source/postgresql/client/list_tables.go +++ b/plugins/source/postgresql/client/list_tables.go @@ -60,7 +60,6 @@ ORDER BY func (c *Client) listTables(ctx context.Context) (schema.Tables, error) { var tables schema.Tables q := fmt.Sprintf(selectTables, c.currentSchemaName) - fmt.Println(q) rows, err := c.Conn.Query(ctx, q) if err != nil { return nil, err diff --git a/plugins/source/postgresql/client/sync.go b/plugins/source/postgresql/client/sync.go index 6d0ccb78f22d93..5ae992d1e31568 100644 --- a/plugins/source/postgresql/client/sync.go +++ b/plugins/source/postgresql/client/sync.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/plugin-sdk/v3/plugins/source" @@ -42,12 +41,10 @@ func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- * } } - syncTime := time.Now() - if c.pluginSpec.CDC && snapshotName == "" { c.logger.Info().Msg("cdc is enabled but replication slot already exists, skipping initial sync") } else { - if err := c.syncTables(ctx, snapshotName, res, syncTime); err != nil { + if err := c.syncTables(ctx, snapshotName, res); err != nil { return err } } @@ -62,7 +59,7 @@ func (c *Client) Sync(ctx context.Context, metrics *source.Metrics, res chan<- * return nil } -func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- *schema.Resource, syncTime time.Time) error { +func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- *schema.Resource) error { tx, err := c.Conn.BeginTx(ctx, pgx.TxOptions{ // this transaction is needed for us to take a snapshot and we need to close it only at the end of the initial sync // https://www.postgresql.org/docs/current/transaction-iso.html @@ -89,7 +86,7 @@ func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- } for _, table := range c.Tables { - if err := c.syncTable(ctx, tx, table, res, syncTime); err != nil { + if err := c.syncTable(ctx, tx, table, res); err != nil { return err } } @@ -99,7 +96,7 @@ func (c *Client) syncTables(ctx context.Context, snapshotName string, res chan<- return nil } -func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, res chan<- *schema.Resource, syncTime time.Time) error { +func (c *Client) syncTable(ctx context.Context, tx pgx.Tx, table *schema.Table, res chan<- *schema.Resource) error { colNames := make([]string, 0, len(table.Columns)-2) for _, col := range table.Columns { colNames = append(colNames, pgx.Identifier{col.Name}.Sanitize()) From d3ff2f3838e2f92f8eb379a65f79ca1da65a19ce Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 21 Jun 2023 16:00:21 +0200 Subject: [PATCH 10/10] refactor: Cleanup --- plugins/source/postgresql/client/cdc.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/source/postgresql/client/cdc.go b/plugins/source/postgresql/client/cdc.go index 1418da894a3149..e07fe61d558eef 100644 --- a/plugins/source/postgresql/client/cdc.go +++ b/plugins/source/postgresql/client/cdc.go @@ -78,7 +78,6 @@ func getReplicationName(specName string) string { } func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) error { - syncTime := time.Now().UTC() connPool, err := c.Conn.Acquire(ctx) if err != nil { return fmt.Errorf("failed to acquire connection: %w", err) @@ -187,7 +186,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) + resource, err := c.resourceFromCDCValues(rel.RelationName, values) if err != nil { return err } @@ -214,7 +213,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) + resource, err := c.resourceFromCDCValues(rel.RelationName, values) if err != nil { return err } @@ -240,7 +239,7 @@ func (c *Client) listenCDC(ctx context.Context, res chan<- *schema.Resource) err values[colName] = val } } - resource, err := c.resourceFromCDCValues(rel.RelationName, values, syncTime) + resource, err := c.resourceFromCDCValues(rel.RelationName, values) if err != nil { return err } @@ -279,7 +278,7 @@ func decodeTextColumnData(mi *pgtype.Map, data []byte, dataType uint32) (any, er return string(data), nil } -func (c *Client) resourceFromCDCValues(tableName string, values map[string]any, syncTime time.Time) (*schema.Resource, error) { +func (c *Client) resourceFromCDCValues(tableName string, values map[string]any) (*schema.Resource, error) { table := c.Tables.Get(tableName) resource := schema.NewResourceData(table, nil, values) for _, col := range table.Columns {