Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e6a74ef

Browse files
embano1embano1
authored
Merge pull request #988 from yanmxa/br_kafka_confluent
2 parents e118aba + 5cd87ea commit e6a74ef

File tree

20 files changed

+1499
-5
lines changed

20 files changed

+1499
-5
lines changed

‎.github/workflows/integration.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,15 @@ jobs:
2727
- 9091:9091
2828
- 9092:9092
2929

30+
kafka_confluent:
31+
image: confluentinc/confluent-local:7.6.0
32+
ports:
33+
- "9192:9192"
34+
env:
35+
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:29192,PLAINTEXT_HOST://localhost:9192'
36+
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:29193'
37+
KAFKA_LISTENERS: 'PLAINTEXT://localhost:29192,CONTROLLER://localhost:29193,PLAINTEXT_HOST://0.0.0.0:9192'
38+
3039
natss:
3140
image: nats-streaming:0.22.1
3241
ports:

‎docs/index.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,8 @@ err := json.Unmarshal(bytes, &event)
124124
| AVRO Event Format | :x: | :x: |
125125
| [HTTP Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/http) | :heavy_check_mark: | :heavy_check_mark: |
126126
| [JSON Event Format](event_data_structure.md#marshalunmarshal-event-to-json) | :heavy_check_mark: | :heavy_check_mark: |
127-
| [Kafka Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/kafka) | :heavy_check_mark: | :heavy_check_mark: |
127+
| [Sarama Kafka Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/kafka) | :heavy_check_mark: | :heavy_check_mark: |
128+
| [Confluent Kafka Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/kafka_confluent) | :heavy_check_mark: | :heavy_check_mark: |
128129
| MQTT Protocol Binding | :x: | :x: |
129130
| [NATS Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/nats) | :heavy_check_mark: | :heavy_check_mark: |
130131
| [STAN Protocol Binding](https://github.com/cloudevents/sdk-go/tree/main/samples/stan) | :heavy_check_mark: | :heavy_check_mark: |

‎protocol/kafka_confluent/v2/go.mod‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
module github.com/cloudevents/sdk-go/protocol/kafka_confluent/v2
2+
3+
go 1.18
4+
5+
replace github.com/cloudevents/sdk-go/v2 => ../../../v2
6+
7+
require (
8+
github.com/cloudevents/sdk-go/v2 v2.15.2
9+
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
10+
github.com/stretchr/testify v1.8.4
11+
)
12+
13+
require (
14+
github.com/davecgh/go-spew v1.1.1 // indirect
15+
github.com/google/go-cmp v0.5.9 // indirect
16+
github.com/google/uuid v1.3.0 // indirect
17+
github.com/json-iterator/go v1.1.11 // indirect
18+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
19+
github.com/modern-go/reflect2 v1.0.1 // indirect
20+
github.com/pmezard/go-difflib v1.0.0 // indirect
21+
go.uber.org/atomic v1.4.0 // indirect
22+
go.uber.org/multierr v1.1.0 // indirect
23+
go.uber.org/zap v1.10.0 // indirect
24+
gopkg.in/yaml.v3 v3.0.1 // indirect
25+
)

‎protocol/kafka_confluent/v2/go.sum‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
2+
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
3+
github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I=
4+
github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
5+
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts=
6+
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8=
7+
github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA=
8+
github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs=
9+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
11+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
12+
github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
13+
github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE=
14+
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
15+
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
16+
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
17+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
18+
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
19+
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
20+
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
21+
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
22+
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
23+
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
24+
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
25+
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
26+
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
27+
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
28+
github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
29+
github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
30+
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
31+
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
32+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
33+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
34+
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
35+
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
36+
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
37+
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
38+
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
39+
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
40+
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
41+
github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w=
42+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
43+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
44+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
45+
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
46+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
47+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
48+
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
49+
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
50+
github.com/testcontainers/testcontainers-go v0.14.0 h1:h0D5GaYG9mhOWr2qHdEKDXpkce/VlvaYOCzTRi6UBi8=
51+
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
52+
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
53+
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
54+
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
55+
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
56+
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
57+
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
58+
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
59+
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
60+
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
61+
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
62+
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08=
63+
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
64+
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
65+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
66+
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
67+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
68+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎protocol/kafka_confluent/v2/message.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
Copyright 2023 The CloudEvents Authors
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package kafka_confluent
7+
8+
import (
9+
"bytes"
10+
"context"
11+
"strconv"
12+
"strings"
13+
14+
"github.com/cloudevents/sdk-go/v2/binding"
15+
"github.com/cloudevents/sdk-go/v2/binding/format"
16+
"github.com/cloudevents/sdk-go/v2/binding/spec"
17+
"github.com/confluentinc/confluent-kafka-go/v2/kafka"
18+
)
19+
20+
const (
21+
prefix = "ce-"
22+
contentTypeKey = "content-type"
23+
)
24+
25+
const (
26+
KafkaOffsetKey = "kafkaoffset"
27+
KafkaPartitionKey = "kafkapartition"
28+
KafkaTopicKey = "kafkatopic"
29+
KafkaMessageKey = "kafkamessagekey"
30+
)
31+
32+
var specs = spec.WithPrefix(prefix)
33+
34+
// Message represents a Kafka message.
35+
// This message *can* be read several times safely
36+
type Message struct {
37+
internal *kafka.Message
38+
properties map[string][]byte
39+
format format.Format
40+
version spec.Version
41+
}
42+
43+
// Check if Message implements binding.Message
44+
var (
45+
_ binding.Message = (*Message)(nil)
46+
_ binding.MessageMetadataReader = (*Message)(nil)
47+
)
48+
49+
// NewMessage returns a binding.Message that holds the provided kafka.Message.
50+
// The returned binding.Message *can* be read several times safely
51+
// This function *doesn't* guarantee that the returned binding.Message is always a kafka_sarama.Message instance
52+
func NewMessage(msg *kafka.Message) *Message {
53+
if msg == nil {
54+
panic("the kafka.Message shouldn't be nil")
55+
}
56+
if msg.TopicPartition.Topic == nil {
57+
panic("the topic of kafka.Message shouldn't be nil")
58+
}
59+
if msg.TopicPartition.Partition < 0 || msg.TopicPartition.Offset < 0 {
60+
panic("the partition or offset of the kafka.Message must be non-negative")
61+
}
62+
63+
var contentType, contentVersion string
64+
properties := make(map[string][]byte, len(msg.Headers)+3)
65+
for _, header := range msg.Headers {
66+
k := strings.ToLower(string(header.Key))
67+
if k == strings.ToLower(contentTypeKey) {
68+
contentType = string(header.Value)
69+
}
70+
if k == specs.PrefixedSpecVersionName() {
71+
contentVersion = string(header.Value)
72+
}
73+
properties[k] = header.Value
74+
}
75+
76+
// add the kafka message key, topic, partition and partition key to the properties
77+
properties[prefix+KafkaOffsetKey] = []byte(strconv.FormatInt(int64(msg.TopicPartition.Offset), 10))
78+
properties[prefix+KafkaPartitionKey] = []byte(strconv.FormatInt(int64(msg.TopicPartition.Partition), 10))
79+
properties[prefix+KafkaTopicKey] = []byte(*msg.TopicPartition.Topic)
80+
if msg.Key != nil {
81+
properties[prefix+KafkaMessageKey] = msg.Key
82+
}
83+
84+
message := &Message{
85+
internal: msg,
86+
properties: properties,
87+
}
88+
if ft := format.Lookup(contentType); ft != nil {
89+
message.format = ft
90+
} else if v := specs.Version(contentVersion); v != nil {
91+
message.version = v
92+
}
93+
94+
return message
95+
}
96+
97+
func (m *Message) ReadEncoding() binding.Encoding {
98+
if m.version != nil {
99+
return binding.EncodingBinary
100+
}
101+
if m.format != nil {
102+
return binding.EncodingStructured
103+
}
104+
return binding.EncodingUnknown
105+
}
106+
107+
func (m *Message) ReadStructured(ctx context.Context, encoder binding.StructuredWriter) error {
108+
if m.format != nil {
109+
return encoder.SetStructuredEvent(ctx, m.format, bytes.NewReader(m.internal.Value))
110+
}
111+
return binding.ErrNotStructured
112+
}
113+
114+
func (m *Message) ReadBinary(ctx context.Context, encoder binding.BinaryWriter) error {
115+
if m.version == nil {
116+
return binding.ErrNotBinary
117+
}
118+
119+
var err error
120+
for k, v := range m.properties {
121+
if strings.HasPrefix(k, prefix) {
122+
attr := m.version.Attribute(k)
123+
if attr != nil {
124+
err = encoder.SetAttribute(attr, string(v))
125+
} else {
126+
err = encoder.SetExtension(strings.TrimPrefix(k, prefix), string(v))
127+
}
128+
} else if k == strings.ToLower(contentTypeKey) {
129+
err = encoder.SetAttribute(m.version.AttributeFromKind(spec.DataContentType), string(v))
130+
}
131+
if err != nil {
132+
return err
133+
}
134+
}
135+
136+
if m.internal.Value != nil {
137+
err = encoder.SetData(bytes.NewBuffer(m.internal.Value))
138+
}
139+
return err
140+
}
141+
142+
func (m *Message) Finish(error) error {
143+
return nil
144+
}
145+
146+
func (m *Message) GetAttribute(k spec.Kind) (spec.Attribute, interface{}) {
147+
attr := m.version.AttributeFromKind(k)
148+
if attr == nil {
149+
return nil, nil
150+
}
151+
return attr, m.properties[attr.PrefixedName()]
152+
}
153+
154+
func (m *Message) GetExtension(name string) interface{} {
155+
return m.properties[prefix+name]
156+
}

0 commit comments

Comments
 (0)