Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 8ac0bf8

Browse files
elekStorj Robot
elek
authored and
Storj Robot
committed
satellite/changestream: log failed partition processing
Change-Id: I6aa08855d0bc23762b9fa98066eb98f703fc8f74
1 parent b56a7e7 commit 8ac0bf8

File tree

4 files changed

+13
-6
lines changed

4 files changed

+13
-6
lines changed

‎satellite/eventing/service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func NewService(log *zap.Logger, sdb changestream.Adapter, projects PublicProjec
6868
func (s *Service) Run(ctx context.Context) error {
6969
// TODO: we need to persist the last processed timestamp, time to time
7070
start := time.Now()
71-
return changestream.Processor(ctx, s.db, s.cfg.Feedname, start, func(record changestream.DataChangeRecord) error {
71+
return changestream.Processor(ctx, s.log, s.db, s.cfg.Feedname, start, func(record changestream.DataChangeRecord) error {
7272
// Ignore errors here, they are logged inside ProcessRecord
7373
_ = s.ProcessRecord(ctx, record)
7474
return nil

‎satellite/metabase/changestream/change_stream_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313

1414
"cloud.google.com/go/spanner"
1515
"github.com/stretchr/testify/require"
16+
"go.uber.org/zap"
17+
"go.uber.org/zap/zaptest"
1618
"google.golang.org/api/iterator"
1719

1820
"storj.io/common/testcontext"
@@ -348,7 +350,8 @@ func setupChangeStreamTest(ctx context.Context, t *testing.T, db *metabase.DB) (
348350
require.NoError(t, err, "Should be able to check change stream existence")
349351
require.True(t, changeStreamExists, "Change stream should exist")
350352

351-
eventCh, errCh, cleanup = startChangeStreamReader(ctx, adapter, "bucket_eventing")
353+
log := zaptest.NewLogger(t)
354+
eventCh, errCh, cleanup = startChangeStreamReader(ctx, log, adapter, "bucket_eventing")
352355

353356
// Give the reader a moment to start up
354357
time.Sleep(2 * time.Second)
@@ -461,7 +464,7 @@ func verifyChangeStreamExists(ctx context.Context, adapter *metabase.SpannerAdap
461464

462465
// startChangeStreamReader starts reading change stream events in the background
463466
// and returns a channel that will receive events as they occur
464-
func startChangeStreamReader(ctx context.Context, adapter *metabase.SpannerAdapter, streamName string) (<-chan changestream.DataChangeRecord, <-chan error, func()) {
467+
func startChangeStreamReader(ctx context.Context, log *zap.Logger, adapter *metabase.SpannerAdapter, streamName string) (<-chan changestream.DataChangeRecord, <-chan error, func()) {
465468
eventCh := make(chan changestream.DataChangeRecord, 100) // Buffer to avoid blocking
466469
errCh := make(chan error, 1)
467470

@@ -475,7 +478,7 @@ func startChangeStreamReader(ctx context.Context, adapter *metabase.SpannerAdapt
475478

476479
startTime := time.Now()
477480

478-
err := changestream.Processor(processorCtx, adapter, streamName, startTime, func(record changestream.DataChangeRecord) error {
481+
err := changestream.Processor(processorCtx, log, adapter, streamName, startTime, func(record changestream.DataChangeRecord) error {
479482
select {
480483
case eventCh <- record:
481484
case <-processorCtx.Done():

‎satellite/metabase/changestream/changestream_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"cloud.google.com/go/spanner"
1212
"github.com/stretchr/testify/require"
13+
"go.uber.org/zap/zaptest"
1314
"google.golang.org/grpc/codes"
1415

1516
"storj.io/common/errs2"
@@ -21,6 +22,7 @@ import (
2122
)
2223

2324
func TestChangeStream(t *testing.T) {
25+
log := zaptest.NewLogger(t)
2426
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
2527
streamId := metabasetest.RandObjectStream()
2628
adapter := db.ChooseAdapter(streamId.ProjectID)
@@ -47,7 +49,7 @@ func TestChangeStream(t *testing.T) {
4749
changes := make(chan changestream.DataChangeRecord)
4850
feedErr := make(chan error)
4951
go func() {
50-
err = changestream.Processor(feedCtx, spannerAdapter, changefeedName, startTime, func(record changestream.DataChangeRecord) error {
52+
err = changestream.Processor(feedCtx, log, spannerAdapter, changefeedName, startTime, func(record changestream.DataChangeRecord) error {
5153
changes <- record
5254
return nil
5355
})

‎satellite/metabase/changestream/processor.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,15 @@ import (
99
"time"
1010

1111
"cloud.google.com/go/spanner"
12+
"go.uber.org/zap"
1213
"golang.org/x/sync/errgroup"
1314
"google.golang.org/grpc/codes"
1415

1516
"storj.io/common/errs2"
1617
)
1718

1819
// Processor processes change stream records in batches (parallel). This contains the logic to follow child partitions.
19-
func Processor(ctx context.Context, adapter Adapter, feedName string, startTime time.Time, fn func(record DataChangeRecord) error) error {
20+
func Processor(ctx context.Context, log *zap.Logger, adapter Adapter, feedName string, startTime time.Time, fn func(record DataChangeRecord) error) error {
2021
tracker := &Tracker{
2122
todo: make(map[string]Todo),
2223
status: make(map[string]TodoStatus),
@@ -41,6 +42,7 @@ func Processor(ctx context.Context, adapter Adapter, feedName string, startTime
4142
tracker.NotifyReady()
4243
}
4344
//nolint
45+
log.Warn("failed to process partition (will be retried)", zap.String("token", todoItem.Token), zap.Error(err))
4446
return nil
4547
}
4648
for _, partition := range partitions {

0 commit comments

Comments
 (0)