Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 65d05bb

Browse files
2403905butonic
2403905
authored and
butonic
committed
feat: fix the graceful shutdown using the new ocis and reva runners
Signed-off-by: Jörn Friedrich Dreyer <[email protected]>
1 parent 7727c3f commit 65d05bb

27 files changed

Lines changed: 912 additions & 815 deletions

File tree

‎changelog/unreleased/fix-graceful-shutdown.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Bugfix: Fix the graceful shutdown
2+
3+
Fix the graceful shutdown using the new ocis and reva runners.
4+
5+
https://github.com/owncloud/ocis/pull/11295
6+
https://github.com/owncloud/ocis/issues/11170

‎opencloud/pkg/runtime/service/service.go‎

Lines changed: 58 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,23 @@ package service
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"net"
78
"net/http"
89
"net/rpc"
9-
"os"
10+
"os/signal"
1011
"sort"
1112
"strings"
13+
"sync"
1214
"time"
1315

1416
"github.com/cenkalti/backoff"
1517
"github.com/mohae/deepcopy"
1618
"github.com/olekukonko/tablewriter"
1719
occfg "github.com/opencloud-eu/opencloud/pkg/config"
1820
"github.com/opencloud-eu/opencloud/pkg/log"
21+
"github.com/opencloud-eu/opencloud/pkg/runner"
1922
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
2023
"github.com/opencloud-eu/opencloud/pkg/shared"
2124
activitylog "github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
@@ -358,8 +361,9 @@ func Start(ctx context.Context, o ...Option) error {
358361
return err
359362
}
360363

361-
// get a cancel function to stop the service
362-
ctx, cancel := context.WithCancel(ctx)
364+
// cancel the context when a signal is received.
365+
notifyCtx, cancel := signal.NotifyContext(ctx, runner.StopSignals...)
366+
defer cancel()
363367

364368
// tolerance controls backoff cycles from the supervisor.
365369
tolerance := 5
@@ -397,14 +401,14 @@ func Start(ctx context.Context, o ...Option) error {
397401
if err != nil {
398402
s.Log.Fatal().Err(err).Msg("could not start listener")
399403
}
404+
srv := new(http.Server)
400405

401406
defer func() {
402407
if r := recover(); r != nil {
403408
reason := strings.Builder{}
404409
if _, err = net.Dial("tcp", net.JoinHostPort(s.cfg.Runtime.Host, s.cfg.Runtime.Port)); err != nil {
405410
reason.WriteString("runtime address already in use")
406411
}
407-
408412
fmt.Println(reason.String())
409413
}
410414
}()
@@ -417,10 +421,7 @@ func Start(ctx context.Context, o ...Option) error {
417421
// go supervisor.Serve()
418422
// because that will briefly create a race condition as it starts up, if you try to .Add() services immediately afterward.
419423
// https://pkg.go.dev/github.com/thejerf/suture/[email protected]#Supervisor
420-
go s.Supervisor.ServeBackground(s.context)
421-
422-
// trap will block on context done channel for interruptions.
423-
go trap(s, ctx)
424+
go s.Supervisor.ServeBackground(s.context) // TODO Why does Supervisor uses s.context?
424425

425426
for i, service := range s.Services {
426427
scheduleServiceTokens(s, service)
@@ -434,7 +435,15 @@ func Start(ctx context.Context, o ...Option) error {
434435
// schedule services that are optional
435436
scheduleServiceTokens(s, s.Additional)
436437

437-
return http.Serve(l, nil)
438+
go func() {
439+
if err = srv.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) {
440+
s.Log.Fatal().Err(err).Msg("could not start rpc server")
441+
}
442+
}()
443+
444+
// trapShutdownCtx will block on the context-done channel for interruptions.
445+
trapShutdownCtx(s, srv, notifyCtx)
446+
return nil
438447
}
439448

440449
// scheduleServiceTokens adds service tokens to the service supervisor.
@@ -501,20 +510,51 @@ func (s *Service) List(_ struct{}, reply *string) error {
501510
return nil
502511
}
503512

504-
// trap blocks on halt channel. When the runtime is interrupted it
505-
// signals the controller to stop any supervised process.
506-
func trap(s *Service, ctx context.Context) {
513+
func trapShutdownCtx(s *Service, srv *http.Server, ctx context.Context) {
507514
<-ctx.Done()
515+
wg := sync.WaitGroup{}
516+
wg.Add(1)
517+
go func() {
518+
defer wg.Done()
519+
// TODO: To discuss the default timeout
520+
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
521+
defer cancel()
522+
if err := srv.Shutdown(ctx); err != nil {
523+
s.Log.Error().Err(err).Msg("could not shutdown tcp listener")
524+
return
525+
}
526+
s.Log.Info().Msg("tcp listener shutdown")
527+
}()
528+
508529
for sName := range s.serviceToken {
509530
for i := range s.serviceToken[sName] {
510-
if err := s.Supervisor.Remove(s.serviceToken[sName][i]); err != nil {
511-
s.Log.Error().Err(err).Str("service", "runtime service").Msgf("terminating with signal: %v", s)
512-
}
531+
wg.Add(1)
532+
go func() {
533+
s.Log.Warn().Msgf("=== RemoveAndWait for %s", sName)
534+
defer wg.Done()
535+
// TODO: To discuss the default timeout
536+
if err := s.Supervisor.RemoveAndWait(s.serviceToken[sName][i], 20*time.Second); err != nil && !errors.Is(err, suture.ErrSupervisorNotRunning) {
537+
s.Log.Error().Err(err).Str("service", sName).Msgf("terminating with signal: %+v", s)
538+
}
539+
s.Log.Warn().Msgf("=== Done RemoveAndWait for %s", sName)
540+
}()
513541
}
514542
}
515-
s.Log.Debug().Str("service", "runtime service").Msgf("terminating with signal: %v", s)
516-
time.Sleep(3 * time.Second) // give the services time to deregister
517-
os.Exit(0) // FIXME this cause an early exit that prevents services from shitting down properly
543+
544+
done := make(chan struct{})
545+
go func() {
546+
wg.Wait()
547+
close(done)
548+
}()
549+
550+
select {
551+
// TODO: To discuss the default timeout
552+
case <-time.After(30 * time.Second):
553+
s.Log.Fatal().Msg("ocis graceful shutdown timeout reached, terminating")
554+
case <-done:
555+
s.Log.Info().Msg("all ocis services gracefully stopped")
556+
return
557+
}
518558
}
519559

520560
// pingNats will attempt to connect to nats, blocking until a connection is established

‎pkg/runner/factory.go‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
1111
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
12+
"github.com/opencloud-eu/reva/v2/cmd/revad/runtime"
1213
"google.golang.org/grpc"
1314
)
1415

@@ -102,7 +103,8 @@ func NewGolangHttpServerRunner(name string, server *http.Server, opts ...Option)
102103
// Since Shutdown might take some time, don't block
103104
go func() {
104105
// give 5 secs for the shutdown to finish
105-
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
106+
// TODO: To discuss the default timeout
107+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
106108
defer cancel()
107109

108110
debugCh <- server.Shutdown(shutdownCtx)
@@ -132,3 +134,23 @@ func NewGolangGrpcServerRunner(name string, server *grpc.Server, listener net.Li
132134

133135
return r
134136
}
137+
138+
func NewRevaServiceRunner(name string, server runtime.RevaDrivenServer, opts ...Option) *Runner {
139+
httpCh := make(chan error, 1)
140+
r := New(name, func() error {
141+
// start the server and return if it fails
142+
if err := server.Start(); err != nil {
143+
return err
144+
}
145+
return <-httpCh // wait for the result
146+
}, func() {
147+
// stop implies deregistering and waiting for the request to finish,
148+
// so don't block
149+
go func() {
150+
httpCh <- server.Stop() // stop and send a result through a channel
151+
close(httpCh)
152+
}()
153+
}, opts...)
154+
155+
return r
156+
}

‎pkg/runner/option.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ import (
77
var (
88
// DefaultInterruptDuration is the default value for the `WithInterruptDuration`
99
// for the "regular" runners. This global value can be adjusted if needed.
10-
DefaultInterruptDuration = 10 * time.Second
10+
// TODO: To discuss the default timeout
11+
DefaultInterruptDuration = 20 * time.Second
1112
// DefaultGroupInterruptDuration is the default value for the `WithInterruptDuration`
1213
// for the group runners. This global value can be adjusted if needed.
13-
DefaultGroupInterruptDuration = 15 * time.Second
14+
// TODO: To discuss the default timeout
15+
DefaultGroupInterruptDuration = 25 * time.Second
1416
)
1517

1618
// Option defines a single option function.

‎services/app-provider/pkg/command/server.go‎

Lines changed: 33 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"os/signal"
78
"path"
89

910
"github.com/gofrs/uuid"
10-
"github.com/oklog/run"
1111
"github.com/opencloud-eu/reva/v2/cmd/revad/runtime"
1212
"github.com/urfave/cli/v2"
1313

1414
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
1515
"github.com/opencloud-eu/opencloud/pkg/registry"
16+
"github.com/opencloud-eu/opencloud/pkg/runner"
1617
"github.com/opencloud-eu/opencloud/pkg/tracing"
1718
"github.com/opencloud-eu/opencloud/pkg/version"
1819
"github.com/opencloud-eu/opencloud/services/app-provider/pkg/config"
@@ -37,66 +38,58 @@ func Server(cfg *config.Config) *cli.Command {
3738
if err != nil {
3839
return err
3940
}
40-
gr := run.Group{}
41-
ctx, cancel := context.WithCancel(c.Context)
4241

43-
defer cancel()
42+
var cancel context.CancelFunc
43+
ctx := cfg.Context
44+
if ctx == nil {
45+
ctx, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
46+
defer cancel()
47+
}
4448

45-
// make sure the run group executes all interrupt handlers when the context is canceled
46-
gr.Add(func() error {
47-
<-ctx.Done()
48-
return nil
49-
}, func(_ error) {
50-
})
49+
gr := runner.NewGroup()
5150

52-
gr.Add(func() error {
51+
{
5352
pidFile := path.Join(os.TempDir(), "revad-"+cfg.Service.Name+"-"+uuid.Must(uuid.NewV4()).String()+".pid")
5453
rCfg := revaconfig.AppProviderConfigFromStruct(cfg)
5554
reg := registry.GetRegistry()
5655

57-
runtime.RunWithOptions(rCfg, pidFile,
56+
revaSrv := runtime.RunDrivenServerWithOptions(rCfg, pidFile,
5857
runtime.WithLogger(&logger.Logger),
5958
runtime.WithRegistry(reg),
6059
runtime.WithTraceProvider(traceProvider),
6160
)
6261

63-
return nil
64-
}, func(err error) {
65-
if err == nil {
66-
logger.Info().
67-
Str("transport", "reva").
68-
Str("server", cfg.Service.Name).
69-
Msg("Shutting down server")
70-
} else {
71-
logger.Error().Err(err).
72-
Str("transport", "reva").
73-
Str("server", cfg.Service.Name).
74-
Msg("Shutting down server")
75-
}
62+
gr.Add(runner.NewRevaServiceRunner("app-provider_revad", revaSrv))
63+
}
7664

77-
cancel()
78-
})
65+
{
66+
debugServer, err := debug.Server(
67+
debug.Logger(logger),
68+
debug.Context(ctx),
69+
debug.Config(cfg),
70+
)
71+
if err != nil {
72+
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
73+
return err
74+
}
7975

80-
debugServer, err := debug.Server(
81-
debug.Logger(logger),
82-
debug.Context(ctx),
83-
debug.Config(cfg),
84-
)
85-
if err != nil {
86-
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
87-
return err
76+
gr.Add(runner.NewGolangHttpServerRunner("app-provider_debug", debugServer))
8877
}
8978

90-
gr.Add(debugServer.ListenAndServe, func(_ error) {
91-
cancel()
92-
})
93-
9479
grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString())
9580
if err := registry.RegisterService(ctx, logger, grpcSvc, cfg.Debug.Addr); err != nil {
9681
logger.Fatal().Err(err).Msg("failed to register the grpc service")
9782
}
9883

99-
return gr.Run()
84+
grResults := gr.Run(ctx)
85+
86+
// return the first non-nil error found in the results
87+
for _, grResult := range grResults {
88+
if grResult.RunnerError != nil {
89+
return grResult.RunnerError
90+
}
91+
}
92+
return nil
10093
},
10194
}
10295
}

0 commit comments

Comments
 (0)