Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fixed panic with signals #707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 26, 2024
Merged

Fixed panic with signals #707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ed21294
Added removed tests dependencies
Zhaars Jun 24, 2024
e6f5d5c
Fixed signals handling
Zhaars Jun 25, 2024
a86a720
Redo extra changes
Zhaars Jun 25, 2024
5f4ee00
Merge branch 'master' into zhars/fixed_panic_with_signals
Zhaars Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 73 additions & 71 deletions cmd/acra-server/acra-server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -693,9 +693,9 @@ func realMain() error {

// SIGTERM should be handled only once but potentially it may be invoked twice
// if HTTP API is running simultaneously with SQL queries handler (Start and StartCommands)
var once sync.Once
var onceSIGTERM sync.Once
sigHandlerSIGTERM.AddCallback(func() {
once.Do(func() {
onceSIGTERM.Do(func() {
log.Infof("Received incoming SIGTERM or SIGINT signal")
server.StopListeners()
server.Close()
Expand All @@ -708,89 +708,91 @@ func realMain() error {

// we initialize pipeWrite only in SIGHUP handler
var pipeWrite *os.File
var onceSIGHUP sync.Once
sigHandlerSIGHUP.AddCallback(func() {
shutdownCurrentInstance := func(err error) {
server.Close()
cancel()
server.Exit(err)
}
onceSIGHUP.Do(func() {
shutdownCurrentInstance := func(err error) {
server.Close()
cancel()
server.Exit(err)
}

log.Infof("Received incoming SIGHUP signal")
log.Debugf("Stop accepting new connections, waiting until current connections close")
log.Infof("Received incoming SIGHUP signal")
log.Debugf("Stop accepting new connections, waiting until current connections close")

// Stop accepting requests
server.StopListeners()
// Stop accepting requests
server.StopListeners()

// Get socket file descriptor to pass it to fork
var fdACRA, fdAPI uintptr
fdACRA, err = network.ListenerFileDescriptor(server.ListenerAcra())
if err != nil {
log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetFileDescriptor).
Errorln("System error: failed to get acra-socket file descriptor:", err)
shutdownCurrentInstance(err)
return
}
if enableHTTPAPI {
fdAPI, err = network.ListenerFileDescriptor(server.ListenerAPI())
// Get socket file descriptor to pass it to fork
var fdACRA, fdAPI uintptr
fdACRA, err = network.ListenerFileDescriptor(server.ListenerAcra())
if err != nil {
log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetFileDescriptor).
Errorln("System error: failed to get api-socket file descriptor:", err)
Errorln("System error: failed to get acra-socket file descriptor:", err)
shutdownCurrentInstance(err)
return
}
}
if enableHTTPAPI {
fdAPI, err = network.ListenerFileDescriptor(server.ListenerAPI())
if err != nil {
log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantGetFileDescriptor).
Errorln("System error: failed to get api-socket file descriptor:", err)
shutdownCurrentInstance(err)
return
}
}

// Set env flag for forked process
if err := os.Setenv(GracefulRestartEnv, "true"); err != nil {
log.WithError(err).Errorf("Unexpected error on os.Setenv, graceful restart won't work. Please check env variables, especially %s", GracefulRestartEnv)
shutdownCurrentInstance(err)
return
}
// Set env flag for forked process
if err := os.Setenv(GracefulRestartEnv, "true"); err != nil {
log.WithError(err).Errorf("Unexpected error on os.Setenv, graceful restart won't work. Please check env variables, especially %s", GracefulRestartEnv)
shutdownCurrentInstance(err)
return
}

// We use inter-process pipe for synchronizing parent process and child forked process.
// When forked process starts, it blocks on reading signal from pipe that parent process
// finished with graceful shutdown. This is important for audit log mechanism that requires
// consistency of log stream
var pipeRead *os.File
pipeRead, pipeWrite, err = os.Pipe()
if err != nil {
log.WithError(err).Errorln("Can't create inter-process pipe")
shutdownCurrentInstance(err)
return
}
// We use inter-process pipe for synchronizing parent process and child forked process.
// When forked process starts, it blocks on reading signal from pipe that parent process
// finished with graceful shutdown. This is important for audit log mechanism that requires
// consistency of log stream
var pipeRead *os.File
pipeRead, pipeWrite, err = os.Pipe()
if err != nil {
log.WithError(err).Errorln("Can't create inter-process pipe")
shutdownCurrentInstance(err)
return
}

execSpec := &syscall.ProcAttr{
Env: os.Environ(),
Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), fdACRA, fdAPI, pipeRead.Fd()},
}
execSpec := &syscall.ProcAttr{
Env: os.Environ(),
Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), fdACRA, fdAPI, pipeRead.Fd()},
}

log.Debugf("Forking new process of %s", ServiceName)
executable, err := os.Executable()
if err != nil {
log.WithError(err).Errorln("Can't find full binary path")
shutdownCurrentInstance(err)
return
}
// Fork new process
fork, err := syscall.ForkExec(executable, os.Args, execSpec)
if err != nil {
log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantForkProcess).
WithField("executable", executable).
Errorln("System error: failed to fork new process", err)
shutdownCurrentInstance(err)
return
}
log.Infof("%s process forked to PID: %v", ServiceName, fork)
log.Debugf("Forking new process of %s", ServiceName)
executable, err := os.Executable()
if err != nil {
log.WithError(err).Errorln("Can't find full binary path")
shutdownCurrentInstance(err)
return
}
// Fork new process
fork, err := syscall.ForkExec(executable, os.Args, execSpec)
if err != nil {
log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantForkProcess).
WithField("executable", executable).
Errorln("System error: failed to fork new process", err)
shutdownCurrentInstance(err)
return
}
log.Infof("%s process forked to PID: %v", ServiceName, fork)

// Wait a maximum of N seconds for existing connections to finish
err = server.WaitWithTimeout(time.Duration(closeConnectionTimeout) * time.Second)
if err == common.ErrWaitTimeout {
log.Warningf("Server shutdown Timeout: %d active connections will be cut", server.ConnectionsCounter())
}
log.Infof("Server graceful restart completed, bye PID: %v", os.Getpid())
// Stop the old server, all the connections have been closed and the new one is running
shutdownCurrentInstance(nil)
return
// Wait a maximum of N seconds for existing connections to finish
err = server.WaitWithTimeout(time.Duration(closeConnectionTimeout) * time.Second)
if err == common.ErrWaitTimeout {
log.Warningf("Server shutdown Timeout: %d active connections will be cut", server.ConnectionsCounter())
}
log.Infof("Server graceful restart completed, bye PID: %v", os.Getpid())
// Stop the old server, all the connections have been closed and the new one is running
shutdownCurrentInstance(nil)
})
})

log.Infof("Start listening to connections. Current PID: %v", os.Getpid())
Expand Down
42 changes: 23 additions & 19 deletions cmd/acra-server/common/listener.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ type SServer struct {
stopListenersSignal chan bool
errCh chan error
lock sync.RWMutex
stopOnce sync.Once
exitOnce sync.Once
}

// ErrWaitTimeout error indicates that server was shutdown and waited N seconds while shutting down all connections.
Expand Down Expand Up @@ -394,27 +396,29 @@ func stopAcceptConnections(listener network.DeadlineListener) (err error) {

// StopListeners stops accepts new connections, and stops existing listeners with deadline.
func (server *SServer) StopListeners() {
// Use this channel for signaling of closed listeners according to
// https://stackoverflow.com/questions/13417095/how-do-i-stop-a-listening-server-in-go
close(server.stopListenersSignal)

var err error
var deadlineListener network.DeadlineListener
log.Debugln("Stopping listeners")
server.lock.RLock()
for _, listener := range server.listeners {

deadlineListener, err = network.CastListenerToDeadline(listener)
if err != nil {
log.WithError(err).Warningln("Listener doesn't support deadlines")
continue
}
server.stopOnce.Do(func() {
// Use this channel for signaling of closed listeners according to
// https://stackoverflow.com/questions/13417095/how-do-i-stop-a-listening-server-in-go
close(server.stopListenersSignal)

var err error
var deadlineListener network.DeadlineListener
log.Debugln("Stopping listeners")
server.lock.RLock()
for _, listener := range server.listeners {

deadlineListener, err = network.CastListenerToDeadline(listener)
if err != nil {
log.WithError(err).Warningln("Listener doesn't support deadlines")
continue
}

if err = stopAcceptConnections(deadlineListener); err != nil {
log.WithError(err).Warningln("Can't set deadline for listener")
if err = stopAcceptConnections(deadlineListener); err != nil {
log.WithError(err).Warningln("Can't set deadline for listener")
}
}
}
server.lock.RUnlock()
server.lock.RUnlock()
})
}

// WaitConnections waits until connection complete or stops them after duration time.
Expand Down
6 changes: 4 additions & 2 deletions cmd/acra-server/common/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,10 @@ func NewEEAcraServerMainComponent(config *Config, proxyFactory base.ProxyFactory
// Exit exits SServer by sending the input error to the internal channel
// that is listened in Start function
func (server *SServer) Exit(err error) {
server.errCh <- err
close(server.errCh)
server.exitOnce.Do(func() {
server.errCh <- err
close(server.errCh)
})
}

// StartServer starts SServer
Expand Down
5 changes: 5 additions & 0 deletions tests/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4396,9 +4396,14 @@ def testAcraServerReload(self):
config['incoming_connection_string'] = connection_string
dump_yaml_config(config, temp_config.name)
acra.send_signal(signal.SIGHUP)
# signal again, Acra should handle it safely
acra.send_signal(signal.SIGHUP)
# close current connections
test_engine.dispose()

# signal SIGTERM after SIGHUP, Acra should handle it safely
acra.send_signal(signal.SIGTERM)

connect_str = get_engine_connection_string(
self.get_acraserver_connection_string(self.ACRASERVER_PORT), DB_NAME)
tls_args['port'] = self.ACRASERVER_PORT
Expand Down