Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[3006.x] Do not allow duplicate message ids #65247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 31, 2023
Merged

[3006.x] Do not allow duplicate message ids #65247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4ca85b7
Do not allow duplicate message ids
dwoz Sep 20, 2023
f56094f
Use closure to preserve message future
dwoz Sep 20, 2023
02b592c
Fix timeout test
dwoz Sep 21, 2023
ca83e01
Preserve futures with messages
dwoz Sep 28, 2023
ccfeed6
Add test coverage
dwoz Oct 11, 2023
9dbe105
Add changelog
dwoz Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/65114.fixed.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix nonce verification, request server replies do not stomp on eachother.
201 changes: 89 additions & 112 deletions salt/transport/tcp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import socket
import threading
import urllib
import uuid

import salt.ext.tornado
import salt.ext.tornado.concurrent
Expand All @@ -34,6 +35,7 @@
import salt.utils.versions
from salt.exceptions import SaltClientError, SaltReqTimeoutError
from salt.utils.network import ip_bracket
from salt.utils.process import SignalHandlingProcess

if salt.utils.platform.is_windows():
USE_LOAD_BALANCER = True
Expand All @@ -42,7 +44,6 @@

if USE_LOAD_BALANCER:
import salt.ext.tornado.util
from salt.utils.process import SignalHandlingProcess

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -127,69 +128,64 @@ def _set_tcp_keepalive(sock, opts):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)


if USE_LOAD_BALANCER:
class LoadBalancerServer(SignalHandlingProcess):
"""
Raw TCP server which runs in its own process and will listen
for incoming connections. Each incoming connection will be
sent via multiprocessing queue to the workers.
Since the queue is shared amongst workers, only one worker will
handle a given connection.
"""

class LoadBalancerServer(SignalHandlingProcess):
"""
Raw TCP server which runs in its own process and will listen
for incoming connections. Each incoming connection will be
sent via multiprocessing queue to the workers.
Since the queue is shared amongst workers, only one worker will
handle a given connection.
"""
# TODO: opts!
# Based on default used in salt.ext.tornado.netutil.bind_sockets()
backlog = 128

# TODO: opts!
# Based on default used in salt.ext.tornado.netutil.bind_sockets()
backlog = 128
def __init__(self, opts, socket_queue, **kwargs):
super().__init__(**kwargs)
self.opts = opts
self.socket_queue = socket_queue
self._socket = None
Copy link
Collaborator

@s0undt3ch s0undt3ch Oct 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
self._socket = None
self._socket = None
self.register_finalize_method(self.close)


def __init__(self, opts, socket_queue, **kwargs):
super().__init__(**kwargs)
self.opts = opts
self.socket_queue = socket_queue
def close(self):
if self._socket is not None:
self._socket.shutdown(socket.SHUT_RDWR)
self._socket.close()
self._socket = None

def close(self):
if self._socket is not None:
self._socket.shutdown(socket.SHUT_RDWR)
self._socket.close()
self._socket = None

# pylint: disable=W1701
def __del__(self):
self.close()

# pylint: enable=W1701
# pylint: disable=W1701
def __del__(self):
self.close()
Comment on lines +156 to +158
Copy link
Collaborator

@s0undt3ch s0undt3ch Oct 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# pylint: disable=W1701
def __del__(self):
self.close()

With the register_finalize_method change above, this is no longer necessary.


def run(self):
"""
Start the load balancer
"""
self._socket = _get_socket(self.opts)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
_set_tcp_keepalive(self._socket, self.opts)
self._socket.setblocking(1)
self._socket.bind(_get_bind_addr(self.opts, "ret_port"))
self._socket.listen(self.backlog)
# pylint: enable=W1701

while True:
try:
# Wait for a connection to occur since the socket is
# blocking.
connection, address = self._socket.accept()
# Wait for a free slot to be available to put
# the connection into.
# Sockets are picklable on Windows in Python 3.
self.socket_queue.put((connection, address), True, None)
except OSError as e:
# ECONNABORTED indicates that there was a connection
# but it was closed while still in the accept queue.
# (observed on FreeBSD).
if (
salt.ext.tornado.util.errno_from_exception(e)
== errno.ECONNABORTED
):
continue
raise
def run(self):
"""
Start the load balancer
"""
self._socket = _get_socket(self.opts)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
_set_tcp_keepalive(self._socket, self.opts)
self._socket.setblocking(1)
self._socket.bind(_get_bind_addr(self.opts, "ret_port"))
self._socket.listen(self.backlog)

while True:
try:
# Wait for a connection to occur since the socket is
# blocking.
connection, address = self._socket.accept()
# Wait for a free slot to be available to put
# the connection into.
# Sockets are picklable on Windows in Python 3.
self.socket_queue.put((connection, address), True, None)
except OSError as e:
# ECONNABORTED indicates that there was a connection
# but it was closed while still in the accept queue.
# (observed on FreeBSD).
if salt.ext.tornado.util.errno_from_exception(e) == errno.ECONNABORTED:
continue
raise


class Resolver:
Expand Down Expand Up @@ -467,45 +463,43 @@ def close(self):
raise


if USE_LOAD_BALANCER:

class LoadBalancerWorker(SaltMessageServer):
"""
This will receive TCP connections from 'LoadBalancerServer' via
a multiprocessing queue.
Since the queue is shared amongst workers, only one worker will handle
a given connection.
"""
class LoadBalancerWorker(SaltMessageServer):
"""
This will receive TCP connections from 'LoadBalancerServer' via
a multiprocessing queue.
Since the queue is shared amongst workers, only one worker will handle
a given connection.
"""

def __init__(self, socket_queue, message_handler, *args, **kwargs):
super().__init__(message_handler, *args, **kwargs)
self.socket_queue = socket_queue
self._stop = threading.Event()
self.thread = threading.Thread(target=self.socket_queue_thread)
self.thread.start()
def __init__(self, socket_queue, message_handler, *args, **kwargs):
super().__init__(message_handler, *args, **kwargs)
self.socket_queue = socket_queue
self._stop = threading.Event()
self.thread = threading.Thread(target=self.socket_queue_thread)
self.thread.start()

def close(self):
self._stop.set()
self.thread.join()
super().close()
def close(self):
self._stop.set()
self.thread.join()
super().close()

def socket_queue_thread(self):
try:
while True:
try:
client_socket, address = self.socket_queue.get(True, 1)
except queue.Empty:
if self._stop.is_set():
break
continue
# 'self.io_loop' initialized in super class
# 'salt.ext.tornado.tcpserver.TCPServer'.
# 'self._handle_connection' defined in same super class.
self.io_loop.spawn_callback(
self._handle_connection, client_socket, address
)
except (KeyboardInterrupt, SystemExit):
pass
def socket_queue_thread(self):
try:
while True:
try:
client_socket, address = self.socket_queue.get(True, 1)
except queue.Empty:
if self._stop.is_set():
break
continue
# 'self.io_loop' initialized in super class
# 'salt.ext.tornado.tcpserver.TCPServer'.
# 'self._handle_connection' defined in same super class.
self.io_loop.spawn_callback(
self._handle_connection, client_socket, address
)
except (KeyboardInterrupt, SystemExit):
pass


class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient):
Expand Down Expand Up @@ -569,10 +563,7 @@ def __init__(
self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
with salt.utils.asynchronous.current_ioloop(self.io_loop):
self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver)
self._mid = 1
self._max_messages = int((1 << 31) - 2) # number of IDs before we wrap
# TODO: max queue size
self.send_queue = [] # queue of messages to be sent
self.send_future_map = {} # mapping of request_id -> Future

self._read_until_future = None
Expand All @@ -585,10 +576,6 @@ def __init__(

self.backoff = opts.get("tcp_reconnect_backoff", 1)

def _stop_io_loop(self):
if self.io_loop is not None:
self.io_loop.stop()

# TODO: timeout inflight sessions
def close(self):
if self._closing:
Expand Down Expand Up @@ -722,18 +709,7 @@ def _stream_return(self):
self._stream_return_running = False

def _message_id(self):
wrap = False
while self._mid in self.send_future_map:
if self._mid >= self._max_messages:
if wrap:
# this shouldn't ever happen, but just in case
raise Exception("Unable to find available messageid")
self._mid = 1
wrap = True
else:
self._mid += 1

return self._mid
return str(uuid.uuid4())

# TODO: return a message object which takes care of multiplexing?
def on_recv(self, callback):
Expand Down Expand Up @@ -975,6 +951,7 @@ def publish_daemon(
"""
io_loop = salt.ext.tornado.ioloop.IOLoop()
io_loop.make_current()
self.io_loop = io_loop

# Spin up the publisher
self.pub_server = pub_server = PubServer(
Expand Down
72 changes: 26 additions & 46 deletions salt/transport/zeromq.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from random import randint

import zmq.error
import zmq.eventloop.future
import zmq.eventloop.zmqstream

import salt.ext.tornado
Expand Down Expand Up @@ -513,16 +514,17 @@ def __init__(self, opts, addr, linger=0, io_loop=None):
else:
self.io_loop = io_loop

self.context = zmq.Context()
self.context = zmq.eventloop.future.Context()

self.send_queue = []

self._closing = False
self._future = None
self._send_future_map = {}
self.lock = salt.ext.tornado.locks.Lock()
self.ident = threading.get_ident()

def connect(self):
if hasattr(self, "stream"):
if hasattr(self, "socket") and self.socket:
return
# wire up sockets
self._init_socket()
Expand All @@ -538,24 +540,10 @@ def close(self):
return
else:
self._closing = True
if hasattr(self, "stream") and self.stream is not None:
if ZMQ_VERSION_INFO < (14, 3, 0):
# stream.close() doesn't work properly on pyzmq < 14.3.0
if self.stream.socket:
self.stream.socket.close()
self.stream.io_loop.remove_handler(self.stream.socket)
# set this to None, more hacks for messed up pyzmq
self.stream.socket = None
self.socket.close()
else:
self.stream.close(1)
self.socket = None
self.stream = None
if self._future:
self._future.set_exception(SaltException("Closing connection"))
self._future = None
if hasattr(self, "socket") and self.socket is not None:
self.socket.close(0)
self.socket = None
if self.context.closed is False:
# This hangs if closing the stream causes an import error
self.context.term()

def _init_socket(self):
Expand All @@ -572,23 +560,8 @@ def _init_socket(self):
self.socket.setsockopt(zmq.IPV6, 1)
elif hasattr(zmq, "IPV4ONLY"):
self.socket.setsockopt(zmq.IPV4ONLY, 0)
self.socket.linger = self.linger
self.socket.setsockopt(zmq.LINGER, self.linger)
self.socket.connect(self.addr)
self.stream = zmq.eventloop.zmqstream.ZMQStream(
self.socket, io_loop=self.io_loop
)
self.stream.on_recv(self.handle_reply)

def timeout_message(self, future):
"""
Handle a message timeout by removing it from the sending queue
and informing the caller

:raises: SaltReqTimeoutError
"""
if self._future == future:
self._future = None
future.set_exception(SaltReqTimeoutError("Message timed out"))

@salt.ext.tornado.gen.coroutine
def send(self, message, timeout=None, callback=None):
Expand All @@ -612,20 +585,27 @@ def handle_future(future):

if timeout is not None:
send_timeout = self.io_loop.call_later(
timeout, self.timeout_message, future
timeout, self._timeout_message, future
)

with (yield self.lock.acquire()):
self._future = future
yield self.stream.send(message)
recv = yield future
self.io_loop.spawn_callback(self._send_recv, message, future)

recv = yield future

raise salt.ext.tornado.gen.Return(recv)

def handle_reply(self, msg):
data = salt.payload.loads(msg[0])
future = self._future
self._future = None
future.set_result(data)
def _timeout_message(self, future):
if not future.done():
future.set_exception(SaltReqTimeoutError("Message timed out"))

@salt.ext.tornado.gen.coroutine
def _send_recv(self, message, future):
with (yield self.lock.acquire()):
yield self.socket.send(message)
recv = yield self.socket.recv()
if not future.done():
data = salt.payload.loads(recv)
future.set_result(data)


class ZeroMQSocketMonitor:
Expand Down
Loading