diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d959a8cd..e379ab3b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: name: build (${{ matrix.ruby }} / ${{ matrix.os }}) strategy: matrix: - ruby: [ head, '3.0', '2.7' ] + ruby: [ head, '3.3', '3.2', '3.1', '3.0', '2.7' ] os: [ ubuntu-latest, macos-latest ] experimental: [false] include: diff --git a/lib/net/imap.rb b/lib/net/imap.rb index ea1bd308..6d1ef845 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -106,6 +106,41 @@ module Net # # This script invokes the FETCH command and the SEARCH command concurrently. # + # When running multiple commands, care must be taken to avoid ambiguity. For + # example, SEARCH responses are ambiguous about which command they are + # responding to, so search commands should not run simultaneously, unless the + # server supports +ESEARCH+ {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] or + # IMAP4rev2[https://www.rfc-editor.org/rfc/rfc9051]. See {RFC9051 + # §5.5}[https://www.rfc-editor.org/rfc/rfc9051.html#section-5.5] for + # other examples of command sequences which should not be pipelined. + # + # == Unbounded memory use + # + # Net::IMAP reads server responses in a separate receiver thread per client. + # Unhandled response data is saved to #responses, and response_handlers run + # inside the receiver thread. See the list of methods for {handling server + # responses}[rdoc-ref:Net::IMAP@Handling+server+responses], below. + # + # Because the receiver thread continuously reads and saves new responses, some + # scenarios must be careful to avoid unbounded memory use: + # + # * Commands such as #list or #fetch can have an enormous number of responses. + # * Commands such as #fetch can result in an enormous size per response. + # * Long-lived connections will gradually accumulate unsolicited server + # responses, especially +EXISTS+, +FETCH+, and +EXPUNGE+ responses. + # * A buggy or untrusted server could send inappropriate responses, which + # could be very numerous, very large, and very rapid. + # + # Use paginated or limited versions of commands whenever possible. + # + # Use #max_response_size to impose a limit on incoming server responses + # as they are being read. This is especially important for untrusted + # servers. + # + # Use #add_response_handler to handle responses after each one is received. + # Use the +response_handlers+ argument to ::new to assign response handlers + # before the receiver thread is started. + # # == Errors # # An IMAP server can send three different types of responses to indicate @@ -220,7 +255,9 @@ module Net # Unicode", RFC-2152[https://tools.ietf.org/html/rfc2152], May 1997. # class IMAP < Protocol - VERSION = "0.2.4" + VERSION = "0.2.5" + + autoload :ResponseReader, File.expand_path("imap/response_reader", __dir__) include MonitorMixin if defined?(OpenSSL::SSL) @@ -251,6 +288,40 @@ class IMAP < Protocol # Seconds to wait until an IDLE response is received. attr_reader :idle_response_timeout + # The maximum allowed server response size. When +nil+, there is no limit + # on response size. + # + # The default value is _unlimited_ (after +v0.5.8+, the default is 512 MiB). + # A _much_ lower value should be used with untrusted servers (for example, + # when connecting to a user-provided hostname). When using a lower limit, + # message bodies should be fetched in chunks rather than all at once. + # + # Please Note: this only limits the size per response. It does + # not prevent a flood of individual responses and it does not limit how + # many unhandled responses may be stored on the responses hash. See + # Net::IMAP@Unbounded+memory+use. + # + # Socket reads are limited to the maximum remaining bytes for the current + # response: max_response_size minus the bytes that have already been read. + # When the limit is reached, or reading a +literal+ _would_ go over the + # limit, ResponseTooLargeError is raised and the connection is closed. + # See also #socket_read_limit. + # + # Note that changes will not take effect immediately, because the receiver + # thread may already be waiting for the next response using the previous + # value. Net::IMAP#noop can force a response and enforce the new setting + # immediately. + # + # ==== Versioned Defaults + # + # Net::IMAP#max_response_size was added in +v0.2.5+ and +v0.3.9+ as an + # attr_accessor, and in +v0.4.20+ and +v0.5.7+ as a delegator to a config + # attribute. + # + # * original: +nil+ (no limit) + # * +0.5+: 512 MiB + attr_accessor :max_response_size + # The thread to receive exceptions. attr_accessor :client_thread @@ -955,6 +1026,11 @@ def uid_sort(sort_keys, search_keys, charset) # end # } # + # Response handlers can also be added when the client is created before the + # receiver thread is started, by the +response_handlers+ argument to ::new. + # This ensures every server response is handled, including the #greeting. + # + # Related: #remove_response_handler, #response_handlers def add_response_handler(handler = nil, &block) raise ArgumentError, "two Procs are passed" if handler && block @response_handlers.push(block || handler) @@ -1070,6 +1146,13 @@ def idle_done # OpenSSL::SSL::SSLContext#set_params as parameters. # open_timeout:: Seconds to wait until a connection is opened # idle_response_timeout:: Seconds to wait until an IDLE response is received + # response_handlers:: A list of response handlers to be added before the + # receiver thread is started. This ensures every server + # response is handled, including the #greeting. Note + # that the greeting is handled in the current thread, + # but all other responses are handled in the receiver + # thread. + # max_response_size:: See #max_response_size. # # The most common errors are: # @@ -1100,8 +1183,10 @@ def initialize(host, port_or_options = {}, @tagno = 0 @open_timeout = options[:open_timeout] || 30 @idle_response_timeout = options[:idle_response_timeout] || 5 + @max_response_size = options[:max_response_size] @parser = ResponseParser.new @sock = tcp_socket(@host, @port) + @reader = ResponseReader.new(self, @sock) begin if options[:ssl] start_tls_session(options[:ssl]) @@ -1112,6 +1197,7 @@ def initialize(host, port_or_options = {}, @responses = Hash.new([].freeze) @tagged_responses = {} @response_handlers = [] + options[:response_handlers]&.each do |h| add_response_handler(h) end @tagged_response_arrival = new_cond @continued_command_tag = nil @continuation_request_arrival = new_cond @@ -1128,6 +1214,7 @@ def initialize(host, port_or_options = {}, if @greeting.name == "BYE" raise ByeResponseError, @greeting end + @response_handlers.each do |handler| handler.call(@greeting) end @client_thread = Thread.current @receiver_thread = Thread.start { @@ -1251,25 +1338,14 @@ def get_tagged_response(tag, cmd, timeout = nil) end def get_response - buff = String.new - while true - s = @sock.gets(CRLF) - break unless s - buff.concat(s) - if /\{(\d+)\}\r\n/n =~ s - s = @sock.read($1.to_i) - buff.concat(s) - else - break - end - end + buff = @reader.read_response_buffer return nil if buff.length == 0 - if @@debug - $stderr.print(buff.gsub(/^/n, "S: ")) - end - return @parser.parse(buff) + $stderr.print(buff.gsub(/^/n, "S: ")) if @@debug + @parser.parse(buff) end + ############################# + def record_response(name, data) unless @responses.has_key?(name) @responses[name] = [] @@ -1447,6 +1523,7 @@ def start_tls_session(params = {}) context.verify_callback = VerifyCallbackProc end @sock = SSLSocket.new(@sock, context) + @reader = ResponseReader.new(self, @sock) @sock.sync_close = true @sock.hostname = @host if @sock.respond_to? :hostname= ssl_socket_connect(@sock, @open_timeout) diff --git a/lib/net/imap/errors.rb b/lib/net/imap/errors.rb index 33dd541c..d53bc7a1 100644 --- a/lib/net/imap/errors.rb +++ b/lib/net/imap/errors.rb @@ -11,6 +11,40 @@ class Error < StandardError class DataFormatError < Error end + # Error raised when the socket cannot be read, due to a configured limit. + class ResponseReadError < Error + end + + # Error raised when a response is larger than IMAP#max_response_size. + class ResponseTooLargeError < ResponseReadError + attr_reader :bytes_read, :literal_size + attr_reader :max_response_size + + def initialize(msg = nil, *args, + bytes_read: nil, + literal_size: nil, + max_response_size: nil, + **kwargs) + @bytes_read = bytes_read + @literal_size = literal_size + @max_response_size = max_response_size + msg ||= [ + "Response size", response_size_msg, "exceeds max_response_size", + max_response_size && "(#{max_response_size}B)", + ].compact.join(" ") + return super(msg, *args) if kwargs.empty? # ruby 2.6 compatibility + super(msg, *args, **kwargs) + end + + private + + def response_size_msg + if bytes_read && literal_size + "(#{bytes_read}B read + #{literal_size}B literal)" + end + end + end + # Error raised when a response from the server is non-parseable. class ResponseParseError < Error end diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb new file mode 100644 index 00000000..fd7561fa --- /dev/null +++ b/lib/net/imap/response_reader.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Net + class IMAP + # See https://www.rfc-editor.org/rfc/rfc9051#section-2.2.2 + class ResponseReader # :nodoc: + attr_reader :client + + def initialize(client, sock) + @client, @sock = client, sock + end + + def read_response_buffer + @buff = String.new + catch :eof do + while true + read_line + break unless (@literal_size = get_literal_size) + read_literal + end + end + buff + ensure + @buff = nil + end + + private + + attr_reader :buff, :literal_size + + def bytes_read; buff.bytesize end + def empty?; buff.empty? end + def done?; line_done? && !get_literal_size end + def line_done?; buff.end_with?(CRLF) end + def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end + + def read_line + buff << (@sock.gets(CRLF, read_limit) or throw :eof) + max_response_remaining! unless line_done? + end + + def read_literal + # check before allocating memory for literal + max_response_remaining! + literal = String.new(capacity: literal_size) + buff << (@sock.read(read_limit(literal_size), literal) or throw :eof) + ensure + @literal_size = nil + end + + def read_limit(limit = nil) + [limit, max_response_remaining!].compact.min + end + + def max_response_size; client.max_response_size end + def max_response_remaining; max_response_size &.- bytes_read end + def response_too_large?; max_response_size &.< min_response_size end + def min_response_size; bytes_read + min_response_remaining end + + def min_response_remaining + empty? ? 3 : done? ? 0 : (literal_size || 0) + 2 + end + + def max_response_remaining! + return max_response_remaining unless response_too_large? + raise ResponseTooLargeError.new( + max_response_size: max_response_size, + bytes_read: bytes_read, + literal_size: literal_size, + ) + end + + end + end +end diff --git a/test/net/imap/test_errors.rb b/test/net/imap/test_errors.rb new file mode 100644 index 00000000..a6a7cb0f --- /dev/null +++ b/test/net/imap/test_errors.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class IMAPErrorsTest < Test::Unit::TestCase + + test "ResponseTooLargeError" do + err = Net::IMAP::ResponseTooLargeError.new + assert_nil err.bytes_read + assert_nil err.literal_size + assert_nil err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new("manually set message") + assert_equal "manually set message", err.message + assert_nil err.bytes_read + assert_nil err.literal_size + assert_nil err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new(max_response_size: 1024) + assert_equal "Response size exceeds max_response_size (1024B)", err.message + assert_nil err.bytes_read + assert_nil err.literal_size + assert_equal 1024, err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new(bytes_read: 1200, + max_response_size: 1200) + assert_equal 1200, err.bytes_read + assert_equal "Response size exceeds max_response_size (1200B)", err.message + + err = Net::IMAP::ResponseTooLargeError.new(bytes_read: 800, + literal_size: 1000, + max_response_size: 1200) + assert_equal 800, err.bytes_read + assert_equal 1000, err.literal_size + assert_equal("Response size (800B read + 1000B literal) " \ + "exceeds max_response_size (1200B)", err.message) + end + +end diff --git a/test/net/imap/test_imap_max_response_size.rb b/test/net/imap/test_imap_max_response_size.rb new file mode 100644 index 00000000..7ec554c3 --- /dev/null +++ b/test/net/imap/test_imap_max_response_size.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class IMAPMaxResponseSizeTest < Test::Unit::TestCase + + def setup + @do_not_reverse_lookup = Socket.do_not_reverse_lookup + Socket.do_not_reverse_lookup = true + @threads = [] + end + + def teardown + if !@threads.empty? + assert_join_threads(@threads) + end + ensure + Socket.do_not_reverse_lookup = @do_not_reverse_lookup + end + + test "#max_response_size reading literals" do + _, port = with_server_socket do |sock| + sock.gets # => NOOP + sock.print("RUBY0001 OK done\r\n") + sock.gets # => NOOP + sock.print("* 1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")\r\n") + sock.print("RUBY0002 OK done\r\n") + "RUBY0003" + end + Timeout.timeout(5) do + imap = Net::IMAP.new("localhost", port: port, max_response_size: 640 << 20) + assert_equal 640 << 20, imap.max_response_size + imap.max_response_size = 12_345 + 30 + assert_equal 12_345 + 30, imap.max_response_size + imap.noop # to reset the get_response limit + imap.noop # to send the FETCH + assert_equal "a" * 12_345, imap.responses["FETCH"].first.attr["BODY[]"] + ensure + imap.logout rescue nil + imap.disconnect rescue nil + end + end + + test "#max_response_size closes connection for too long line" do + _, port = with_server_socket do |sock| + sock.gets or next # => never called + fail "client disconnects first" + end + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/ + ) do + Net::IMAP.new("localhost", port: port, max_response_size: 10) + fail "should not get here (greeting longer than max_response_size)" + end + end + + test "#max_response_size closes connection for too long literal" do + _, port = with_server_socket(ignore_io_error: true) do |sock| + sock.gets # => NOOP + sock.print "* 1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")\r\n" + sock.print("RUBY0001 OK done\r\n") + end + client = Net::IMAP.new("localhost", port: port, max_response_size: 1000) + assert_equal 1000, client.max_response_size + client.max_response_size = 50 + assert_equal 50, client.max_response_size + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, + /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/ + ) do + client.noop + fail "should not get here (FETCH literal longer than max_response_size)" + end + end + + def with_server_socket(ignore_io_error: false) + server = create_tcp_server + port = server.addr[1] + start_server do + Timeout.timeout(5) do + sock = server.accept + sock.print("* OK connection established\r\n") + logout_tag = yield sock if block_given? + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("#{logout_tag} OK LOGOUT completed\r\n") if logout_tag + rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNRESET, + Errno::EPIPE, Errno::ETIMEDOUT + ignore_io_error or raise + ensure + sock.close rescue nil + server.close rescue nil + end + end + return server, port + end + + def start_server + th = Thread.new do + yield + end + @threads << th + sleep 0.1 until th.stop? + end + + def create_tcp_server + return TCPServer.new(server_addr, 0) + end + + def server_addr + Addrinfo.tcp("localhost", 0).ip_address + end +end diff --git a/test/net/imap/test_imap_response_handlers.rb b/test/net/imap/test_imap_response_handlers.rb new file mode 100644 index 00000000..3786f242 --- /dev/null +++ b/test/net/imap/test_imap_response_handlers.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class IMAPResponseHandlersTest < Test::Unit::TestCase + + def setup + @do_not_reverse_lookup = Socket.do_not_reverse_lookup + Socket.do_not_reverse_lookup = true + @threads = [] + end + + def teardown + if !@threads.empty? + assert_join_threads(@threads) + end + ensure + Socket.do_not_reverse_lookup = @do_not_reverse_lookup + end + + test "#add_response_handlers" do + server = create_tcp_server + port = server.addr[1] + start_server do + sock = server.accept + Timeout.timeout(5) do + sock.print("* OK connection established\r\n") + sock.gets # => NOOP + sock.print("* 1 EXPUNGE\r\n") + sock.print("* 2 EXPUNGE\r\n") + sock.print("* 3 EXPUNGE\r\n") + sock.print("RUBY0001 OK NOOP completed\r\n") + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("RUBY0002 OK LOGOUT completed\r\n") + ensure + sock.close + server.close + end + end + begin + responses = [] + imap = Net::IMAP.new(server_addr, port: port) + assert_equal 0, imap.response_handlers.length + imap.add_response_handler do |r| responses << [:block, r] end + assert_equal 1, imap.response_handlers.length + imap.add_response_handler(->(r) { responses << [:proc, r] }) + assert_equal 2, imap.response_handlers.length + + imap.noop + responses = responses[0, 6].map {|which, resp| + [which, resp.class, resp.name, resp.data] + } + assert_equal [ + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 1], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 1], + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 2], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 2], + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], + ], responses + ensure + imap&.logout + imap&.disconnect + end + end + + test "::new with response_handlers kwarg" do + greeting = nil + expunges = [] + alerts = [] + untagged = 0 + handler0 = ->(r) { greeting ||= r } + handler1 = ->(r) { alerts << r.data.text if r.data.code.name == "ALERT" rescue nil } + handler2 = ->(r) { expunges << r.data if r.name == "EXPUNGE" } + handler3 = ->(r) { untagged += 1 if r.is_a?(Net::IMAP::UntaggedResponse) } + response_handlers = [handler0, handler1, handler2, handler3] + + server = create_tcp_server + port = server.addr[1] + start_server do + sock = server.accept + Timeout.timeout(5) do + sock.print("* OK connection established\r\n") + sock.gets # => NOOP + sock.print("* 1 EXPUNGE\r\n") + sock.print("* 1 EXPUNGE\r\n") + sock.print("* OK [ALERT] The first alert.\r\n") + sock.print("RUBY0001 OK [ALERT] Did you see the alert?\r\n") + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("RUBY0002 OK LOGOUT completed\r\n") + ensure + sock.close + server.close + end + end + begin + imap = Net::IMAP.new("localhost", port: port, + response_handlers: response_handlers) + assert_equal response_handlers, imap.response_handlers + refute_same response_handlers, imap.response_handlers + + # handler0 recieved the greeting and handler3 counted it + assert_equal imap.greeting, greeting + assert_equal 1, untagged + + imap.noop + assert_equal 4, untagged + assert_equal [1, 1], expunges # from handler2 + assert_equal ["The first alert.", "Did you see the alert?"], alerts + ensure + imap&.logout + imap&.disconnect + end + end + + def start_server + th = Thread.new do + yield + end + @threads << th + sleep 0.1 until th.stop? + end + + def create_tcp_server + return TCPServer.new(server_addr, 0) + end + + def server_addr + Addrinfo.tcp("localhost", 0).ip_address + end +end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb new file mode 100644 index 00000000..d2c1c11a --- /dev/null +++ b/test/net/imap/test_response_reader.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "net/imap" +require "stringio" +require "test/unit" + +class ResponseReaderTest < Test::Unit::TestCase + class FakeClient + attr_accessor :max_response_size + end + + def literal(str) "{#{str.bytesize}}\r\n#{str}" end + + test "#read_response_buffer" do + client = FakeClient.new + aaaaaaaaa = "a" * (20 << 10) + many_crs = "\r" * 1000 + many_crlfs = "\r\n" * 500 + simple = "* OK greeting\r\n" + long_line = "tag ok #{aaaaaaaaa} #{aaaaaaaaa}\r\n" + literal_aaaa = "* fake #{literal aaaaaaaaa}\r\n" + literal_crlf = "tag ok #{literal many_crlfs} #{literal many_crlfs}\r\n" + zero_literal = "tag ok #{literal ""} #{literal ""}\r\n" + illegal_crs = "tag ok #{many_crs} #{many_crs}\r\n" + illegal_lfs = "tag ok #{literal "\r"}\n#{literal "\r"}\n\r\n" + io = StringIO.new([ + simple, + long_line, + literal_aaaa, + literal_crlf, + zero_literal, + illegal_crs, + illegal_lfs, + simple, + ].join) + rcvr = Net::IMAP::ResponseReader.new(client, io) + assert_equal simple, rcvr.read_response_buffer.to_str + assert_equal long_line, rcvr.read_response_buffer.to_str + assert_equal literal_aaaa, rcvr.read_response_buffer.to_str + assert_equal literal_crlf, rcvr.read_response_buffer.to_str + assert_equal zero_literal, rcvr.read_response_buffer.to_str + assert_equal illegal_crs, rcvr.read_response_buffer.to_str + assert_equal illegal_lfs, rcvr.read_response_buffer.to_str + assert_equal simple, rcvr.read_response_buffer.to_str + assert_equal "", rcvr.read_response_buffer.to_str + end + + test "#read_response_buffer with max_response_size" do + client = FakeClient.new + client.max_response_size = 10 + under = "+ 3456\r\n" + exact = "+ 345678\r\n" + over = "+ 3456789\r\n" + io = StringIO.new([under, exact, over].join) + rcvr = Net::IMAP::ResponseReader.new(client, io) + assert_equal under, rcvr.read_response_buffer.to_str + assert_equal exact, rcvr.read_response_buffer.to_str + assert_raise Net::IMAP::ResponseTooLargeError do + rcvr.read_response_buffer + end + end + +end