diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst index 07244b7..0aedb4d 100644 --- a/doc/reference/configuration/records.config.en.rst +++ b/doc/reference/configuration/records.config.en.rst @@ -2168,6 +2168,13 @@ SSL Termination buffering at the SSL layer. The default of ``0`` means to always write all available data into a single SSL record. + A value of ``-1`` means TLS record size is dynamically determined. The + strategy employed is to use small TLS records that fit into a single + TCP segment for the first ~1 MB of data, but, increase the record size to + 16 KB after that to optimize throughput. The record size is reset back to + a single segment after ~1 second of inactivity and the record size ramping + mechanism is repeated again. + .. ts:cv:: CONFIG proxy.config.ssl.session_cache INT 2 Enables the SSL Session Cache: diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 51b7391..e567b64 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -52,6 +52,17 @@ #define SSL_TLSEXT_ERR_NOACK 3 #endif +// TS-2503: dynamic TLS record sizing +// For smaller records, we should also reserve space for various TCP options +// (timestamps, SACKs.. up to 40 bytes [1]), and account for TLS record overhead +// (another 20-60 bytes on average, depending on the negotiated ciphersuite [2]). +// All in all: 1500 - 40 (IP) - 20 (TCP) - 40 (TCP options) - TLS overhead (60-100) +// For larger records, the size is determined by TLS protocol record size +#define SSL_DEF_TLS_RECORD_SIZE 1300 // 1500 - 40 (IP) - 20 (TCP) - 40 (TCP options) - TLS overhead (60-100) +#define SSL_MAX_TLS_RECORD_SIZE 16383 // 2^14 - 1 +#define SSL_DEF_TLS_RECORD_BYTE_THRESHOLD 1000000 +#define SSL_DEF_TLS_RECORD_MSEC_THRESHOLD 1000 + class SSLNextProtocolSet; struct SSLCertLookup; @@ -105,6 +116,8 @@ public: SSL *ssl; ink_hrtime sslHandshakeBeginTime; + ink_hrtime sslLastWriteTime; + ink64_t sslTotalBytesSent; static int advertise_next_protocol(SSL * ssl, const unsigned char ** out, unsigned * outlen, void *); static int select_next_protocol(SSL * ssl, const unsigned char ** out, unsigned char * outlen, const unsigned char * in, unsigned inlen, void *); diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index ec7b0b9..3b77e7d 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -70,6 +70,8 @@ enum SSL_Stats ssl_total_tickets_verified_stat, ssl_total_tickets_not_found_stat, ssl_total_tickets_renewed_stat, + ssl_total_dyn_def_tls_record_count, + ssl_total_dyn_max_tls_record_count, ssl_session_cache_hit, ssl_session_cache_miss, ssl_session_cache_eviction, diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index ba6f435..4a654f1 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -617,12 +617,21 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i ProxyMutex *mutex = this_ethread()->mutex; int64_t r = 0; int64_t l = 0; + uint32_t dynamic_tls_record_size = 0; ssl_error_t err = SSL_ERROR_NONE; // XXX Rather than dealing with the block directly, we should use the IOBufferReader API. int64_t offset = buf.reader()->start_offset; IOBufferBlock *b = buf.reader()->block; + ink_hrtime now = ink_get_hrtime_internal(); + int msec_since_last_write = ink_hrtime_to_msec(now - sslLastWriteTime); + if (msec_since_last_write > SSL_DEF_TLS_RECORD_MSEC_THRESHOLD) { + // reset sslTotalBytesSent after 1 sec inactivity + sslTotalBytesSent = 0; + } + Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, now %" PRId64 ",lastwrite %" PRId64 " ,msec_since_last_write %d", now, sslLastWriteTime, msec_since_last_write); + if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) { return this->super::load_buffer_and_write(towrite, wattempted, total_wrote, buf, needs); } @@ -648,7 +657,22 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i // operations. int64_t orig_l = l; if (SSLConfigParams::ssl_maxrecord > 0 && l > SSLConfigParams::ssl_maxrecord) { - l = SSLConfigParams::ssl_maxrecord; + l = SSLConfigParams::ssl_maxrecord; + // flush at TLS record boundaries always + if (l > SSL_MAX_TLS_RECORD_SIZE) { + l -= (l % SSL_MAX_TLS_RECORD_SIZE); + } + } else if (SSLConfigParams::ssl_maxrecord == -1) { + if (sslTotalBytesSent < SSL_DEF_TLS_RECORD_BYTE_THRESHOLD || msec_since_last_write > SSL_DEF_TLS_RECORD_MSEC_THRESHOLD) { + dynamic_tls_record_size = SSL_DEF_TLS_RECORD_SIZE; + SSL_INCREMENT_DYN_STAT(ssl_total_dyn_def_tls_record_count); + } else { + dynamic_tls_record_size = SSL_MAX_TLS_RECORD_SIZE; + SSL_INCREMENT_DYN_STAT(ssl_total_dyn_max_tls_record_count); + } + if (l > dynamic_tls_record_size) { + l = dynamic_tls_record_size; + } } if (!l) { @@ -660,6 +684,10 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, before SSLWriteBuffer, l=%" PRId64", towrite=%" PRId64", b=%p", l, towrite, b); err = SSLWriteBuffer(ssl, b->start() + offset, l, r); + + sslLastWriteTime = now; + sslTotalBytesSent += total_wrote; + if (r == l) { wattempted = total_wrote; } @@ -744,6 +772,8 @@ SSLNetVConnection::SSLNetVConnection(): npnSet(NULL), npnEndpoint(NULL) { + sslLastWriteTime = 0; + sslTotalBytesSent = 0; } void @@ -936,6 +966,8 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } sslHandShakeComplete = true; + sslLastWriteTime = 0; + sslTotalBytesSent = 0; if (sslHandshakeBeginTime) { const ink_hrtime ssl_handshake_time = ink_get_hrtime() - sslHandshakeBeginTime; @@ -1058,6 +1090,8 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) } sslHandShakeComplete = true; + sslLastWriteTime = 0; + sslTotalBytesSent = 0; return EVENT_DONE; case SSL_ERROR_WANT_WRITE: diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 27e0c19..286ba69 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1794,6 +1794,10 @@ SSLWriteBuffer(SSL * ssl, const void * buf, int64_t nbytes, int64_t& nwritten) int ret = SSL_write(ssl, buf, (int)nbytes); if (ret > 0) { nwritten = ret; + BIO *bio = SSL_get_wbio(ssl); + if (bio != NULL) { + BIO_flush(bio); + } return SSL_ERROR_NONE; }