Skip to content

Fix cpu utilization problem in session cache #7719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
duke8253 merged 1 commit into from
May 11, 2021
Merged

Fix cpu utilization problem in session cache #7719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 50 additions & 37 deletions iocore/net/SSLSessionCache.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,24 @@ SSLSessionCache::insertSession(const SSLSessionID &sid, SSL_SESSION *sess, SSL *
void
SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess, SSL *ssl)
{
std::shared_lock r_lock(mutex, std::try_to_lock);
if (!r_lock.owns_lock()) {
if (ssl_rsb) {
SSL_INCREMENT_DYN_STAT(ssl_session_cache_lock_contention);
}
if (SSLConfigParams::session_cache_skip_on_lock_contention) {
return;
}
r_lock.lock();
}

// Don't insert if it is already there
if (bucket_map.find(id) != bucket_map.end()) {
return;
}

r_lock.unlock();

size_t len = i2d_SSL_SESSION(sess, nullptr); // make sure we're not going to need more than SSL_MAX_SESSION_SIZE bytes
/* do not cache a session that's too big. */
if (len > static_cast<size_t>(SSL_MAX_SESSION_SIZE)) {
Expand All @@ -140,39 +158,33 @@ SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess, SSL *
ink_release_assert(static_cast<size_t>(buf_exdata->block_size()) >= len_exdata);
ssl_session_cache_exdata *exdata = reinterpret_cast<ssl_session_cache_exdata *>(buf_exdata->data());
// This could be moved to a function in charge of populating exdata
exdata->curve = (ssl == nullptr) ? 0 : SSLGetCurveNID(ssl);
ink_hrtime now = Thread::get_hrtime_updated();
exdata->curve = (ssl == nullptr) ? 0 : SSLGetCurveNID(ssl);

ats_scoped_obj<SSLSession> ssl_session(new SSLSession(id, buf, len, buf_exdata, now));
ats_scoped_obj<SSLSession> ssl_session(new SSLSession(id, buf, len, buf_exdata));

std::unique_lock lock(mutex, std::try_to_lock);
if (!lock.owns_lock()) {
std::unique_lock w_lock(mutex, std::try_to_lock);
if (!w_lock.owns_lock()) {
if (ssl_rsb) {
SSL_INCREMENT_DYN_STAT(ssl_session_cache_lock_contention);
}
if (SSLConfigParams::session_cache_skip_on_lock_contention) {
return;
}
lock.lock();
w_lock.lock();
}

PRINT_BUCKET("insertSession before")
if (bucket_data.size() >= SSLConfigParams::session_cache_max_bucket_size) {
if (bucket_map.size() >= SSLConfigParams::session_cache_max_bucket_size) {
if (ssl_rsb) {
SSL_INCREMENT_DYN_STAT(ssl_session_cache_eviction);
}
removeOldestSession(lock);
}

// Don't insert if it is already there
if (bucket_data.find(id) != bucket_data.end()) {
return;
removeOldestSession(w_lock);
}

/* do the actual insert */
auto node = ssl_session.release();
bucket_data[id] = node;
bucket_data_ts[now] = node;
auto node = ssl_session.release();
bucket_que.enqueue(node);
bucket_map[id] = node;

PRINT_BUCKET("insertSession after")
}
Expand All @@ -192,10 +204,10 @@ SSLSessionBucket::getSessionBuffer(const SSLSessionID &id, char *buffer, int &le
lock.lock();
}

auto node = bucket_data.find(id);
if (buffer && node != bucket_data.end()) {
true_len = node->second->len_asn1_data;
const unsigned char *loc = reinterpret_cast<const unsigned char *>(node->second->asn1_data->data());
auto entry = bucket_map.find(id);
if (buffer && entry != bucket_map.end()) {
true_len = entry->second->len_asn1_data;
const unsigned char *loc = reinterpret_cast<const unsigned char *>(entry->second->asn1_data->data());
if (true_len < len) {
len = true_len;
}
Expand Down Expand Up @@ -229,15 +241,15 @@ SSLSessionBucket::getSession(const SSLSessionID &id, SSL_SESSION **sess, ssl_ses

PRINT_BUCKET("getSession")

auto node = bucket_data.find(id);
if (node == bucket_data.end()) {
auto entry = bucket_map.find(id);
if (entry == bucket_map.end()) {
Debug("ssl.session_cache", "Session with id '%s' not found in bucket %p.", buf, this);
return false;
}
const unsigned char *loc = reinterpret_cast<const unsigned char *>(node->second->asn1_data->data());
*sess = d2i_SSL_SESSION(nullptr, &loc, node->second->len_asn1_data);
const unsigned char *loc = reinterpret_cast<const unsigned char *>(entry->second->asn1_data->data());
*sess = d2i_SSL_SESSION(nullptr, &loc, entry->second->len_asn1_data);
if (data != nullptr) {
ssl_session_cache_exdata *exdata = reinterpret_cast<ssl_session_cache_exdata *>(node->second->extra_data->data());
ssl_session_cache_exdata *exdata = reinterpret_cast<ssl_session_cache_exdata *>(entry->second->extra_data->data());
*data = exdata;
}
return true;
Expand All @@ -251,10 +263,10 @@ void inline SSLSessionBucket::print(const char *ref_str) const
}

fprintf(stderr, "-------------- BUCKET %p (%s) ----------------\n", this, ref_str);
fprintf(stderr, "Current Size: %ld, Max Size: %zd\n", bucket_data.size(), SSLConfigParams::session_cache_max_bucket_size);
fprintf(stderr, "Current Size: %ld, Max Size: %zd\n", bucket_map.size(), SSLConfigParams::session_cache_max_bucket_size);
fprintf(stderr, "Bucket: \n");

for (auto &x : bucket_data) {
for (auto &x : bucket_map) {
char s_buf[2 * x.second->session_id.len + 1];
x.second->session_id.toString(s_buf, sizeof(s_buf));
fprintf(stderr, " %s\n", s_buf);
Expand All @@ -268,10 +280,11 @@ void inline SSLSessionBucket::removeOldestSession(const std::unique_lock<std::sh

PRINT_BUCKET("removeOldestSession before")

auto node = bucket_data_ts.begin();
bucket_data.erase(node->second->session_id);
delete node->second;
bucket_data_ts.erase(node);
while (bucket_que.head && bucket_que.size >= static_cast<int>(SSLConfigParams::session_cache_max_bucket_size)) {
auto node = bucket_que.pop();
bucket_map.erase(node->session_id);
delete node;
}

PRINT_BUCKET("removeOldestSession after")
}
Expand All @@ -282,14 +295,14 @@ SSLSessionBucket::removeSession(const SSLSessionID &id)
// We can't bail on contention here because this session MUST be removed.
std::unique_lock lock(mutex);

auto node = bucket_data.find(id);

PRINT_BUCKET("removeSession before")

if (node != bucket_data.end()) {
bucket_data_ts.erase(node->second->time_stamp);
delete node->second;
bucket_data.erase(node);
auto entry = bucket_map.find(id);
if (entry != bucket_map.end()) {
auto node = entry->second;
bucket_que.remove(node);
bucket_map.erase(entry);
delete node;
}

PRINT_BUCKET("removeSession after")
Expand Down
46 changes: 15 additions & 31 deletions iocore/net/SSLSessionCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,33 +113,19 @@ struct SSLSessionID : public TSSslSessionID {
uint64_t
hash() const
{
if (hash_value == 0) {
// because the session ids should be uniformly random, we can treat the bits as a hash value
// however we need to combine them if the length is longer than 64bits
if (len >= sizeof(uint64_t)) {
uint64_t seed = 0;
for (uint64_t i = 0; i < len; i += sizeof(uint64_t)) {
hash_combine(seed, static_cast<uint64_t>(bytes[i]));
}
hash_value = seed;
} else if (len) {
hash_value = static_cast<uint64_t>(bytes[0]);
} else {
hash_value = 0;
// because the session ids should be uniformly random, we can treat the bits as a hash value
// however we need to combine them if the length is longer than 64bits
if (len >= sizeof(uint64_t)) {
uint64_t seed = 0;
for (uint64_t i = 0; i < len; i += sizeof(uint64_t)) {
hash_combine(seed, static_cast<uint64_t>(bytes[i]));
}
return seed;
} else if (len) {
return static_cast<uint64_t>(bytes[0]);
} else {
return 0;
}
return hash_value;
}

private:
mutable uint64_t hash_value = 0;
};

struct SSLSessionIDHash {
uint64_t
operator()(const SSLSessionID &id) const
{
return id.hash();
}
};

Expand All @@ -150,11 +136,9 @@ class SSLSession
Ptr<IOBufferData> asn1_data; /* this is the ASN1 representation of the SSL_CTX */
size_t len_asn1_data;
Ptr<IOBufferData> extra_data;
ink_hrtime time_stamp;

SSLSession(const SSLSessionID &id, const Ptr<IOBufferData> &ssl_asn1_data, size_t len_asn1, Ptr<IOBufferData> &exdata,
ink_hrtime ts)
: session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1), extra_data(exdata), time_stamp(ts)
SSLSession(const SSLSessionID &id, const Ptr<IOBufferData> &ssl_asn1_data, size_t len_asn1, Ptr<IOBufferData> &exdata)
: session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1), extra_data(exdata)
{
}

Expand All @@ -177,8 +161,8 @@ class SSLSessionBucket
void removeOldestSession(const std::unique_lock<std::shared_mutex> &lock);

mutable std::shared_mutex mutex;
std::unordered_map<SSLSessionID, SSLSession *, SSLSessionIDHash> bucket_data;
std::map<ink_hrtime, SSLSession *> bucket_data_ts;
CountQueue<SSLSession> bucket_que;
std::map<SSLSessionID, SSLSession *> bucket_map;
};

class SSLSessionCache
Expand Down