From ed5295b215eddc79ccf31649a820911c2a15a88e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 25 Jun 2019 11:56:33 -0500 Subject: [PATCH 1/6] Shared cores for messages --- onvm/onvm_nflib/onvm_common.h | 2 +- onvm/onvm_nflib/onvm_nflib.c | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) mode change 100644 => 100755 onvm/onvm_nflib/onvm_common.h mode change 100644 => 100755 onvm/onvm_nflib/onvm_nflib.c diff --git a/onvm/onvm_nflib/onvm_common.h b/onvm/onvm_nflib/onvm_common.h old mode 100644 new mode 100755 index c2a947def..1037a17cc --- a/onvm/onvm_nflib/onvm_common.h +++ b/onvm/onvm_nflib/onvm_common.h @@ -469,7 +469,7 @@ get_sem_name(unsigned id) { static inline int whether_wakeup_client(struct onvm_nf *nf, struct nf_wakeup_info *nf_wakeup_info) { - if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD) + if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD && rte_ring_count(nf->msg_q) == 0) return 0; /* Check if its already woken up */ diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c old mode 100644 new mode 100755 index 6ee08db17..b978903f7 --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -947,8 +947,13 @@ onvm_nflib_dequeue_messages(struct onvm_nf_local_ctx *nf_local_ctx) { // Check and see if this NF has any messages from the manager if (likely(rte_ring_count(msg_q) == 0)) { + if (ONVM_NF_SHARE_CORES) { + rte_atomic16_set(nf->shared_core.sleep_state, 1); + sem_wait(nf->shared_core.nf_mutex); + } return; } + msg = NULL; rte_ring_dequeue(msg_q, (void **)(&msg)); onvm_nflib_handle_msg(msg, nf_local_ctx); From e700b5794e422655126e407e8f7216e8f84a1fd2 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 26 Jun 2019 22:03:14 -0500 Subject: [PATCH 2/6] Rm line --- onvm/onvm_nflib/onvm_nflib.c | 1 - 1 file changed, 1 deletion(-) diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c index b978903f7..bf0482312 100755 --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -953,7 +953,6 @@ onvm_nflib_dequeue_messages(struct onvm_nf_local_ctx *nf_local_ctx) { } return; } - msg = NULL; rte_ring_dequeue(msg_q, (void **)(&msg)); onvm_nflib_handle_msg(msg, nf_local_ctx); From e34497a6a95a407493a0f8d7760740f82c0a5318 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 26 Jun 2019 22:12:46 -0500 Subject: [PATCH 3/6] nf context, not nf --- onvm/onvm_nflib/onvm_nflib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c index bf0482312..4881b3f51 100755 --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -948,8 +948,8 @@ onvm_nflib_dequeue_messages(struct onvm_nf_local_ctx *nf_local_ctx) { // Check and see if this NF has any messages from the manager if (likely(rte_ring_count(msg_q) == 0)) { if (ONVM_NF_SHARE_CORES) { - rte_atomic16_set(nf->shared_core.sleep_state, 1); - sem_wait(nf->shared_core.nf_mutex); + rte_atomic16_set(nf_local_ctx->nf->shared_core.sleep_state, 1); + sem_wait(nf_local_ctx->nf->shared_core.nf_mutex); } return; } From be13c4a85f9d7a022cf162b0810135d2ca35ce92 Mon Sep 17 00:00:00 2001 From: dennisa Date: Mon, 8 Jul 2019 10:57:17 -0500 Subject: [PATCH 4/6] Moved functionality to main thread --- onvm/onvm_nflib/onvm_nflib.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/onvm/onvm_nflib/onvm_nflib.c b/onvm/onvm_nflib/onvm_nflib.c index 4881b3f51..2e3ecee9e 100755 --- a/onvm/onvm_nflib/onvm_nflib.c +++ b/onvm/onvm_nflib/onvm_nflib.c @@ -555,6 +555,14 @@ onvm_nflib_thread_main_loop(void *arg) { start_time = rte_get_tsc_cycles(); for (;rte_atomic16_read(&nf_local_ctx->keep_running) && rte_atomic16_read(&main_nf_local_ctx->keep_running);) { + /* Possibly sleep if in shared core mode, otherwise continue */ + if (ONVM_NF_SHARE_CORES) { + if (unlikely(rte_ring_count(nf->rx_q) == 0) && likely(rte_ring_count(nf->msg_q) == 0)) { + rte_atomic16_set(nf->shared_core.sleep_state, 1); + sem_wait(nf->shared_core.nf_mutex); + } + } + nb_pkts_added = onvm_nflib_dequeue_packets((void **)pkts, nf_local_ctx, nf->function_table->pkt_handler); @@ -908,12 +916,7 @@ onvm_nflib_dequeue_packets(void **pkts, struct onvm_nf_local_ctx *nf_local_ctx, /* Dequeue all packets in ring up to max possible. */ nb_pkts = rte_ring_dequeue_burst(nf->rx_q, pkts, PACKET_READ_SIZE, NULL); - /* Possibly sleep if in shared core mode, otherwise return */ if (unlikely(nb_pkts == 0)) { - if (ONVM_NF_SHARE_CORES) { - rte_atomic16_set(nf->shared_core.sleep_state, 1); - sem_wait(nf->shared_core.nf_mutex); - } return 0; } @@ -947,10 +950,6 @@ onvm_nflib_dequeue_messages(struct onvm_nf_local_ctx *nf_local_ctx) { // Check and see if this NF has any messages from the manager if (likely(rte_ring_count(msg_q) == 0)) { - if (ONVM_NF_SHARE_CORES) { - rte_atomic16_set(nf_local_ctx->nf->shared_core.sleep_state, 1); - sem_wait(nf_local_ctx->nf->shared_core.nf_mutex); - } return; } msg = NULL; From 9d435db1173abf03ff9fa966390c5a8be27fcc8e Mon Sep 17 00:00:00 2001 From: dennisa Date: Fri, 19 Jul 2019 10:44:27 -0500 Subject: [PATCH 5/6] Adding macro for msg threshold --- onvm/onvm_nflib/onvm_common.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onvm/onvm_nflib/onvm_common.h b/onvm/onvm_nflib/onvm_common.h index 1037a17cc..4368790d4 100755 --- a/onvm/onvm_nflib/onvm_common.h +++ b/onvm/onvm_nflib/onvm_common.h @@ -75,6 +75,7 @@ #define ONVM_NF_ACTION_OUT 3 // send the packet out the NIC port set in the argument field #define PKT_WAKEUP_THRESHOLD 1 // for shared core mode, how many packets are required to wake up the NF +#define MSG_WAKEUP_THRESHOLD 1 // for shared core mode, how many messages on an NF's ring are required to wake up the NF /* Used in setting bit flags for core options */ #define MANUAL_CORE_ASSIGNMENT_BIT 0 @@ -469,7 +470,7 @@ get_sem_name(unsigned id) { static inline int whether_wakeup_client(struct onvm_nf *nf, struct nf_wakeup_info *nf_wakeup_info) { - if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD && rte_ring_count(nf->msg_q) == 0) + if (rte_ring_count(nf->rx_q) < PKT_WAKEUP_THRESHOLD && rte_ring_count(nf->msg_q) < MSG_WAKEUP_THRESHOLD) return 0; /* Check if its already woken up */ From cf8aa2ab7571bac36858e49283d634feb22af4a6 Mon Sep 17 00:00:00 2001 From: Dennis Afanasev <32916582+dennisafa@users.noreply.github.com> Date: Thu, 25 Jul 2019 09:13:29 -0400 Subject: [PATCH 6/6] Update NF_Dev.md --- docs/NF_Dev.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/NF_Dev.md b/docs/NF_Dev.md index 389080997..282711983 100644 --- a/docs/NF_Dev.md +++ b/docs/NF_Dev.md @@ -43,7 +43,7 @@ NFs can scale by running multiple threads. For launching more threads the main N Example use of Multithreading NF scaling functionality can be seen in the scaling_example NF. ### Shared core mode -This is an **EXPERIMENTAL** mode for OpenNetVM. It allows multiple NFs to run on a shared core. In "normal" OpenNetVM, each NF will poll its RX queue for packets, monopolizing the CPU even if it has a low load. This branch adds a semaphore-based communication system so that NFs will block when there are no packets available. The NF Manger will then signal the semaphore once one or more packets arrive. +This is an **EXPERIMENTAL** mode for OpenNetVM. It allows multiple NFs to run on a shared core. In "normal" OpenNetVM, each NF will poll its RX queue and message queue for packets and messages respectively, monopolizing the CPU even if it has a low load. This branch adds a semaphore-based communication system so that NFs will block when there are no packets and messages available. The NF Manger will then signal the semaphore once one or more packets or messages arrive. This code allows you to evaluate resource management techniques for NFs that share cores, however it has not been fully tested with complex NFs, therefore if you encounter any bugs please create an issue or a pull request with a proposed fix. @@ -52,7 +52,7 @@ The code is based on the hybrid-polling model proposed in [_Flurries: Countless Usage / Known Limitations: - To enable pass a `-c` flag to the onvm_mgr, and use a `-s` flag when starting a NF to specify that they want to share cores - All code for sharing CPUs is within `if (ONVM_NF_SHARE_CORES)` blocks - - When enabled, you can run multiple NFs on the same CPU core with much less interference than if they are polling for packets + - When enabled, you can run multiple NFs on the same CPU core with much less interference than if they are polling for packets and messages - This code does not provide any particular intelligence for how NFs are scheduled or when they wakeup/sleep - Note that the manager threads all still use polling