From 74676e5b592318c01e1d20687691a21b72871231 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 4 Jun 2019 21:33:02 -0400 Subject: [PATCH 01/42] Save performance in ci --- ci/install.sh | 32 ++++++++++++++++++++++++++++++++ ci/manager.sh | 23 ----------------------- 2 files changed, 32 insertions(+), 23 deletions(-) create mode 100755 ci/install.sh diff --git a/ci/install.sh b/ci/install.sh new file mode 100755 index 000000000..a5fcd7f3e --- /dev/null +++ b/ci/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# install all dependencies only once +set -e +. helper-functions.sh +SCRIPT_LOC=$(pwd) + +print_header "Installing dependencies" + +set +e + +sudo apt update +sudo apt install -y build-essential libssl-dev libffi-dev +sudo apt install -y python3 +sudo apt install -y python3-pip + +python3 -V +check_exit_code "ERROR: Python not installed" + +pip3 -V +check_exit_code "ERROR: Pip not installed" + +sudo -H pip3 install virtualenv --upgrade +check_exit_code "ERROR: virtualenv had trouble upgrading" + +sudo -H pip3 install --pre github3.py +check_exit_code "ERROR: GitHub3.py not installed or failed to install" + +sudo -H pip3 install paramiko +check_exit_code "ERROR: Paramiko not installed or failed to install" + +sudo -H pip3 install pexpect +check_exit_code "ERROR: Pexpect not installed or failed to install" diff --git a/ci/manager.sh b/ci/manager.sh index a4b9bbd96..d8a830d92 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -47,7 +47,6 @@ fi . $1 # source the variables from config file -print_header "ONVM Performance Testing" print_header "Checking Required Variables" if [[ -z "$WORKER_LIST" ]] @@ -100,28 +99,6 @@ done # turn off error checking set +e -print_header "Ensuring Dependencies are Installed" - -sudo apt install -y python3 -sudo apt install -y python3-pip - -python3 -V -check_exit_code "ERROR: Python not installed" - -pip3 -V -check_exit_code "ERROR: Pip not installed" - -sudo -H pip3 install virtualenv --upgrade - -sudo -H pip3 install paramiko -check_exit_code "ERROR: Paramiko not installed or failed to install" - -sudo -H pip3 install --pre github3.py -check_exit_code "ERROR: GitHub3.py not installed or failed to install" - -sudo -H pip3 install pexpect -check_exit_code "ERROR: Pexpect not installed or failed to install" - print_header "Fetching and Checking Out Pull Request" python3 clone-and-checkout-pr.py $GITHUB_CREDS "{\"id\": $PR_ID}" $REPO_OWNER $REPO_NAME check_exit_code "ERROR: Failed to fetch and checkout pull request" From 3a97ae46a889b82e5117dbfff3003b58990ee193 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 5 Jun 2019 18:48:25 -0400 Subject: [PATCH 02/42] Add flask to install --- ci/{install.sh => run_ci.sh} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename ci/{install.sh => run_ci.sh} (77%) diff --git a/ci/install.sh b/ci/run_ci.sh similarity index 77% rename from ci/install.sh rename to ci/run_ci.sh index a5fcd7f3e..9f4a370dd 100755 --- a/ci/install.sh +++ b/ci/run_ci.sh @@ -6,8 +6,6 @@ SCRIPT_LOC=$(pwd) print_header "Installing dependencies" -set +e - sudo apt update sudo apt install -y build-essential libssl-dev libffi-dev sudo apt install -y python3 @@ -22,6 +20,9 @@ check_exit_code "ERROR: Pip not installed" sudo -H pip3 install virtualenv --upgrade check_exit_code "ERROR: virtualenv had trouble upgrading" +sudo -H pip3 install flask --upgrade +check_exit_code "ERROR: Flask not installed or failed to install" + sudo -H pip3 install --pre github3.py check_exit_code "ERROR: GitHub3.py not installed or failed to install" @@ -30,3 +31,7 @@ check_exit_code "ERROR: Paramiko not installed or failed to install" sudo -H pip3 install pexpect check_exit_code "ERROR: Pexpect not installed or failed to install" + +print_header "Done Installing, running CI" +# run the web server with the input arguments +python3 webhook-receiver.py $1 $2 $3 $4 From ef255d0199acdccdb109f42be455d5bb9e28b079 Mon Sep 17 00:00:00 2001 From: Mykola Date: Wed, 5 Jun 2019 19:01:46 -0400 Subject: [PATCH 03/42] Allowed for public linting of unauthorized users --- ci/helper-functions.sh | 2 -- ci/manager.sh | 16 ++++++++++++++++ ci/post-msg.py | 15 ++++++++------- ci/webhook-receiver.py | 9 ++++----- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 6a269e54e..0e28417f5 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -36,8 +36,6 @@ print_header() { done echo "--" echo "" - - sleep 1 } # sets up dpdk, sets env variables, and runs the install script diff --git a/ci/manager.sh b/ci/manager.sh index a4b9bbd96..95d7f9ef8 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -45,6 +45,13 @@ else REQUEST=$4 fi +if [[ -z "$5" ]] +then + AUTHORIZED=true +else + AUTHORIZED=false +fi + . $1 # source the variables from config file print_header "ONVM Performance Testing" @@ -132,6 +139,15 @@ rm -f ../linter-output.txt run_linter ../linter-output.txt cd .. +if ! $AUTHORIZED ; +then + # only run linter and develop checks if unauthorized + print_header "Posting Results in Comment on GitHub" + python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\",\"linter\": 1,\"review\": 1}" $REPO_OWNER $REPO_NAME "Run successful see results:" + check_exit_code "ERROR: Failed to post results to GitHub" + exit 0 +fi + print_header "Preparing Workers" for worker_tuple in "${WORKER_LIST[@]}" diff --git a/ci/post-msg.py b/ci/post-msg.py index 3c0f8771d..b9b1124c9 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -61,13 +61,14 @@ ACTION = 'REQUEST_CHANGES' # PR must not affect performance - with open('./results_summary.stats') as f: - results = json.load(f) - if (results['pass_performance_check']): - comment_body += " :heavy_check_mark: Speed tester performance check passed\n" - else: - comment_body += " :x: PR drops speed tester perforamce bellow minimum requirement\n" - ACTION = 'REQUEST_CHANGES' + if POST_RESULTS: + with open('./results_summary.stats') as f: + results = json.load(f) + if (results['pass_performance_check']): + comment_body += " :heavy_check_mark: Speed tester performance check passed\n" + else: + comment_body += " :x: PR drops speed tester perforamce below minimum requirement\n" + ACTION = 'REQUEST_CHANGES' # PR must pass linter check linter_output = None diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index e6230b6e4..dc588679d 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -179,12 +179,11 @@ def init_ci_pipeline(): log_access_denied(request_ctx, "Incoming webhook has an invalid secret") return jsonify({"success": True}) + unauthorized_run="" if (request_ctx['repo'] == 'openNetVM' and request_ctx['user'] not in authorized_users): + unauthorized_run="True" + # not an authorized user, tell manager to only run linter print("Incoming request is from an unathorized user") - log_access_denied(request_ctx, "Incoming request is from an unathorized user") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"User not authorized to run CI, please contact one of the repo maintainers\"" - .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) - return jsonify({"success": True}) print("Request matches filter, we should RUN CI. {}".format(get_request_info(request_ctx))) @@ -202,7 +201,7 @@ def init_ci_pipeline(): .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], unauthorized_run)) return jsonify({"status": "ONLINE"}) From 8ba65deefa551b135a40eb5b04354a315bddbb9b Mon Sep 17 00:00:00 2001 From: Mykola Date: Wed, 5 Jun 2019 20:43:40 -0400 Subject: [PATCH 04/42] Starting the queue --- ci/manager.sh | 2 +- ci/webhook-receiver.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index a4b9bbd96..e95e6a3c9 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -133,7 +133,7 @@ run_linter ../linter-output.txt cd .. print_header "Preparing Workers" - +exit 1 for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index e6230b6e4..02c970837 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -13,6 +13,7 @@ import os import subprocess import logging +from queue import Queue # Global vars EVENT_URL = "/github-webhook" @@ -23,6 +24,7 @@ secret_file_name = None private_key_file = None secret = None +ci_queue = None app = Flask(__name__) @@ -197,7 +199,7 @@ def init_ci_pipeline(): if (out): print("Can't run CI, another CI run in progress") - log_access_granted(request_ctx, "CI busy, posting busy msg") + log_access_granted(request_ctx, "CI busy, placing request in queue") os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: @@ -254,5 +256,7 @@ def parse_config(cfg_name): secret = decrypt_secret() + ci_queue = Queue() + logging.info("Starting the CI service") app.run(host=host, port=port) From 3a1874b1af5d94523d753572783e1243b10d2c64 Mon Sep 17 00:00:00 2001 From: Mykola Date: Thu, 6 Jun 2019 20:01:10 -0400 Subject: [PATCH 05/42] Start to fixing the queue problem --- ci/webhook-receiver.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 02c970837..293d08c33 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -13,6 +13,7 @@ import os import subprocess import logging +import threading from queue import Queue # Global vars @@ -24,6 +25,7 @@ secret_file_name = None private_key_file = None secret = None +queue_lock = None ci_queue = None app = Flask(__name__) @@ -160,6 +162,11 @@ def filter_to_prs_and_pr_comments(json): return None +# run the manager, and rest of CI process +def run_manager(request_ctx): + log_access_granted(request_ctx, "Running CI") + os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): request_ctx = filter_to_prs_and_pr_comments(request.json) @@ -198,13 +205,28 @@ def init_ci_pipeline(): out, err = proc2.communicate() if (out): - print("Can't run CI, another CI run in progress") + print("Can't run CI, another CI run in progress, placing in the queue") log_access_granted(request_ctx, "CI busy, placing request in queue") - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, please try again in 15 minutes\"" + duplicate_req = False + with queue_lock: + # make sure we don't bypass thread safety + for req in list(ci_queue): + if req['id'] == req['repo'] and req['repo'] == request_ctx['repo']: + duplicate_req = True + break + if not duplicate_req: + # don't have this request yet, put in queue + ci_queue.push(request_ctx) + # ending frees the lock + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, placing request in the queue\"" .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: - log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + # no manager running yet, run with current request + run_manager(request_ctx) + while not queue.empty(): + # new requests were made since our last call, run them in order + print("Running next request in queue") + run_manager(ci_queue.get()) return jsonify({"status": "ONLINE"}) @@ -256,6 +278,7 @@ def parse_config(cfg_name): secret = decrypt_secret() + queue_lock = threading.Lock() ci_queue = Queue() logging.info("Starting the CI service") From 17882a602dc62a5185e65e3df6a203910df64cb0 Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 8 Jun 2019 14:48:28 -0400 Subject: [PATCH 06/42] Added run mode for future advancements --- ci/manager.sh | 9 +++++---- ci/webhook-receiver.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 95d7f9ef8..9696038de 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -47,9 +47,10 @@ fi if [[ -z "$5" ]] then - AUTHORIZED=true + echo "ERROR: Missing fifth argument, Run mode type!" + exit 1 else - AUTHORIZED=false + RUN_MODE=$5 fi . $1 # source the variables from config file @@ -139,7 +140,7 @@ rm -f ../linter-output.txt run_linter ../linter-output.txt cd .. -if ! $AUTHORIZED ; +if [[ "$RUN_MODE" -eq "1" ]] then # only run linter and develop checks if unauthorized print_header "Posting Results in Comment on GitHub" @@ -149,7 +150,7 @@ then fi print_header "Preparing Workers" - +exit 0 for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index dc588679d..1549a9d7a 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -160,6 +160,7 @@ def filter_to_prs_and_pr_comments(json): @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): + run_mode = 0 request_ctx = filter_to_prs_and_pr_comments(request.json) if request_ctx is None: logging.debug("Request filter doesn't match request") @@ -179,9 +180,8 @@ def init_ci_pipeline(): log_access_denied(request_ctx, "Incoming webhook has an invalid secret") return jsonify({"success": True}) - unauthorized_run="" if (request_ctx['repo'] == 'openNetVM' and request_ctx['user'] not in authorized_users): - unauthorized_run="True" + run_mode = 1 # not an authorized user, tell manager to only run linter print("Incoming request is from an unathorized user") @@ -201,7 +201,7 @@ def init_ci_pipeline(): .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) else: log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], unauthorized_run)) + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], run_mode)) return jsonify({"status": "ONLINE"}) From 1c2f52d6fad2d04b4b9a3f0e3c6c8ced2214b123 Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 8 Jun 2019 16:31:59 -0400 Subject: [PATCH 07/42] Major fixes to ci list system --- ci/manager.sh | 4 +++- ci/webhook-receiver.py | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index e95e6a3c9..1795edb9f 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -8,6 +8,8 @@ set -e SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" +sleep 5 +print_header "DONE SLEEPING" if [[ -z "$1" ]] then @@ -133,7 +135,7 @@ run_linter ../linter-output.txt cd .. print_header "Preparing Workers" -exit 1 +exit 0 for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 293d08c33..dc56e3c9e 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -25,8 +25,8 @@ secret_file_name = None private_key_file = None secret = None -queue_lock = None -ci_queue = None +lock = None +ci_list = None app = Flask(__name__) @@ -208,25 +208,27 @@ def init_ci_pipeline(): print("Can't run CI, another CI run in progress, placing in the queue") log_access_granted(request_ctx, "CI busy, placing request in queue") duplicate_req = False + busy_msg = "Duplicate request already waiting, ignoring message" with queue_lock: # make sure we don't bypass thread safety - for req in list(ci_queue): - if req['id'] == req['repo'] and req['repo'] == request_ctx['repo']: + for req in ci_list: + if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: + # make sure this is the same PR duplicate_req = True break if not duplicate_req: # don't have this request yet, put in queue - ci_queue.push(request_ctx) + ci_list.append(request_ctx) + busy_msg = "Another CI run in progress, adding request to the end of the list" # ending frees the lock - os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"Another CI run in progress, placing request in the queue\"" - .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" + .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], busy_msg)) else: # no manager running yet, run with current request run_manager(request_ctx) - while not queue.empty(): + while len(ci_list) > 0: # new requests were made since our last call, run them in order - print("Running next request in queue") - run_manager(ci_queue.get()) + run_manager(ci_list.pop(0)) return jsonify({"status": "ONLINE"}) @@ -279,7 +281,7 @@ def parse_config(cfg_name): secret = decrypt_secret() queue_lock = threading.Lock() - ci_queue = Queue() + ci_list = [] logging.info("Starting the CI service") app.run(host=host, port=port) From 29bbef6eca1af27ab4268266eb90fc02f7cb8cf2 Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 8 Jun 2019 16:33:48 -0400 Subject: [PATCH 08/42] Remove debug lines --- ci/manager.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 1795edb9f..a4b9bbd96 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -8,8 +8,6 @@ set -e SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" -sleep 5 -print_header "DONE SLEEPING" if [[ -z "$1" ]] then @@ -135,7 +133,7 @@ run_linter ../linter-output.txt cd .. print_header "Preparing Workers" -exit 0 + for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) From a1749cb4fa187f32fe2221631f1b8169af4e0495 Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 8 Jun 2019 16:36:05 -0400 Subject: [PATCH 09/42] Clean up definitions --- ci/webhook-receiver.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index dc56e3c9e..3d1800047 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -14,7 +14,6 @@ import subprocess import logging import threading -from queue import Queue # Global vars EVENT_URL = "/github-webhook" @@ -25,7 +24,7 @@ secret_file_name = None private_key_file = None secret = None -lock = None +queue_lock = None ci_list = None app = Flask(__name__) @@ -277,9 +276,7 @@ def parse_config(cfg_name): cfg_name = sys.argv[4] parse_config(cfg_name) - secret = decrypt_secret() - queue_lock = threading.Lock() ci_list = [] From 0c3a90bf271ef3f2327bf3a220d43c7a1568627f Mon Sep 17 00:00:00 2001 From: Mykola Date: Sat, 8 Jun 2019 16:40:20 -0400 Subject: [PATCH 10/42] Remove exit statement --- ci/manager.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/manager.sh b/ci/manager.sh index 9696038de..d07e5f13e 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -150,7 +150,7 @@ then fi print_header "Preparing Workers" -exit 0 + for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) From c10dbb4d3db3d03a33972971999a3155b29f6d4b Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 9 Jun 2019 18:55:05 -0400 Subject: [PATCH 11/42] Add error handling in run_ci and fix docs --- ci/README.md | 2 +- ci/run_ci.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/ci/README.md b/ci/README.md index e490ea5d2..a84513c52 100644 --- a/ci/README.md +++ b/ci/README.md @@ -3,7 +3,7 @@ ### Setting up CI Run a Flask server that listens for new events from github, will get triggered when a new PR is created or when keyword `@onvm` is mentioned. ```sh -python3 webhook-receiver.py 0.0.0.0 8080 @onvm webhook-config.json +./run_ci.sh 0.0.0.0 8080 @onvm webhook-config.json ``` To run CI tests manually, requires a config file, the github PR ID, request message and a response message. diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 9f4a370dd..9c2ab0636 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -1,9 +1,54 @@ #!/bin/bash -# install all dependencies only once + +# exit on error set -e + +# source helper functions file . helper-functions.sh SCRIPT_LOC=$(pwd) +print_header "Validating Input Variables" + +if [[ -z "$1" ]] +then + echo "ERROR: Missing first argument, Host!" + exit 1 +else + HOST=$1 +fi + +if [[ -z "$2" ]] +then + echo "ERROR: Missing second argument, Flask server port!" + exit 1 +else + PORT=$2 +fi + +if [[ -z "$3" ]] +then + echo "ERROR: Missing third argument, Request Keyword!" + exit 1 +else + KEYWORD=$3 +fi + + +if [[ -z "$4" ]] +then + echo "ERROR: Missing fourth argument, path to config file!" + exit 1 +else + if [[ ! -f $4 ]] + then + echo "ERROR: Fourth argument, Config file, is not a file!" + exit 1 + else + CFG_NAME=$4 + fi +fi + +# install all dependencies only once at beginning of CI print_header "Installing dependencies" sudo apt update @@ -33,5 +78,6 @@ sudo -H pip3 install pexpect check_exit_code "ERROR: Pexpect not installed or failed to install" print_header "Done Installing, running CI" + # run the web server with the input arguments -python3 webhook-receiver.py $1 $2 $3 $4 +python3 webhook-receiver.py $HOST $PORT $KEYWORD $CFG_NAME From 5b06b38e45797ac43fde58bb09b828220bd627e2 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 9 Jun 2019 20:18:47 -0400 Subject: [PATCH 12/42] Create polling thread for requests --- ci/webhook-receiver.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 3d1800047..1fcf6e71b 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -13,6 +13,7 @@ import os import subprocess import logging +import time import threading # Global vars @@ -25,7 +26,7 @@ private_key_file = None secret = None queue_lock = None -ci_list = None +ci_request_list = None app = Flask(__name__) @@ -166,6 +167,15 @@ def run_manager(request_ctx): log_access_granted(request_ctx, "Running CI") os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) +def poll_request_list(): + while True: + # continuously check for waiting requests + if len(ci_request_list) > 0: + run_manager(ci_request_list.pop(0)) + else: + # no requests, sleep + time.sleep(1) + @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): request_ctx = filter_to_prs_and_pr_comments(request.json) @@ -208,26 +218,24 @@ def init_ci_pipeline(): log_access_granted(request_ctx, "CI busy, placing request in queue") duplicate_req = False busy_msg = "Duplicate request already waiting, ignoring message" + # make sure we don't bypass thread safety with queue_lock: - # make sure we don't bypass thread safety - for req in ci_list: + for req in ci_request_list: + # make sure this is the same PR if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: - # make sure this is the same PR duplicate_req = True break if not duplicate_req: # don't have this request yet, put in queue - ci_list.append(request_ctx) + ci_request_list.append(request_ctx) busy_msg = "Another CI run in progress, adding request to the end of the list" # ending frees the lock os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], busy_msg)) else: - # no manager running yet, run with current request - run_manager(request_ctx) - while len(ci_list) > 0: - # new requests were made since our last call, run them in order - run_manager(ci_list.pop(0)) + # no manager running yet, put in the list (presumably the top) + with queue_lock: + ci_request_list.append(request_ctx) return jsonify({"status": "ONLINE"}) @@ -278,7 +286,13 @@ def parse_config(cfg_name): parse_config(cfg_name) secret = decrypt_secret() queue_lock = threading.Lock() - ci_list = [] + ci_request_list = [] + + # dedicate thread to check for and run new requests + poll_request = threading.Thread(target=poll_request_list, args=[]) + # run as daemon to catch ^C termination + poll_request.daemon = True + poll_request.start() logging.info("Starting the CI service") app.run(host=host, port=port) From 12935e3c9938e5f1cdfc05affe48ca7ad61b8e2c Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 11 Jun 2019 19:43:17 -0400 Subject: [PATCH 13/42] Delete new line --- ci/run_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 9c2ab0636..4b05c4e3d 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -33,7 +33,6 @@ else KEYWORD=$3 fi - if [[ -z "$4" ]] then echo "ERROR: Missing fourth argument, path to config file!" From b7717454c59cb73c18397c15a93db125f5457375 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 11 Jun 2019 20:09:20 -0400 Subject: [PATCH 14/42] Account for run mode from #140 --- ci/webhook-receiver.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 1fcf6e71b..7d51af8f5 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -163,9 +163,12 @@ def filter_to_prs_and_pr_comments(json): return None # run the manager, and rest of CI process -def run_manager(request_ctx): +def run_manager(request): + # requests are tuples of (context, run mode) + request_ctx = request[0] + mode = request[1] log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\"".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'])) + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], mode)) def poll_request_list(): while True: @@ -178,6 +181,7 @@ def poll_request_list(): @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): + run_mode = 0 request_ctx = filter_to_prs_and_pr_comments(request.json) if request_ctx is None: logging.debug("Request filter doesn't match request") @@ -227,7 +231,7 @@ def init_ci_pipeline(): break if not duplicate_req: # don't have this request yet, put in queue - ci_request_list.append(request_ctx) + ci_request_list.append((request_ctx, run_mode)) busy_msg = "Another CI run in progress, adding request to the end of the list" # ending frees the lock os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" @@ -235,7 +239,7 @@ def init_ci_pipeline(): else: # no manager running yet, put in the list (presumably the top) with queue_lock: - ci_request_list.append(request_ctx) + ci_request_list.append((request_ctx, run_mode)) return jsonify({"status": "ONLINE"}) From ed6587998b1795dcdbff89cd77c4d253bad3ad02 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 11 Jun 2019 21:04:34 -0400 Subject: [PATCH 15/42] Errors are much more difficult in threads :( --- ci/webhook-receiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index e9e6e0cc6..2e8f3c3e1 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -224,7 +224,7 @@ def init_ci_pipeline(): with queue_lock: for req in ci_request_list: # make sure this is the same PR - if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: + if req[0]['id'] == request_ctx['id'] and req[0]['repo'] == request_ctx['repo']: duplicate_req = True break if not duplicate_req: From 3b016602bb1678d70cecd48ae6387d173ed68ce1 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 11 Jun 2019 23:17:04 -0400 Subject: [PATCH 16/42] Smarter than using tuples --- ci/webhook-receiver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 2e8f3c3e1..3eef49f4a 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -163,12 +163,10 @@ def filter_to_prs_and_pr_comments(json): return None # run the manager, and rest of CI process -def run_manager(request): +def run_manager(request_ctx): # requests are tuples of (context, run mode) - request_ctx = request[0] - mode = request[1] log_access_granted(request_ctx, "Running CI") - os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], mode)) + os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], request_ctx['mode'])) def poll_request_list(): while True: @@ -208,6 +206,8 @@ def init_ci_pipeline(): print("Request matches filter, we should RUN CI. {}".format(get_request_info(request_ctx))) + request_ctx['mode'] = run_mode + # Check if there is another CI run in progress proc1 = subprocess.Popen(['ps', 'cax'], stdout=subprocess.PIPE) proc2 = subprocess.Popen(['grep', 'manager.sh'], stdin=proc1.stdout, @@ -224,12 +224,12 @@ def init_ci_pipeline(): with queue_lock: for req in ci_request_list: # make sure this is the same PR - if req[0]['id'] == request_ctx['id'] and req[0]['repo'] == request_ctx['repo']: + if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: duplicate_req = True break if not duplicate_req: # don't have this request yet, put in queue - ci_request_list.append((request_ctx, run_mode)) + ci_request_list.append(request_ctx) busy_msg = "Another CI run in progress, adding request to the end of the list" # ending frees the lock os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" @@ -237,7 +237,7 @@ def init_ci_pipeline(): else: # no manager running yet, put in the list (presumably the top) with queue_lock: - ci_request_list.append((request_ctx, run_mode)) + ci_request_list.append(request_ctx) return jsonify({"status": "ONLINE"}) From 05a89f8556a49b018bae46df732d7b31a251c668 Mon Sep 17 00:00:00 2001 From: OnvmStats Date: Mon, 17 Jun 2019 00:25:28 -0400 Subject: [PATCH 17/42] Added event-based handling instead of polling --- ci/webhook-receiver.py | 75 +++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index 3eef49f4a..ce34963f8 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -26,7 +26,9 @@ private_key_file = None secret = None queue_lock = None +request_event = None ci_request_list = None +manager_running = False app = Flask(__name__) @@ -168,14 +170,26 @@ def run_manager(request_ctx): log_access_granted(request_ctx, "Running CI") os.system("./manager.sh config {} \"{}\" \"{}\" {}".format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], request_ctx['mode'])) -def poll_request_list(): +def request_handler(): + global manager_running while True: - # continuously check for waiting requests - if len(ci_request_list) > 0: + if ci_request_list: + # we have a request + manager_running = True run_manager(ci_request_list.pop(0)) - else: - # no requests, sleep - time.sleep(1) + # remove flag for other threads + manager_running = False + if not ci_request_list: + # list empty, go to sleep until signaled + request_event.wait() + +def add_request(request_ctx): + ci_request_list.append(request_ctx) + if not request_event.isSet(): + # wake up event listener, if it was asleep + request_event.set() + # remove the flag, so we don't get stuck in loop + request_event.clear() @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): @@ -208,36 +222,29 @@ def init_ci_pipeline(): request_ctx['mode'] = run_mode - # Check if there is another CI run in progress - proc1 = subprocess.Popen(['ps', 'cax'], stdout=subprocess.PIPE) - proc2 = subprocess.Popen(['grep', 'manager.sh'], stdin=proc1.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - proc1.stdout.close() - out, err = proc2.communicate() + duplicate_req = False - if (out): - print("Can't run CI, another CI run in progress, placing in the queue") + if (manager_running or ci_request_list): + busy_msg = "Another CI run in progress, adding request to the end of the list" log_access_granted(request_ctx, "CI busy, placing request in queue") - duplicate_req = False - busy_msg = "Duplicate request already waiting, ignoring message" - # make sure we don't bypass thread safety - with queue_lock: - for req in ci_request_list: - # make sure this is the same PR - if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: - duplicate_req = True - break - if not duplicate_req: - # don't have this request yet, put in queue - ci_request_list.append(request_ctx) - busy_msg = "Another CI run in progress, adding request to the end of the list" - # ending frees the lock + if ci_request_list: + # only check for duplicates if list isn't empty + with queue_lock: + for req in ci_request_list: + # make sure this is the same PR + if req['id'] == request_ctx['id'] and req['repo'] == request_ctx['repo']: + duplicate_req = True + break + # ending frees the lock + if duplicate_req: + # let user know we're not running this request, it's a duplicate + busy_msg = "Duplicate request already waiting, ignoring message" os.system("./ci_busy.sh config {} \"{}\" \"{}\" \"{}\"" .format(request_ctx['id'], request_ctx['repo'], request_ctx['body'], busy_msg)) - else: - # no manager running yet, put in the list (presumably the top) - with queue_lock: - ci_request_list.append(request_ctx) + + if not duplicate_req: + # we are not a duplicate, add to list + add_request(request_ctx) return jsonify({"status": "ONLINE"}) @@ -290,8 +297,10 @@ def parse_config(cfg_name): queue_lock = threading.Lock() ci_request_list = [] + # create an event handler for requests + request_event = threading.Event() # dedicate thread to check for and run new requests - poll_request = threading.Thread(target=poll_request_list, args=[]) + poll_request = threading.Thread(target=request_handler, args=[]) # run as daemon to catch ^C termination poll_request.daemon = True poll_request.start() From 24965be95982e17f67d4ee13afd04fba15bee6d1 Mon Sep 17 00:00:00 2001 From: OnvmStats Date: Tue, 18 Jun 2019 20:14:18 -0400 Subject: [PATCH 18/42] Let request handler clears event --- ci/webhook-receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/webhook-receiver.py b/ci/webhook-receiver.py index ce34963f8..f97a00a8b 100644 --- a/ci/webhook-receiver.py +++ b/ci/webhook-receiver.py @@ -182,14 +182,14 @@ def request_handler(): if not ci_request_list: # list empty, go to sleep until signaled request_event.wait() + # remove the flag, so we don't get stuck in loop + request_event.clear() def add_request(request_ctx): ci_request_list.append(request_ctx) if not request_event.isSet(): # wake up event listener, if it was asleep request_event.set() - # remove the flag, so we don't get stuck in loop - request_event.clear() @app.route(EVENT_URL, methods=['POST']) def init_ci_pipeline(): From 7e71ec2f59981a0c6500b238f6860cf0b1beddbc Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sat, 22 Jun 2019 19:39:53 -0400 Subject: [PATCH 19/42] Initialize ci for pktgen --- ci/helper-functions.sh | 9 +++- ci/install.py | 13 ++++++ ci/pktgen-analysis.py | 45 ++++++++++++++++++++ ci/pktgen-worker.sh | 37 +++++++++++++++++ ci/run-pktgen.py | 22 ++++++++++ ci/run_test.sh | 3 ++ ci/speed-worker.sh | 20 +++++++++ ci/worker-config | 2 + ci/worker.sh | 93 +++++++++++++++++++++++++++--------------- 9 files changed, 209 insertions(+), 35 deletions(-) create mode 100644 ci/install.py create mode 100644 ci/pktgen-analysis.py create mode 100755 ci/pktgen-worker.sh create mode 100644 ci/run-pktgen.py create mode 100755 ci/run_test.sh create mode 100755 ci/speed-worker.sh create mode 100755 ci/worker-config diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 0e28417f5..440beb1e6 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -62,8 +62,13 @@ install_env() { sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" cd ../ - pwd - . ./scripts/install.sh + # check if we need to bind an interface + if [[ -z $1 || ! $1 ]] + then + . ./scripts/install.sh + else + python3 ~/install.py + fi } # makes all onvm code diff --git a/ci/install.py b/ci/install.py new file mode 100644 index 000000000..991884b84 --- /dev/null +++ b/ci/install.py @@ -0,0 +1,13 @@ +import pexpect +import os + +cwd = os.path.dirname(os.path.realpath(__file__)) +cmd = cwd + "/repository/scripts/install.sh" +child = pexpect.spawn(cmd) + +# bind correct interface to DPDK +child.expect("Bind interface 0000:07:00.0 to DPDK.*") +child.sendline("N") +child.expect("Bind interface 0000:07:00.1 to DPDK.*") +child.sendline("Y\n") +child.interact() diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py new file mode 100644 index 000000000..b24112b85 --- /dev/null +++ b/ci/pktgen-analysis.py @@ -0,0 +1,45 @@ +import sys +import os +import json + +AVG_SPEED = 6000000 + +""" +get median value of tx pps +""" +def median(array): + array = sorted(array) + half, odd = divmod(len(array), 2) + if odd: + return array[half] + else: + return (array[half - 1] + array[half]) / 2.0 + +if(len(sys.argv) != 4): + print("ERROR: Invalid arguments.") + sys.exit(1) + +STATS_FILE = sys.argv[1] +STATS_NODE_NAME = sys.argv[2] +OUT_FILE = sys.argv[3] + +contents = tuple(open(STATS_FILE, "r")) +# data = [int(x.rstrip()) if x.rstrip() for x in contents] +data = [int(x.rstrip()) for x in contents if x.rstrip()] + +median_speed = median(data) + +results = {} +results['node'] = STATS_NODE_NAME +results['speed'] = median_speed + +performance_rating = (median_speed / AVG_SPEED) * 100 +if (performance_rating < 97): + results['pass_performance_check'] = False +else: + results['pass_performance_check'] = True +results['performance_rating'] = performance_rating +results['summary'] = "[Results from %s]\n - Median TX pps for Pktgen: %d\n - Performance rating - %.2f%% (compared to %d average)" % (STATS_NODE_NAME, median_speed, performance_rating, AVG_SPEED) + +with open(OUT_FILE, 'w') as outfile: + json.dump(results, outfile) diff --git a/ci/pktgen-worker.sh b/ci/pktgen-worker.sh new file mode 100755 index 000000000..92ff5f702 --- /dev/null +++ b/ci/pktgen-worker.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +cd ~ +. helper-functions.sh + +if [[ -z $1 ]] +then + echo "ERROR: Missing first argument, path to pktgen config file" + exit 1 +fi + +if [[ ! -f $1 ]] +then + echo "ERROR: Could not find config file at given path!" + exit 1 +fi + +. $1 # source the variables from pktgen config file + +print_header "Running Basic Monitor NF" +cd repository/examples/basic_monitor +./go.sh 1 &>~/bsc_stats & +bsc_mntr_pid=$? +if [ $bsc_mntr_pid -ne 0 ] +then + echo "ERROR: Starting basic monitor failed" + return 1 +fi + +# run pktgen +print_header "Collecting Pktgen Statistics" +python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE +# get Pktgen stats from server +scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/port_stats + +print_header "Killing Basic Monitor" +sudo pkill -f /basic_monitor diff --git a/ci/run-pktgen.py b/ci/run-pktgen.py new file mode 100644 index 000000000..ca175cb5d --- /dev/null +++ b/ci/run-pktgen.py @@ -0,0 +1,22 @@ +from paramiko import RSAKey +from paramiko import SSHClient +from paramiko import AutoAddPolicy +import sys + +worker_ip = sys.argv[1] +key_file = sys.argv[2] + +key = RSAKey.from_private_key_file(key_file) + +client = SSHClient() +client.set_missing_host_key_policy(AutoAddPolicy()) + +client.connect(worker_ip, timeout = 30, pkey = key) + +(stdin, stdout, stderr) = client.exec_command("sudo ~/repository/tools/Pktgen/openNetVM-Scripts/run-pktgen.sh 1", get_pty=True) +# block until script finishes +exit_status = stdout.channel.recv_exit_status() + +print("Successfully ran pktgen".format(worker_ip)) + +client.close() diff --git a/ci/run_test.sh b/ci/run_test.sh new file mode 100755 index 000000000..50d204079 --- /dev/null +++ b/ci/run_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "thet" diff --git a/ci/speed-worker.sh b/ci/speed-worker.sh new file mode 100755 index 000000000..fbc71f274 --- /dev/null +++ b/ci/speed-worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +. ~/helper-functions.sh + +print_header "Running Speed Tester NF" +cd ~/repository/examples/speed_tester +./go.sh 1 -d 1 &>~/spd_stats & +spd_tstr_pid=$? +if [ $spd_tstr_pid -ne 0 ] +then + echo "ERROR: Starting speed tester failed" + # exit 1 +fi + +# wait for speed tester to come online +print_header "Collecting Speed Tester Statistics" +sleep 15 + +print_header "Killing Speed Tester" +sudo pkill -f /speed_tester diff --git a/ci/worker-config b/ci/worker-config new file mode 100755 index 000000000..ac716c1bb --- /dev/null +++ b/ci/worker-config @@ -0,0 +1,2 @@ +MODES="0" +PKT_CONFIG=~/pkt-config diff --git a/ci/worker.sh b/ci/worker.sh index c49baae9d..0819a4013 100755 --- a/ci/worker.sh +++ b/ci/worker.sh @@ -1,21 +1,56 @@ #!/bin/bash - # source helper functions file +if [[ -z $1 ]] +then + echo "ERROR: Missing first argument, path to config file!" + exit 1 +fi + +if [[ ! -f $1 ]] +then + echo "ERROR: Could not find config file at given path!" + exit 1 +fi + +. $1 # source config file + +if [[ -z $MODES ]] +then + echo "ERROR: Missing mode argument of config!" + exit 1 +fi + +RUN_PKT=false +if [[ $MODES == *0* ]] +then + # will be running pktgen + if [[ ! -f $PKT_CONFIG ]] + then + echo "ERROR: Mode 0 (Pktgen) must have a config file" + exit 1 + fi + RUN_PKT=true +fi + +# source helper functions file . helper-functions.sh -sudo apt-get update -sudo apt-get upgrade -y +# sudo apt-get update +# sudo apt-get upgrade -y + +# sudo apt-get install -y build-essential linux-headers-$(uname -r) git +# sudo apt-get install -y libnuma1 +# sudo apt-get install -y libnuma-dev -sudo apt-get install -y build-essential linux-headers-$(uname -r) git -sudo apt-get install -y libnuma1 -sudo apt-get install -y libnuma-dev +# clear lingering stats +rm -rf *stats* cd repository print_header "Beginning Execution of Workload" print_header "Installing Environment" -install_env +install_env $RUN_PKT check_exit_code "ERROR: Installing environment failed" print_header "Building ONVM" @@ -24,7 +59,7 @@ check_exit_code "ERROR: Building ONVM failed" print_header "Running ONVM Manager" cd onvm -./go.sh 0,1,2,3 0 0xF0 -s web -a 0x7f000000000 & +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & mgr_pid=$? if [ $mgr_pid -ne 0 ] then @@ -34,35 +69,27 @@ fi # wait for the manager to come online sleep 15 - -# Some weirdness with PIDS, look into this -# Works, but we don't know why - -print_header "Running Speed Tester NF" -cd ../examples/speed_tester -./go.sh 1 -d 1 &>~/stats & -spd_tstr_pid=$? -if [ $spd_tstr_pid -ne 0 ] -then - echo "ERROR: Starting speed tester failed" - sudo kill $mgr_pid - exit 1 -fi - - # wait for speed tester to come online -print_header "Collecting Statistics" -sleep 15 +print_header "Manager is live" + +for mode in $MODES +do + # run functionality for each mode + case "$mode" in + "0") + ~/pktgen-worker.sh $PKT_CONFIG + ;; + "1") + ~/speed-worker.sh + ;; + *) + echo "Mode $MODE has not been implemented" + ;; + esac +done print_header "Exiting ONVM" -echo "Speed tester pid: ${spd_tstr_pid}" -echo $spd_tstr_pid echo "Manager pid: ${mgr_pid}" -echo $mgr_pid - -sudo kill $spd_tstr_pid -check_exit_code "Error: Killing speed tester failed!" - sudo kill $mgr_pid check_exit_code "ERROR: Killing manager failed" From ee2220c71d09774d37de210d5f1bd322be485c92 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 23 Jun 2019 21:53:46 -0400 Subject: [PATCH 20/42] Huge update to pktgen configuration --- ci/install-pktgen/helper-functions.sh | 100 ++++++++++++++++++++++ ci/install-pktgen/pktgen-config.lua | 111 +++++++++++++++++++++++++ ci/install-pktgen/start-pktgen.sh | 50 +++++++++++ ci/manager.sh | 11 ++- ci/prepare-worker.py | 7 +- ci/run-workload.py | 5 +- ci/run_test.sh | 3 - ci/worker-files/.worker.sh.swp | Bin 0 -> 12288 bytes ci/worker-files/config-test | 1 + ci/worker-files/helper-functions.sh | 100 ++++++++++++++++++++++ ci/{ => worker-files}/install.py | 6 +- ci/worker-files/pkt-config | 3 + ci/{ => worker-files}/pktgen-worker.sh | 4 +- ci/{ => worker-files}/run-pktgen.py | 8 +- ci/{ => worker-files}/speed-worker.sh | 0 ci/{ => worker-files}/worker-config | 0 ci/{ => worker-files}/worker.sh | 10 +-- 17 files changed, 396 insertions(+), 23 deletions(-) create mode 100644 ci/install-pktgen/helper-functions.sh create mode 100644 ci/install-pktgen/pktgen-config.lua create mode 100755 ci/install-pktgen/start-pktgen.sh delete mode 100755 ci/run_test.sh create mode 100644 ci/worker-files/.worker.sh.swp create mode 100644 ci/worker-files/config-test create mode 100644 ci/worker-files/helper-functions.sh rename ci/{ => worker-files}/install.py (66%) create mode 100644 ci/worker-files/pkt-config rename ci/{ => worker-files}/pktgen-worker.sh (80%) rename ci/{ => worker-files}/run-pktgen.py (74%) rename ci/{ => worker-files}/speed-worker.sh (100%) rename ci/{ => worker-files}/worker-config (100%) rename ci/{ => worker-files}/worker.sh (89%) diff --git a/ci/install-pktgen/helper-functions.sh b/ci/install-pktgen/helper-functions.sh new file mode 100644 index 000000000..4e6c1b404 --- /dev/null +++ b/ci/install-pktgen/helper-functions.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# simple print function that takes a single argument +# prints the argument as a header wrapped in dashes and pipes +print_header() { + # check that the argument exists + if [ -z "$1" ] + then + echo "No argument supplied to print_header!" + return 1 + fi + + # get the string length for formatting + strlen=${#1} + + echo "" + + # echo first line of dashes + echo -n "--" + for i in `eval echo {1..$strlen}` + do + echo -n "-" + done + echo "--" + + # echo the argument + echo -n "| " + echo -n $1 + echo " |" + + # echo the second line of dashes + echo -n "--" + for i in `eval echo {1..$strlen}` + do + echo -n "-" + done + echo "--" + echo "" +} + + # sets up dpdk, sets env variables, and runs the install script +install_env() { + git submodule sync + git submodule update --init + + echo export ONVM_HOME=$(pwd) >> ~/.bashrc + export ONVM_HOME=$(pwd) + + cd dpdk + + echo export RTE_SDK=$(pwd) >> ~/.bashrc + export RTE_SDK=$(pwd) + + echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc + export RTE_TARGET=x86_64-native-linuxapp-gcc + + echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc + export ONVM_NUM_HUGEPAGES=1024 + + echo $RTE_SDK + + sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" + + cd ../ + # check if we need to bind an interface + if [[ -z $1 || ! $1 ]] + then + . ./scripts/install.sh + else + python3 ~/install.py + fi +} + +# makes all onvm code +build_onvm() { + cd onvm + make clean && make + cd ../ + + cd examples + make clean && make + cd ../ +} + +# obtains core config in cores.out file +obtain_core_config() { + cd scripts + ./corehelper.py > cores.out +} + +# checks if a command has failed (exited with code != 0) +# if it does, print the error message, exit the build, and post to github +check_exit_code() { + if [ $? -ne 0 ] + then + echo $1 + cd $SCRIPT_LOC + exit 1 + fi +} diff --git a/ci/install-pktgen/pktgen-config.lua b/ci/install-pktgen/pktgen-config.lua new file mode 100644 index 000000000..cd0f5b7d5 --- /dev/null +++ b/ci/install-pktgen/pktgen-config.lua @@ -0,0 +1,111 @@ +-- openNetVM +-- https://github.com/sdnfv/openNetVM +-- +-- BSD LICENSE +-- +-- Copyright(c) +-- 2015-2016 George Washington University +-- 2015-2016 University of California Riverside +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions +-- are met: + +-- Redistributions of source code must retain the above copyright +-- notice, this list of conditions and the following disclaimer. +-- Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in +-- the documentation and/or other materials provided with the +-- distribution. +-- The name of the author may not be used to endorse or promote +-- products derived from this software without specific prior +-- written permission. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +-- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +-- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +-- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +-- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +-- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +-- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- Change any of the settings below to configure Pktgen-DPDK + +-- A list of the test script for Pktgen and Lua. +-- Each command somewhat mirrors the pktgen command line versions. +-- A couple of the arguments have be changed to be more like the others. + +package.path = package.path ..";?.lua;test/?.lua;app/?.lua;" + +require "Pktgen" + +local function doWait(port, waitTime) + local idx; + + pktgen.delay(1000); + + pkt_rate_file = io.open("port_stats", "w"); + + if ( waitTime == 0 ) then + return; + end + waitTime = waitTime - 1; + + -- Try to wait for the total number of packets to be sent. + local idx = 0; + while( idx < waitTime ) do + + -- Write port stats to output file separated by line + pkt_rate_file:write(pktgen.portStats("all", "rate")[0]["pkts_rx"] .. "\n"); + idx = idx + 1; + + local sending = pktgen.isSending(port); + if ( sending[tonumber(port)] == "n" ) then + break; + end + pktgen.delay(1000); + end + + pkt_rate_file:close() +end + +printf("Lua Version : %s\n", pktgen.info.Lua_Version); +printf("Pktgen Version : %s\n", pktgen.info.Pktgen_Version); +printf("Pktgen Copyright : %s\n", pktgen.info.Pktgen_Copyright); + +prints("pktgen.info", pktgen.info); + +printf("Port Count %d\n", pktgen.portCount()); +printf("Total port Count %d\n", pktgen.totalPorts()); + + +-- set up a mac address to set flow to +-- +-- TO DO LIST: +-- +-- Please update this part with the destination mac address, source and destination ip address you would like to sent packets to + +pktgen.set_mac("0", "90:e2:ba:5e:73:21"); +pktgen.set_ipaddr("0", "dst", "10.11.1.17"); +pktgen.set_ipaddr("0", "src", "10.11.1.16"); + +pktgen.set_proto("all", "udp"); +pktgen.set_type("all", "ipv4"); + +pktgen.set("all", "size", 64) +pktgen.set("all", "burst", 32); +pktgen.set("all", "sport", 1234); +pktgen.set("all", "dport", 1234); +pktgen.set("all", "count", 1000000000); +pktgen.set("all", "rate",100); + +pktgen.vlan_id("all", "start", 1); + +pktgen.start("all"); +doWait("all", 30); +pktgen.quit(); diff --git a/ci/install-pktgen/start-pktgen.sh b/ci/install-pktgen/start-pktgen.sh new file mode 100755 index 000000000..30235c9cd --- /dev/null +++ b/ci/install-pktgen/start-pktgen.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +. helper-functions.sh + +set -e + +sudo rm -rf repository + +git clone https://github.com/sdnfv/openNetVM.git repository +check_exit_code "ERROR: Failed installing onvm" + +sudo apt-get update +sudo apt-get upgrade -y + +sudo apt-get install -y build-essential linux-headers-$(uname -r) git +sudo apt-get install -y libnuma1 +sudo apt-get install -y libnuma-dev + +cd repository + +print_header "Beginning Execution of Workload" + +print_header "Installing Environment" +install_env $RUN_PKT +check_exit_code "ERROR: Installing environment failed" + +print_header "Building ONVM" +build_onvm +check_exit_code "ERROR: Building ONVM failed" + +cd ~ + +# install dependencies +sudo apt-get install libpcap-dev +sudo apt-get install libreadline-dev + +print_header "Installing Lua" +curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz +tar zxf lua-5.3.5.tar.gz +cd lua-5.3.5 +sudo make install + +print_header "Make pktgen-dpdk" +cd ~/repository/tools/Pktgen/pktgen-dpdk/ +make + +print_header "Updating lua script" +cp ~/pktgen-config.lua ~/repository/tools/Pktgen/openNetVM-Scripts/ + +print_header "Pktgen installed" diff --git a/ci/manager.sh b/ci/manager.sh index dce469143..ffe1333ed 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -159,7 +159,9 @@ do worker_key_file="${tuple_arr[1]}" scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./repository $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null helper-functions.sh worker.sh $worker_ip: + # make sure the config file is updated with the correct run mode + sed -i "/MODES*/c\\MODES=\"${RUN_MODE}\"" worker-files/worker-config + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./worker-files/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" done @@ -185,7 +187,12 @@ do scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:stats ./$worker_ip.stats check_exit_code "ERROR: Failed to fetch results from $worker_ip" # TODO: this will overwrite results if we have more than 1 worker, investigate this case - python3 speed-tester-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + if [[ "$RUN_MODE" -eq "0" ]] + then + python3 pktgen-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + else + python3 speed-tester-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + fi check_exit_code "ERROR: Failed to analyze results from $worker_ip" done diff --git a/ci/prepare-worker.py b/ci/prepare-worker.py index 8195c45d5..50f469ed2 100644 --- a/ci/prepare-worker.py +++ b/ci/prepare-worker.py @@ -19,9 +19,10 @@ (stdin, stdout, stderr) = client.exec_command("sudo rm -rf /mnt/huge/*") (stdin, stdout, stderr) = client.exec_command("sudo rm -rf repository") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf stats") -(stdin, stdout, stderr) = client.exec_command("sudo rm -f worker.sh") -(stdin, stdout, stderr) = client.exec_command("sudo rm -f helper-functions.sh") +(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *stats*") +(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.py") +(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *config") +(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.sh") (stdin, stdout, stderr) = client.exec_command("sudo reboot") print("Successfully sent {} to reboot".format(worker_ip)) diff --git a/ci/run-workload.py b/ci/run-workload.py index 9f7dc202f..31a880da0 100644 --- a/ci/run-workload.py +++ b/ci/run-workload.py @@ -30,9 +30,6 @@ def line_buffered(f): client.connect(ip, timeout = 30, pkey = key) -(stdin, stdout, stderr) = client.exec_command("sudo ./worker.sh") - -for l in line_buffered(stdout): - print(l.strip("\n")) +(stdin, stdout, stderr) = client.exec_command("sudo ./worker.sh worker-config", get_pty=True) print(str(stderr.read())) diff --git a/ci/run_test.sh b/ci/run_test.sh deleted file mode 100755 index 50d204079..000000000 --- a/ci/run_test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "thet" diff --git a/ci/worker-files/.worker.sh.swp b/ci/worker-files/.worker.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..c18f8b041f3a219cfb81947320882300da170d42 GIT binary patch literal 12288 zcmeI2O^*~s7{?1@;wxe>8iC8Rvl9ZF?VTAGqk*g>;9^!XyJUA+4`egcbXU)m+f&`? zt{!$p5I=xMG-~i5#)Epni$@g?5)VeBAHYP7L{BD0!N`Hce|7f^v#Tr#H6|f3e1*`&A0jq#jz$$R3Dxjkc>?It2W8uK@;=W_z}DZ-UP3M zv)~Lk3HF0Mpa!boQSb=Z2(CZK*w^3-@EQ0NTmWx@vp|CdPzRd<1J^b&_8YhiegT)j z$KV1u2VMqez+SKwTz!DCE8qw4E%*jp1fPRf!7HEzX2Eu_4P3vUu|L5j@Fh48-Ujnv z3XFp>a2L3CA7g)mtKd8E75ET*0Nwzvf#<;jsDm9~JJURD9CfK|XMU=^?m+@1naER}Aw1@}Zur6VHt!??qhD`;LM zI-!GMCkliX9?hk$E0V-dgJ7k^nA;ZaQbR0D-Ei@7Bb6p+XJ=;j(qS0{Qnd(1cndat zj!zf85=&Aqq$Lbexzmapk@T2w+aZ%X7b?LB zmq@sqJ(!n$`}&WCb#u0Iedg&&YV(9tsHrK0B5B;}2nEkd*^-f6{gneoPFIph2+zqs z!!%!B60TCMbd%Z9l5PogV>*?O9G-6+oS8l}^-R9BS}b*W0@2DHoUcSuWrM>Mr;7Yk zxmt#b2U4#P;z5GDZ89_AH8Ve$XDrGcn_H_?Pf?jrAiDzE2SP+tdz=l^2`+VJQ@2Gw zMmi)n><9`KLMIsWWp*4BT&m@dS*VH-&uJcKp;#MZ*i=mVbgUm^a+IJdEwUo46XpGE ztkMdhBvnUiqq|4PX`HIdhpOao+@J8Py^EZL677YlrX5e6u7s-FX(XB>CwDN{!&$qX zH5oFSD%HynDnwW(QxBvv@cQ$6dD=9;HvY%@s_E;Pol-buUIRjPWgIG#X8k&l`EUbc zRGvLL-9Un@@ngWk87 zkw6s51G9U&?Boq7amqVW#gU0lyo*{=uQ^K^wPG|XoMpB7H1fDc-i}dQ1d;h%x+tN< zRO~P%VH%^O<%9kDO2~Xw-Ou(S785#71!YNJlmX!yD242GupH{97^}4NNLKs1sFNld z&891eqpXxKTiHf=o5@PVm@s8>jE2bZ2YUo9E;2KULN%?jDNIMmk~;4!tY2cNG1JxN zXkvm!sv~A$&jX(a$ql~-j-I!~qKfM>NlehQUc%#6nx!ccC_nR1X>c#+h2D192RfH) zIAS5|fRBAKZ?@^dFb%w{3;R-egYpp9h> ~/.bashrc + export ONVM_HOME=$(pwd) + + cd dpdk + + echo export RTE_SDK=$(pwd) >> ~/.bashrc + export RTE_SDK=$(pwd) + + echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc + export RTE_TARGET=x86_64-native-linuxapp-gcc + + echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc + export ONVM_NUM_HUGEPAGES=1024 + + echo $RTE_SDK + + sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" + + cd ../ + # check if we need to bind an interface + if [[ -z $1 || ! $1 ]] + then + . ./scripts/install.sh + else + python3 ~/install.py + fi +} + +# makes all onvm code +build_onvm() { + cd onvm + make clean && make + cd ../ + + cd examples + make clean && make + cd ../ +} + +# obtains core config in cores.out file +obtain_core_config() { + cd scripts + ./corehelper.py > cores.out +} + +# checks if a command has failed (exited with code != 0) +# if it does, print the error message, exit the build, and post to github +check_exit_code() { + if [ $? -ne 0 ] + then + echo $1 + cd $SCRIPT_LOC + exit 1 + fi +} diff --git a/ci/install.py b/ci/worker-files/install.py similarity index 66% rename from ci/install.py rename to ci/worker-files/install.py index 991884b84..335c1f210 100644 --- a/ci/install.py +++ b/ci/worker-files/install.py @@ -2,9 +2,11 @@ import os cwd = os.path.dirname(os.path.realpath(__file__)) -cmd = cwd + "/repository/scripts/install.sh" -child = pexpect.spawn(cmd) +cwd += "/repository/scripts" +child = pexpect.spawn(cwd + "/install.sh", cwd=cwd) +# script takes a while, increase timeout +child.timeout = 120 # bind correct interface to DPDK child.expect("Bind interface 0000:07:00.0 to DPDK.*") child.sendline("N") diff --git a/ci/worker-files/pkt-config b/ci/worker-files/pkt-config new file mode 100644 index 000000000..b72fa6a89 --- /dev/null +++ b/ci/worker-files/pkt-config @@ -0,0 +1,3 @@ +PKT_WORKER_IP="nimbnode18" +PKT_WORKER_KEY_FILE=~/ci18key +WORKER_USER="kdeems" diff --git a/ci/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh similarity index 80% rename from ci/pktgen-worker.sh rename to ci/worker-files/pktgen-worker.sh index 92ff5f702..ca9f180e2 100755 --- a/ci/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -29,9 +29,9 @@ fi # run pktgen print_header "Collecting Pktgen Statistics" -python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE +python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE $WORKER_USER # get Pktgen stats from server -scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/port_stats +scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/stats print_header "Killing Basic Monitor" sudo pkill -f /basic_monitor diff --git a/ci/run-pktgen.py b/ci/worker-files/run-pktgen.py similarity index 74% rename from ci/run-pktgen.py rename to ci/worker-files/run-pktgen.py index ca175cb5d..63bdcdaed 100644 --- a/ci/run-pktgen.py +++ b/ci/worker-files/run-pktgen.py @@ -3,15 +3,19 @@ from paramiko import AutoAddPolicy import sys +if len(sys.argv) != 4: + print("ERROR! Incorrect number of arguments") + sys.exit(1) + worker_ip = sys.argv[1] key_file = sys.argv[2] +worker_user = sys.argv[3] key = RSAKey.from_private_key_file(key_file) client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy()) - -client.connect(worker_ip, timeout = 30, pkey = key) +client.connect(worker_ip, timeout = 30, username=worker_user, pkey = key) (stdin, stdout, stderr) = client.exec_command("sudo ~/repository/tools/Pktgen/openNetVM-Scripts/run-pktgen.sh 1", get_pty=True) # block until script finishes diff --git a/ci/speed-worker.sh b/ci/worker-files/speed-worker.sh similarity index 100% rename from ci/speed-worker.sh rename to ci/worker-files/speed-worker.sh diff --git a/ci/worker-config b/ci/worker-files/worker-config similarity index 100% rename from ci/worker-config rename to ci/worker-files/worker-config diff --git a/ci/worker.sh b/ci/worker-files/worker.sh similarity index 89% rename from ci/worker.sh rename to ci/worker-files/worker.sh index 0819a4013..1a45ca976 100755 --- a/ci/worker.sh +++ b/ci/worker-files/worker.sh @@ -35,12 +35,12 @@ fi # source helper functions file . helper-functions.sh -# sudo apt-get update -# sudo apt-get upgrade -y +sudo apt-get update +sudo apt-get upgrade -y -# sudo apt-get install -y build-essential linux-headers-$(uname -r) git -# sudo apt-get install -y libnuma1 -# sudo apt-get install -y libnuma-dev +sudo apt-get install -y build-essential linux-headers-$(uname -r) git +sudo apt-get install -y libnuma1 +sudo apt-get install -y libnuma-dev # clear lingering stats rm -rf *stats* From 1c68a5f3cbc6a8c864115ced71af664b59edba66 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 23 Jun 2019 21:54:26 -0400 Subject: [PATCH 21/42] Updated gitignore for log files --- ci/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/.gitignore b/ci/.gitignore index 100ae87b0..b98f83975 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -1,4 +1,3 @@ -access_log config webhook-config.json encrypted_secret.bin @@ -11,3 +10,5 @@ linter-output.txt *key *.pub *.stats +*out* +*log* From 32766cd43983984d63bb378fe766efe41159593f Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 25 Jun 2019 00:22:38 -0400 Subject: [PATCH 22/42] Added speed tester and pktgen in conjunction --- ci/.gitignore | 2 +- ci/manager.sh | 19 +++++++---- ci/pktgen-analysis.py | 4 +-- ci/post-msg.py | 55 +++++++++++++++++++++++--------- ci/prepare-worker.py | 2 +- ci/speed-tester-analysis.py | 3 +- ci/worker-files/pktgen-worker.sh | 2 +- ci/worker-files/speed-worker.sh | 2 +- ci/worker-files/worker.sh | 10 +++--- 9 files changed, 65 insertions(+), 34 deletions(-) diff --git a/ci/.gitignore b/ci/.gitignore index b98f83975..988cfde5e 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -9,6 +9,6 @@ repository linter-output.txt *key *.pub -*.stats +*stats *out* *log* diff --git a/ci/manager.sh b/ci/manager.sh index ffe1333ed..b626f1b0d 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -177,21 +177,28 @@ done print_header "Obtaining Performance Results from all workers" -rm -f results_summary.stats +rm -f *summary.stats for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:stats ./$worker_ip.stats - check_exit_code "ERROR: Failed to fetch results from $worker_ip" - # TODO: this will overwrite results if we have more than 1 worker, investigate this case + # TODO: this will overwrite results if we have more than 1 worker, investigate this case if [[ "$RUN_MODE" -eq "0" ]] then - python3 pktgen-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + # fetch pktgen stats + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:pktgen_stats ./$worker_ip.pktgen_stats + check_exit_code "ERROR: Failed to fetch results from $worker_ip" + python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats + # fetch speed_tester stats + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:speed_stats ./$worker_ip.speed_stats + check_exit_code "ERROR: Failed to fetch results from $worker_ip" + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats else - python3 speed-tester-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:speed_stats ./$worker_ip.speed_stats + check_exit_code "ERROR: Failed to fetch results from $worker_ip" + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats fi check_exit_code "ERROR: Failed to analyze results from $worker_ip" done diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py index b24112b85..2898761ae 100644 --- a/ci/pktgen-analysis.py +++ b/ci/pktgen-analysis.py @@ -24,7 +24,6 @@ def median(array): OUT_FILE = sys.argv[3] contents = tuple(open(STATS_FILE, "r")) -# data = [int(x.rstrip()) if x.rstrip() for x in contents] data = [int(x.rstrip()) for x in contents if x.rstrip()] median_speed = median(data) @@ -39,7 +38,8 @@ def median(array): else: results['pass_performance_check'] = True results['performance_rating'] = performance_rating -results['summary'] = "[Results from %s]\n - Median TX pps for Pktgen: %d\n - Performance rating - %.2f%% (compared to %d average)" % (STATS_NODE_NAME, median_speed, performance_rating, AVG_SPEED) +results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) +results['summary'] = "\n - Median TX pps for Pktgen: %d\n - Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) with open(OUT_FILE, 'w') as outfile: json.dump(results, outfile) diff --git a/ci/post-msg.py b/ci/post-msg.py index b9b1124c9..c87f31f32 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -4,6 +4,7 @@ import json import sys +from os import path REPO_OWNER = None REPO_NAME = None @@ -14,6 +15,28 @@ print("ERROR! Incorrect number of arguments") sys.exit(1) +def apply_stats(file, name): + global comment_body + global ACTION + if path.exists(file): + with open(file) as f: + results = json.load(f) + if (results['pass_performance_check']): + comment_body += " :heavy_check_mark: {} performance check passed\n".format(name) + else: + comment_body += " :x: PR drops {} performance below minimum requirement\n".format(name) + ACTION = 'REQUEST_CHANGES' + +def push_results(file): + global comment_body + global previous_results_from + with open(file) as f: + results = json.load(f) + if previous_results_from != results['results_from']: + comment_body += "\n " + results['results_from'] + previous_results_from = results['results_from'] + comment_body += "\n " + results['summary'] + with open(sys.argv[1], "r") as credsfile: creds = [x.strip() for x in credsfile.readlines()] @@ -45,6 +68,7 @@ print("ERROR: Could not get PR information from GitHub for PR %d" % int(sys.argv[2])) sys.exit(1) +previous_results_from = "" comment_body="" for line in REQUEST.split('\n'): comment_body += "> " + line + "\n" @@ -62,13 +86,10 @@ # PR must not affect performance if POST_RESULTS: - with open('./results_summary.stats') as f: - results = json.load(f) - if (results['pass_performance_check']): - comment_body += " :heavy_check_mark: Speed tester performance check passed\n" - else: - comment_body += " :x: PR drops speed tester perforamce below minimum requirement\n" - ACTION = 'REQUEST_CHANGES' + file = './pktgen_summary.stats' + apply_stats(file, "Pktgen") + file = './speed_summary.stats' + apply_stats(file, "Speed Test") # PR must pass linter check linter_output = None @@ -85,9 +106,10 @@ comment_body += " :heavy_check_mark: Linter passed\n" if POST_RESULTS: - with open('./results_summary.stats') as f: - results = json.load(f) - comment_body += "\n " + results['summary'] + file = './pktgen_summary.stats' + push_results(file) + file = './speed_summary.stats' + push_results(file) if POST_LINTER_OUTPUT: linter_output = None @@ -102,10 +124,13 @@ if POST_REVIEW: # Actual review is required - pull_request.create_review( - body=comment_body, - event=ACTION - ) + # pull_request.create_review( + # body=comment_body, + # event=ACTION + # ) + print("comment body", comment_body) else: # Just a general info comment - pull_request.create_comment(comment_body) + # pull_request.create_comment(comment_body) + print("COMMENT BODY", comment_body) + diff --git a/ci/prepare-worker.py b/ci/prepare-worker.py index 50f469ed2..ab8679b16 100644 --- a/ci/prepare-worker.py +++ b/ci/prepare-worker.py @@ -23,7 +23,7 @@ (stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.py") (stdin, stdout, stderr) = client.exec_command("sudo rm -rf *config") (stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.sh") -(stdin, stdout, stderr) = client.exec_command("sudo reboot") +# (stdin, stdout, stderr) = client.exec_command("sudo reboot") print("Successfully sent {} to reboot".format(worker_ip)) client.close() diff --git a/ci/speed-tester-analysis.py b/ci/speed-tester-analysis.py index 8d34194f7..9c7df69de 100644 --- a/ci/speed-tester-analysis.py +++ b/ci/speed-tester-analysis.py @@ -41,7 +41,8 @@ def median(array): else: results['pass_performance_check'] = True results['performance_rating'] = performance_rating -results['summary'] = "[Results from %s]\n - Median TX pps for Speed Tester: %d\n - Performance rating - %.2f%% (compared to %d average)" % (STATS_NODE_NAME, median_speed, performance_rating, AVG_SPEED) +results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) +results['summary'] = "\n - Median TX pps for Speed Tester: %d\n - Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) with open(OUT_FILE, 'w') as outfile: json.dump(results, outfile) diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh index ca9f180e2..4e0e95142 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -31,7 +31,7 @@ fi print_header "Collecting Pktgen Statistics" python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE $WORKER_USER # get Pktgen stats from server -scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/stats +scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats print_header "Killing Basic Monitor" sudo pkill -f /basic_monitor diff --git a/ci/worker-files/speed-worker.sh b/ci/worker-files/speed-worker.sh index fbc71f274..bf52f88a8 100755 --- a/ci/worker-files/speed-worker.sh +++ b/ci/worker-files/speed-worker.sh @@ -4,7 +4,7 @@ print_header "Running Speed Tester NF" cd ~/repository/examples/speed_tester -./go.sh 1 -d 1 &>~/spd_stats & +./go.sh 1 -d 1 &>~/speed_stats & spd_tstr_pid=$? if [ $spd_tstr_pid -ne 0 ] then diff --git a/ci/worker-files/worker.sh b/ci/worker-files/worker.sh index 1a45ca976..379249562 100755 --- a/ci/worker-files/worker.sh +++ b/ci/worker-files/worker.sh @@ -35,9 +35,6 @@ fi # source helper functions file . helper-functions.sh -sudo apt-get update -sudo apt-get upgrade -y - sudo apt-get install -y build-essential linux-headers-$(uname -r) git sudo apt-get install -y libnuma1 sudo apt-get install -y libnuma-dev @@ -76,10 +73,11 @@ do # run functionality for each mode case "$mode" in "0") - ~/pktgen-worker.sh $PKT_CONFIG + sudo ~/pktgen-worker.sh $PKT_CONFIG + sudo ~/speed-worker.sh ;; - "1") - ~/speed-worker.sh + "2") + sudo ~/speed-worker.sh ;; *) echo "Mode $MODE has not been implemented" From 709c61bfa03a0b6d9dcad3c4eff4f70c340ff8e3 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Thu, 27 Jun 2019 22:35:47 -0400 Subject: [PATCH 23/42] Fix redundancies and bugs --- ci/.gitignore | 1 + ci/helper-functions.sh | 5 +++++ ci/manager.sh | 30 ++++++++++++++++-------------- ci/post-msg.py | 12 +++++------- ci/prepare-worker.py | 11 ++++------- ci/worker-files/.worker.sh.swp | Bin 12288 -> 0 bytes ci/worker-files/config-test | 1 - ci/worker-files/pkt-config | 3 --- ci/worker-files/worker.sh | 3 --- 9 files changed, 31 insertions(+), 35 deletions(-) delete mode 100644 ci/worker-files/.worker.sh.swp delete mode 100644 ci/worker-files/config-test delete mode 100644 ci/worker-files/pkt-config diff --git a/ci/.gitignore b/ci/.gitignore index 988cfde5e..1bc848e3c 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -12,3 +12,4 @@ linter-output.txt *stats *out* *log* +nimbnode* diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 440beb1e6..8fb5c0ba1 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -145,3 +145,8 @@ run_linter() { fi done } + +fetch_files() { + scp -i $1 -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $2:$3 ./$2.$3 + check_exit_code "ERROR: Failed to fetch results from $2" +} diff --git a/ci/manager.sh b/ci/manager.sh index b626f1b0d..c83575ab7 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -78,7 +78,7 @@ fi print_header "Cleaning up Old Results" sudo rm -f *.txt -sudo rm -rf stats +sudo rm -rf *stats sudo rm -rf repository print_header "Checking Worker and GitHub Creds Exist" @@ -127,7 +127,6 @@ then fi print_header "Preparing Workers" - for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) @@ -157,12 +156,16 @@ do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./repository $worker_ip: - check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" # make sure the config file is updated with the correct run mode sed -i "/MODES*/c\\MODES=\"${RUN_MODE}\"" worker-files/worker-config - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./worker-files/* $worker_ip: + # put all files in one temporary folder for one scp + cp -r ./$worker_ip temp + cp -r ./repository temp/ + cp -r ./worker-files/* temp/ + scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./temp/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" + # get rid of the temp folder now for next worker + sudo rm -rf temp done print_header "Running Workloads on Workers" @@ -176,9 +179,6 @@ do done print_header "Obtaining Performance Results from all workers" - -rm -f *summary.stats - for worker_tuple in "${WORKER_LIST[@]}" do tuple_arr=($worker_tuple) @@ -188,17 +188,19 @@ do if [[ "$RUN_MODE" -eq "0" ]] then # fetch pktgen stats - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:pktgen_stats ./$worker_ip.pktgen_stats - check_exit_code "ERROR: Failed to fetch results from $worker_ip" + fetch_files $worker_key_file $worker_ip pktgen_stats + print_header "getting here" python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats + check_exit_code "Failed to parse Pktgen stats" # fetch speed_tester stats - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:speed_stats ./$worker_ip.speed_stats - check_exit_code "ERROR: Failed to fetch results from $worker_ip" + fetch_files $worker_key_file $worker_ip speed_stats python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats + check_exit_code "Failed to parse Speed Tester stats" else - scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:speed_stats ./$worker_ip.speed_stats - check_exit_code "ERROR: Failed to fetch results from $worker_ip" + # only fetch speed tester stats if mode is not 0 + fetch_files $worker_key_file $worker_ip speed_stats python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats + check_exit_code "Failed to parse Speed Tester stats" fi check_exit_code "ERROR: Failed to analyze results from $worker_ip" done diff --git a/ci/post-msg.py b/ci/post-msg.py index c87f31f32..d054087f4 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -124,13 +124,11 @@ def push_results(file): if POST_REVIEW: # Actual review is required - # pull_request.create_review( - # body=comment_body, - # event=ACTION - # ) - print("comment body", comment_body) + pull_request.create_review( + body=comment_body, + event=ACTION + ) else: # Just a general info comment - # pull_request.create_comment(comment_body) - print("COMMENT BODY", comment_body) + pull_request.create_comment(comment_body) diff --git a/ci/prepare-worker.py b/ci/prepare-worker.py index ab8679b16..a8ba22b0b 100644 --- a/ci/prepare-worker.py +++ b/ci/prepare-worker.py @@ -17,13 +17,10 @@ client.connect(worker_ip, timeout = 30, pkey = key) -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf /mnt/huge/*") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf repository") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *stats*") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.py") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *config") -(stdin, stdout, stderr) = client.exec_command("sudo rm -rf *.sh") -# (stdin, stdout, stderr) = client.exec_command("sudo reboot") +# put all removals in one line to save execution time +wipe_files = "sudo rm -rf /mnt/huge/* repository *stats* *config* *key* *.py *.sh" +(stdin, stdout, stderr) = client.exec_command(wipe_files) +(stdin, stdout, stderr) = client.exec_command("sudo reboot") print("Successfully sent {} to reboot".format(worker_ip)) client.close() diff --git a/ci/worker-files/.worker.sh.swp b/ci/worker-files/.worker.sh.swp deleted file mode 100644 index c18f8b041f3a219cfb81947320882300da170d42..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O^*~s7{?1@;wxe>8iC8Rvl9ZF?VTAGqk*g>;9^!XyJUA+4`egcbXU)m+f&`? zt{!$p5I=xMG-~i5#)Epni$@g?5)VeBAHYP7L{BD0!N`Hce|7f^v#Tr#H6|f3e1*`&A0jq#jz$$R3Dxjkc>?It2W8uK@;=W_z}DZ-UP3M zv)~Lk3HF0Mpa!boQSb=Z2(CZK*w^3-@EQ0NTmWx@vp|CdPzRd<1J^b&_8YhiegT)j z$KV1u2VMqez+SKwTz!DCE8qw4E%*jp1fPRf!7HEzX2Eu_4P3vUu|L5j@Fh48-Ujnv z3XFp>a2L3CA7g)mtKd8E75ET*0Nwzvf#<;jsDm9~JJURD9CfK|XMU=^?m+@1naER}Aw1@}Zur6VHt!??qhD`;LM zI-!GMCkliX9?hk$E0V-dgJ7k^nA;ZaQbR0D-Ei@7Bb6p+XJ=;j(qS0{Qnd(1cndat zj!zf85=&Aqq$Lbexzmapk@T2w+aZ%X7b?LB zmq@sqJ(!n$`}&WCb#u0Iedg&&YV(9tsHrK0B5B;}2nEkd*^-f6{gneoPFIph2+zqs z!!%!B60TCMbd%Z9l5PogV>*?O9G-6+oS8l}^-R9BS}b*W0@2DHoUcSuWrM>Mr;7Yk zxmt#b2U4#P;z5GDZ89_AH8Ve$XDrGcn_H_?Pf?jrAiDzE2SP+tdz=l^2`+VJQ@2Gw zMmi)n><9`KLMIsWWp*4BT&m@dS*VH-&uJcKp;#MZ*i=mVbgUm^a+IJdEwUo46XpGE ztkMdhBvnUiqq|4PX`HIdhpOao+@J8Py^EZL677YlrX5e6u7s-FX(XB>CwDN{!&$qX zH5oFSD%HynDnwW(QxBvv@cQ$6dD=9;HvY%@s_E;Pol-buUIRjPWgIG#X8k&l`EUbc zRGvLL-9Un@@ngWk87 zkw6s51G9U&?Boq7amqVW#gU0lyo*{=uQ^K^wPG|XoMpB7H1fDc-i}dQ1d;h%x+tN< zRO~P%VH%^O<%9kDO2~Xw-Ou(S785#71!YNJlmX!yD242GupH{97^}4NNLKs1sFNld z&891eqpXxKTiHf=o5@PVm@s8>jE2bZ2YUo9E;2KULN%?jDNIMmk~;4!tY2cNG1JxN zXkvm!sv~A$&jX(a$ql~-j-I!~qKfM>NlehQUc%#6nx!ccC_nR1X>c#+h2D192RfH) zIAS5|fRBAKZ?@^dFb%w{3;R-egYpp9h Date: Sat, 29 Jun 2019 16:51:17 -0400 Subject: [PATCH 24/42] Disabled flow lookup and created better benchmarks --- ci/manager.sh | 3 ++- ci/pktgen-analysis.py | 2 +- ci/speed-tester-analysis.py | 2 +- ci/worker-files/helper-functions.sh | 3 +++ ci/worker-files/pktgen-worker.sh | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index c83575ab7..8020972d6 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -113,6 +113,8 @@ check_exit_code "ERROR: Failed to fetch and checkout pull request" print_header "Running Linter" cd repository +# ensure we can copy correct files +chmod -R +rw .git rm -f ../linter-output.txt run_linter ../linter-output.txt cd .. @@ -189,7 +191,6 @@ do then # fetch pktgen stats fetch_files $worker_key_file $worker_ip pktgen_stats - print_header "getting here" python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats check_exit_code "Failed to parse Pktgen stats" # fetch speed_tester stats diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py index 2898761ae..7ee2dd046 100644 --- a/ci/pktgen-analysis.py +++ b/ci/pktgen-analysis.py @@ -2,7 +2,7 @@ import os import json -AVG_SPEED = 6000000 +AVG_SPEED = 10000000 """ get median value of tx pps diff --git a/ci/speed-tester-analysis.py b/ci/speed-tester-analysis.py index 9c7df69de..1ec044ce6 100644 --- a/ci/speed-tester-analysis.py +++ b/ci/speed-tester-analysis.py @@ -3,7 +3,7 @@ import json import re -AVG_SPEED = 35000000 +AVG_SPEED = 37500000 """ get median value of tx pps diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index 4e6c1b404..fcce71b76 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -67,7 +67,10 @@ install_env() { then . ./scripts/install.sh else + # means we're running Pktgen python3 ~/install.py + # disable flow table lookup for faster results + sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile fi } diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh index 4e0e95142..b36ce27ba 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -27,6 +27,9 @@ then return 1 fi +# make sure basic monitor initializes +sleep 5 + # run pktgen print_header "Collecting Pktgen Statistics" python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE $WORKER_USER From e57a6df44eda1e3238cac71f43596ef101b12d46 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 30 Jun 2019 19:50:59 -0400 Subject: [PATCH 25/42] Fixed things so nn30 works with pktgen --- ci/manager.sh | 6 ++++-- ci/pktgen-analysis.py | 5 ++--- ci/speed-tester-analysis.py | 5 ++--- ci/worker-files/helper-functions.sh | 9 ++++++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 8020972d6..2eeacdc5e 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -186,16 +186,18 @@ do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" + # get the benchmarks for each node (some servers are faster) + . ./$worker_ip/benchmarks # TODO: this will overwrite results if we have more than 1 worker, investigate this case if [[ "$RUN_MODE" -eq "0" ]] then # fetch pktgen stats fetch_files $worker_key_file $worker_ip pktgen_stats - python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats + python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats $PKTGEN check_exit_code "Failed to parse Pktgen stats" # fetch speed_tester stats fetch_files $worker_key_file $worker_ip speed_stats - python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $SPEED check_exit_code "Failed to parse Speed Tester stats" else # only fetch speed tester stats if mode is not 0 diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py index 7ee2dd046..e2527d532 100644 --- a/ci/pktgen-analysis.py +++ b/ci/pktgen-analysis.py @@ -2,8 +2,6 @@ import os import json -AVG_SPEED = 10000000 - """ get median value of tx pps """ @@ -15,13 +13,14 @@ def median(array): else: return (array[half - 1] + array[half]) / 2.0 -if(len(sys.argv) != 4): +if(len(sys.argv) != 5): print("ERROR: Invalid arguments.") sys.exit(1) STATS_FILE = sys.argv[1] STATS_NODE_NAME = sys.argv[2] OUT_FILE = sys.argv[3] +AVG_SPEED = int(sys.argv[4]) contents = tuple(open(STATS_FILE, "r")) data = [int(x.rstrip()) for x in contents if x.rstrip()] diff --git a/ci/speed-tester-analysis.py b/ci/speed-tester-analysis.py index 1ec044ce6..f09224d47 100644 --- a/ci/speed-tester-analysis.py +++ b/ci/speed-tester-analysis.py @@ -3,8 +3,6 @@ import json import re -AVG_SPEED = 37500000 - """ get median value of tx pps """ @@ -16,13 +14,14 @@ def median(array): else: return (array[half - 1] + array[half]) / 2.0 -if(len(sys.argv) != 4): +if(len(sys.argv) != 5): print("ERROR: Invalid arguments.") sys.exit(1) STATS_FILE = sys.argv[1] STATS_NODE_NAME = sys.argv[2] OUT_FILE = sys.argv[3] +AVG_SPEED = int(sys.argv[4]) with open(STATS_FILE, "r") as f: contents = f.read() diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index fcce71b76..5b0035b73 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -60,15 +60,18 @@ install_env() { echo $RTE_SDK sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - + cd ../ # check if we need to bind an interface if [[ -z $1 || ! $1 ]] then . ./scripts/install.sh else - # means we're running Pktgen - python3 ~/install.py + # make sure interfaces are accesible by dpdk + sudo ifconfig p2p1 down + sudo ifconfig p2p2 down + # we're running Pktgen + python3 ~/install.py # disable flow table lookup for faster results sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile fi From 6f53017a098f948f36a6304f45de59f3aab7cc04 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 3 Jul 2019 23:25:45 -0400 Subject: [PATCH 26/42] Fixes from github comments --- ci/helper-functions.sh | 45 +------------------ ...gen-config.lua => pktgen-timed-config.lua} | 0 ci/install-pktgen/start-pktgen.sh | 9 ++-- ci/manager.sh | 4 +- ci/worker-files/helper-functions.sh | 6 --- 5 files changed, 7 insertions(+), 57 deletions(-) rename ci/install-pktgen/{pktgen-config.lua => pktgen-timed-config.lua} (100%) diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 8fb5c0ba1..7e1732605 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -38,50 +38,6 @@ print_header() { echo "" } - # sets up dpdk, sets env variables, and runs the install script -install_env() { - git submodule sync - git submodule update --init - - echo export ONVM_HOME=$(pwd) >> ~/.bashrc - export ONVM_HOME=$(pwd) - - cd dpdk - - echo export RTE_SDK=$(pwd) >> ~/.bashrc - export RTE_SDK=$(pwd) - - echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc - export RTE_TARGET=x86_64-native-linuxapp-gcc - - echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc - export ONVM_NUM_HUGEPAGES=1024 - - echo $RTE_SDK - - sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - - cd ../ - # check if we need to bind an interface - if [[ -z $1 || ! $1 ]] - then - . ./scripts/install.sh - else - python3 ~/install.py - fi -} - -# makes all onvm code -build_onvm() { - cd onvm - make clean && make - cd ../ - - cd examples - make clean && make - cd ../ -} - # obtains core config in cores.out file obtain_core_config() { cd scripts @@ -147,6 +103,7 @@ run_linter() { } fetch_files() { + # inputs are key_file, worker ip address, stats file - in that order 1,2,3 scp -i $1 -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $2:$3 ./$2.$3 check_exit_code "ERROR: Failed to fetch results from $2" } diff --git a/ci/install-pktgen/pktgen-config.lua b/ci/install-pktgen/pktgen-timed-config.lua similarity index 100% rename from ci/install-pktgen/pktgen-config.lua rename to ci/install-pktgen/pktgen-timed-config.lua diff --git a/ci/install-pktgen/start-pktgen.sh b/ci/install-pktgen/start-pktgen.sh index 30235c9cd..e33a2304c 100755 --- a/ci/install-pktgen/start-pktgen.sh +++ b/ci/install-pktgen/start-pktgen.sh @@ -9,12 +9,15 @@ sudo rm -rf repository git clone https://github.com/sdnfv/openNetVM.git repository check_exit_code "ERROR: Failed installing onvm" +print_header "Installing Dependencies" sudo apt-get update sudo apt-get upgrade -y sudo apt-get install -y build-essential linux-headers-$(uname -r) git sudo apt-get install -y libnuma1 sudo apt-get install -y libnuma-dev +sudo apt-get install libpcap-dev +sudo apt-get install libreadline-dev cd repository @@ -30,10 +33,6 @@ check_exit_code "ERROR: Building ONVM failed" cd ~ -# install dependencies -sudo apt-get install libpcap-dev -sudo apt-get install libreadline-dev - print_header "Installing Lua" curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz tar zxf lua-5.3.5.tar.gz @@ -45,6 +44,6 @@ cd ~/repository/tools/Pktgen/pktgen-dpdk/ make print_header "Updating lua script" -cp ~/pktgen-config.lua ~/repository/tools/Pktgen/openNetVM-Scripts/ +cp ~/pktgen-timed-config.lua ~/repository/tools/Pktgen/openNetVM-Scripts/pktgen-config.lua print_header "Pktgen installed" diff --git a/ci/manager.sh b/ci/manager.sh index 2eeacdc5e..00da996f0 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -193,11 +193,11 @@ do then # fetch pktgen stats fetch_files $worker_key_file $worker_ip pktgen_stats - python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats $PKTGEN + python3 pktgen-analysis.py ./$worker_ip.pktgen_stats $worker_ip pktgen_summary.stats $AVG_PKTGEN_SPEED check_exit_code "Failed to parse Pktgen stats" # fetch speed_tester stats fetch_files $worker_key_file $worker_ip speed_stats - python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $SPEED + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $AVG_SPEED_TESTER_SPEED check_exit_code "Failed to parse Speed Tester stats" else # only fetch speed tester stats if mode is not 0 diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index 5b0035b73..4a0f37b3c 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -88,12 +88,6 @@ build_onvm() { cd ../ } -# obtains core config in cores.out file -obtain_core_config() { - cd scripts - ./corehelper.py > cores.out -} - # checks if a command has failed (exited with code != 0) # if it does, print the error message, exit the build, and post to github check_exit_code() { From b270224d8051d348ac594b1795018496910691e5 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sat, 6 Jul 2019 13:07:58 -0400 Subject: [PATCH 27/42] Make results more stable by restarting manager --- ci/install-pktgen/helper-functions.sh | 6 ----- ci/worker-files/pktgen-worker.sh | 29 ++++++++++++++++++----- ci/worker-files/speed-worker.sh | 24 ++++++++++++++++--- ci/worker-files/worker.sh | 34 ++++++--------------------- 4 files changed, 51 insertions(+), 42 deletions(-) diff --git a/ci/install-pktgen/helper-functions.sh b/ci/install-pktgen/helper-functions.sh index 4e6c1b404..d6d1fd79b 100644 --- a/ci/install-pktgen/helper-functions.sh +++ b/ci/install-pktgen/helper-functions.sh @@ -82,12 +82,6 @@ build_onvm() { cd ../ } -# obtains core config in cores.out file -obtain_core_config() { - cd scripts - ./corehelper.py > cores.out -} - # checks if a command has failed (exited with code != 0) # if it does, print the error message, exit the build, and post to github check_exit_code() { diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh index b36ce27ba..ad6a3f199 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -1,8 +1,5 @@ #!/bin/bash -cd ~ -. helper-functions.sh - if [[ -z $1 ]] then echo "ERROR: Missing first argument, path to pktgen config file" @@ -17,11 +14,25 @@ fi . $1 # source the variables from pktgen config file +print_header "Running ONVM Manager" +cd ~/repository/onvm +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & +mgr_pid=0 +if [ $mgr_pid -ne 0 ] +then + echo "ERROR: Starting manager failed" + return 1 +fi + +# wait for the manager to come online +sleep 15 +print_header "Manager is live" + print_header "Running Basic Monitor NF" -cd repository/examples/basic_monitor +cd ~/repository/examples/basic_monitor ./go.sh 1 &>~/bsc_stats & -bsc_mntr_pid=$? -if [ $bsc_mntr_pid -ne 0 ] +bsc_mntr_pid=0 +if [ $bsc_mntr_pid -ne 0 ] then echo "ERROR: Starting basic monitor failed" return 1 @@ -38,3 +49,9 @@ scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev print_header "Killing Basic Monitor" sudo pkill -f /basic_monitor + +print_header "Exiting ONVM" + +echo "Manager pid: ${mgr_pid}" +sudo pkill -f onvm_mgr +check_exit_code "ERROR: Killing manager failed" diff --git a/ci/worker-files/speed-worker.sh b/ci/worker-files/speed-worker.sh index bf52f88a8..b4d37afae 100755 --- a/ci/worker-files/speed-worker.sh +++ b/ci/worker-files/speed-worker.sh @@ -1,15 +1,27 @@ #!/bin/bash -. ~/helper-functions.sh +print_header "Running ONVM Manager" +cd ~/repository/onvm +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & +mgr_pid=$? +if [ $mgr_pid -ne 0 ] +then + echo "ERROR: Starting manager failed" + exit 1 +fi + +# wait for the manager to come online +sleep 15 +print_header "Manager is live" print_header "Running Speed Tester NF" cd ~/repository/examples/speed_tester ./go.sh 1 -d 1 &>~/speed_stats & spd_tstr_pid=$? -if [ $spd_tstr_pid -ne 0 ] +if [ $spd_tstr_pid -ne 0 ] then echo "ERROR: Starting speed tester failed" - # exit 1 + return 1 fi # wait for speed tester to come online @@ -18,3 +30,9 @@ sleep 15 print_header "Killing Speed Tester" sudo pkill -f /speed_tester + +print_header "Exiting Manager" + +echo "Manager pid: ${mgr_pid}" +sudo pkill -f onvm_mgr +check_exit_code "ERROR: Killing manager failed" diff --git a/ci/worker-files/worker.sh b/ci/worker-files/worker.sh index 1b6dffc48..41cb2ba1f 100755 --- a/ci/worker-files/worker.sh +++ b/ci/worker-files/worker.sh @@ -51,41 +51,21 @@ print_header "Building ONVM" build_onvm check_exit_code "ERROR: Building ONVM failed" -print_header "Running ONVM Manager" -cd onvm -./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & -mgr_pid=$? -if [ $mgr_pid -ne 0 ] -then - echo "ERROR: Starting manager failed" - exit 1 -fi - -# wait for the manager to come online -sleep 15 -print_header "Manager is live" - for mode in $MODES do # run functionality for each mode case "$mode" in "0") - sudo ~/pktgen-worker.sh $PKT_CONFIG - sudo ~/speed-worker.sh - ;; + . ~/pktgen-worker.sh $PKT_CONFIG + . ~/speed-worker.sh + ;; "2") - sudo ~/speed-worker.sh - ;; - *) + . ~/speed-worker.sh + ;; + *) echo "Mode $MODE has not been implemented" - ;; + ;; esac done -print_header "Exiting ONVM" - -echo "Manager pid: ${mgr_pid}" -sudo kill $mgr_pid -check_exit_code "ERROR: Killing manager failed" - print_header "Performance Tests Completed Successfully" From 5fa102d4cf250aa8f579ba4e14c33b729073997d Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 7 Jul 2019 17:47:39 -0400 Subject: [PATCH 28/42] Added fixes from review and redundancies --- ci/install-pktgen/helper-functions.sh | 95 +-------------------------- ci/install-pktgen/start-pktgen.sh | 20 ++---- ci/pktgen-analysis.py | 2 +- ci/post-msg.py | 4 +- ci/speed-tester-analysis.py | 2 +- ci/worker-files/helper-functions.sh | 49 +++++--------- ci/worker-files/pktgen-worker.sh | 16 ++--- ci/worker-files/speed-worker.sh | 14 ++-- ci/worker-files/worker.sh | 9 ++- 9 files changed, 49 insertions(+), 162 deletions(-) mode change 100644 => 120000 ci/install-pktgen/helper-functions.sh diff --git a/ci/install-pktgen/helper-functions.sh b/ci/install-pktgen/helper-functions.sh deleted file mode 100644 index d6d1fd79b..000000000 --- a/ci/install-pktgen/helper-functions.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -# simple print function that takes a single argument -# prints the argument as a header wrapped in dashes and pipes -print_header() { - # check that the argument exists - if [ -z "$1" ] - then - echo "No argument supplied to print_header!" - return 1 - fi - - # get the string length for formatting - strlen=${#1} - - echo "" - - # echo first line of dashes - echo -n "--" - for i in `eval echo {1..$strlen}` - do - echo -n "-" - done - echo "--" - - # echo the argument - echo -n "| " - echo -n $1 - echo " |" - - # echo the second line of dashes - echo -n "--" - for i in `eval echo {1..$strlen}` - do - echo -n "-" - done - echo "--" - echo "" -} - - # sets up dpdk, sets env variables, and runs the install script -install_env() { - git submodule sync - git submodule update --init - - echo export ONVM_HOME=$(pwd) >> ~/.bashrc - export ONVM_HOME=$(pwd) - - cd dpdk - - echo export RTE_SDK=$(pwd) >> ~/.bashrc - export RTE_SDK=$(pwd) - - echo export RTE_TARGET=x86_64-native-linuxapp-gcc >> ~/.bashrc - export RTE_TARGET=x86_64-native-linuxapp-gcc - - echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc - export ONVM_NUM_HUGEPAGES=1024 - - echo $RTE_SDK - - sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - - cd ../ - # check if we need to bind an interface - if [[ -z $1 || ! $1 ]] - then - . ./scripts/install.sh - else - python3 ~/install.py - fi -} - -# makes all onvm code -build_onvm() { - cd onvm - make clean && make - cd ../ - - cd examples - make clean && make - cd ../ -} - -# checks if a command has failed (exited with code != 0) -# if it does, print the error message, exit the build, and post to github -check_exit_code() { - if [ $? -ne 0 ] - then - echo $1 - cd $SCRIPT_LOC - exit 1 - fi -} diff --git a/ci/install-pktgen/helper-functions.sh b/ci/install-pktgen/helper-functions.sh new file mode 120000 index 000000000..d61897ac9 --- /dev/null +++ b/ci/install-pktgen/helper-functions.sh @@ -0,0 +1 @@ +../worker-files/helper-functions.sh \ No newline at end of file diff --git a/ci/install-pktgen/start-pktgen.sh b/ci/install-pktgen/start-pktgen.sh index e33a2304c..b345c798f 100755 --- a/ci/install-pktgen/start-pktgen.sh +++ b/ci/install-pktgen/start-pktgen.sh @@ -12,13 +12,19 @@ check_exit_code "ERROR: Failed installing onvm" print_header "Installing Dependencies" sudo apt-get update sudo apt-get upgrade -y - sudo apt-get install -y build-essential linux-headers-$(uname -r) git sudo apt-get install -y libnuma1 sudo apt-get install -y libnuma-dev sudo apt-get install libpcap-dev sudo apt-get install libreadline-dev +print_header "Installing Lua" +curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz +tar zxf lua-5.3.5.tar.gz +cd lua-5.3.5 +sudo make linux test +sudo make install + cd repository print_header "Beginning Execution of Workload" @@ -27,18 +33,6 @@ print_header "Installing Environment" install_env $RUN_PKT check_exit_code "ERROR: Installing environment failed" -print_header "Building ONVM" -build_onvm -check_exit_code "ERROR: Building ONVM failed" - -cd ~ - -print_header "Installing Lua" -curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz -tar zxf lua-5.3.5.tar.gz -cd lua-5.3.5 -sudo make install - print_header "Make pktgen-dpdk" cd ~/repository/tools/Pktgen/pktgen-dpdk/ make diff --git a/ci/pktgen-analysis.py b/ci/pktgen-analysis.py index e2527d532..9dfb6ffa0 100644 --- a/ci/pktgen-analysis.py +++ b/ci/pktgen-analysis.py @@ -38,7 +38,7 @@ def median(array): results['pass_performance_check'] = True results['performance_rating'] = performance_rating results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) -results['summary'] = "\n - Median TX pps for Pktgen: %d\n - Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) +results['summary'] = "\n - Median TX pps for Pktgen: %d\n Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) with open(OUT_FILE, 'w') as outfile: json.dump(results, outfile) diff --git a/ci/post-msg.py b/ci/post-msg.py index d054087f4..03350cc56 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -33,9 +33,9 @@ def push_results(file): with open(file) as f: results = json.load(f) if previous_results_from != results['results_from']: - comment_body += "\n " + results['results_from'] + comment_body += "\n" + results['results_from'] previous_results_from = results['results_from'] - comment_body += "\n " + results['summary'] + comment_body += "\n" + results['summary'] with open(sys.argv[1], "r") as credsfile: creds = [x.strip() for x in credsfile.readlines()] diff --git a/ci/speed-tester-analysis.py b/ci/speed-tester-analysis.py index f09224d47..a18c82bd4 100644 --- a/ci/speed-tester-analysis.py +++ b/ci/speed-tester-analysis.py @@ -41,7 +41,7 @@ def median(array): results['pass_performance_check'] = True results['performance_rating'] = performance_rating results['results_from'] = "[Results from %s]" % (STATS_NODE_NAME) -results['summary'] = "\n - Median TX pps for Speed Tester: %d\n - Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) +results['summary'] = "\n - Median TX pps for Speed Tester: %d\n Performance rating - %.2f%% (compared to %d average)" % (median_speed, performance_rating, AVG_SPEED) with open(OUT_FILE, 'w') as outfile: json.dump(results, outfile) diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index 4a0f37b3c..c48d4c026 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -1,43 +1,30 @@ #!/bin/bash -# simple print function that takes a single argument -# prints the argument as a header wrapped in dashes and pipes -print_header() { +LOGFILE=worker.log +# remove previous log if it's over 300 lines +RETAIN_NUM_LINES=300 + +# print to stdout and the logfile +logsetup() { + TMP=$(tail -n $RETAIN_NUM_LINES $LOGFILE 2>/dev/null) && echo "${TMP}" > $LOGFILE + exec > >(tee -a $LOGFILE) + exec 2>&1 +} + +log() { # check that the argument exists if [ -z "$1" ] then - echo "No argument supplied to print_header!" + echo "No argument supplied to log!" return 1 fi - - # get the string length for formatting - strlen=${#1} - - echo "" - - # echo first line of dashes - echo -n "--" - for i in `eval echo {1..$strlen}` - do - echo -n "-" - done - echo "--" - - # echo the argument - echo -n "| " - echo -n $1 - echo " |" - - # echo the second line of dashes - echo -n "--" - for i in `eval echo {1..$strlen}` - do - echo -n "-" - done - echo "--" - echo "" + + echo "[$(date --rfc-3339=seconds)]: $*" } +# set up our log file +logsetup + # sets up dpdk, sets env variables, and runs the install script install_env() { git submodule sync diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh index ad6a3f199..4bbdd2243 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -14,9 +14,9 @@ fi . $1 # source the variables from pktgen config file -print_header "Running ONVM Manager" +log "Running ONVM Manager" cd ~/repository/onvm -./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_pktgen_stats & mgr_pid=0 if [ $mgr_pid -ne 0 ] then @@ -26,9 +26,9 @@ fi # wait for the manager to come online sleep 15 -print_header "Manager is live" +log "Manager is live" -print_header "Running Basic Monitor NF" +log "Running Basic Monitor NF" cd ~/repository/examples/basic_monitor ./go.sh 1 &>~/bsc_stats & bsc_mntr_pid=0 @@ -39,18 +39,18 @@ then fi # make sure basic monitor initializes -sleep 5 +sleep 10 # run pktgen -print_header "Collecting Pktgen Statistics" +log "Collecting Pktgen Statistics" python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE $WORKER_USER # get Pktgen stats from server scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats -print_header "Killing Basic Monitor" +log "Killing Basic Monitor" sudo pkill -f /basic_monitor -print_header "Exiting ONVM" +log "Exiting ONVM" echo "Manager pid: ${mgr_pid}" sudo pkill -f onvm_mgr diff --git a/ci/worker-files/speed-worker.sh b/ci/worker-files/speed-worker.sh index b4d37afae..a849ac5da 100755 --- a/ci/worker-files/speed-worker.sh +++ b/ci/worker-files/speed-worker.sh @@ -1,8 +1,8 @@ #!/bin/bash -print_header "Running ONVM Manager" +log "Running ONVM Manager" cd ~/repository/onvm -./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_stats & +./go.sh 0,1,2,3 3 0xF0 -a 0x7f000000000 -s stdout &>~/onvm_speed_stats & mgr_pid=$? if [ $mgr_pid -ne 0 ] then @@ -12,9 +12,9 @@ fi # wait for the manager to come online sleep 15 -print_header "Manager is live" +log "Manager is live" -print_header "Running Speed Tester NF" +log "Running Speed Tester NF" cd ~/repository/examples/speed_tester ./go.sh 1 -d 1 &>~/speed_stats & spd_tstr_pid=$? @@ -25,13 +25,13 @@ then fi # wait for speed tester to come online -print_header "Collecting Speed Tester Statistics" +log "Collecting Speed Tester Statistics" sleep 15 -print_header "Killing Speed Tester" +log "Killing Speed Tester" sudo pkill -f /speed_tester -print_header "Exiting Manager" +log "Exiting Manager" echo "Manager pid: ${mgr_pid}" sudo pkill -f onvm_mgr diff --git a/ci/worker-files/worker.sh b/ci/worker-files/worker.sh index 41cb2ba1f..e20cae434 100755 --- a/ci/worker-files/worker.sh +++ b/ci/worker-files/worker.sh @@ -40,14 +40,13 @@ sudo apt-get install -y libnuma1 sudo apt-get install -y libnuma-dev cd repository +log "Beginning Execution of Workload" -print_header "Beginning Execution of Workload" - -print_header "Installing Environment" +log "Installing Environment" install_env $RUN_PKT check_exit_code "ERROR: Installing environment failed" -print_header "Building ONVM" +log "Building ONVM" build_onvm check_exit_code "ERROR: Building ONVM failed" @@ -68,4 +67,4 @@ do esac done -print_header "Performance Tests Completed Successfully" +log "Performance Tests Completed Successfully" From 8c48dd6de6bd4fe6db84ecb65ccbe82959a89914 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 9 Jul 2019 19:55:03 -0400 Subject: [PATCH 29/42] Fixes from github review --- ci/helper-functions.sh | 2 +- .../{start-pktgen.sh => install-pktgen.sh} | 0 ci/post-msg.py | 15 ++++++++------- .../{speed-worker.sh => speed-test-worker.sh} | 0 ci/worker-files/worker.sh | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) rename ci/install-pktgen/{start-pktgen.sh => install-pktgen.sh} (100%) rename ci/worker-files/{speed-worker.sh => speed-test-worker.sh} (100%) diff --git a/ci/helper-functions.sh b/ci/helper-functions.sh index 7e1732605..c60d8fb8f 100644 --- a/ci/helper-functions.sh +++ b/ci/helper-functions.sh @@ -102,8 +102,8 @@ run_linter() { done } +# inputs are key_file, worker ip address, stats file - in that order 1,2,3 fetch_files() { - # inputs are key_file, worker ip address, stats file - in that order 1,2,3 scp -i $1 -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $2:$3 ./$2.$3 check_exit_code "ERROR: Failed to fetch results from $2" } diff --git a/ci/install-pktgen/start-pktgen.sh b/ci/install-pktgen/install-pktgen.sh similarity index 100% rename from ci/install-pktgen/start-pktgen.sh rename to ci/install-pktgen/install-pktgen.sh diff --git a/ci/post-msg.py b/ci/post-msg.py index 03350cc56..f105ec514 100644 --- a/ci/post-msg.py +++ b/ci/post-msg.py @@ -15,7 +15,8 @@ print("ERROR! Incorrect number of arguments") sys.exit(1) -def apply_stats(file, name): +# add results of statistics file to Github comments +def process_results_from_worker(file, name): global comment_body global ACTION if path.exists(file): @@ -27,7 +28,8 @@ def apply_stats(file, name): comment_body += " :x: PR drops {} performance below minimum requirement\n".format(name) ACTION = 'REQUEST_CHANGES' -def push_results(file): +# workers have many stats files, make sure its name only displays once +def add_results_from_worker(file): global comment_body global previous_results_from with open(file) as f: @@ -87,9 +89,9 @@ def push_results(file): # PR must not affect performance if POST_RESULTS: file = './pktgen_summary.stats' - apply_stats(file, "Pktgen") + process_results_from_worker(file, "Pktgen") file = './speed_summary.stats' - apply_stats(file, "Speed Test") + process_results_from_worker(file, "Speed Test") # PR must pass linter check linter_output = None @@ -107,9 +109,9 @@ def push_results(file): if POST_RESULTS: file = './pktgen_summary.stats' - push_results(file) + add_results_from_worker(file) file = './speed_summary.stats' - push_results(file) + add_results_from_worker(file) if POST_LINTER_OUTPUT: linter_output = None @@ -131,4 +133,3 @@ def push_results(file): else: # Just a general info comment pull_request.create_comment(comment_body) - diff --git a/ci/worker-files/speed-worker.sh b/ci/worker-files/speed-test-worker.sh similarity index 100% rename from ci/worker-files/speed-worker.sh rename to ci/worker-files/speed-test-worker.sh diff --git a/ci/worker-files/worker.sh b/ci/worker-files/worker.sh index e20cae434..771738a7f 100755 --- a/ci/worker-files/worker.sh +++ b/ci/worker-files/worker.sh @@ -56,10 +56,10 @@ do case "$mode" in "0") . ~/pktgen-worker.sh $PKT_CONFIG - . ~/speed-worker.sh + . ~/speed-test-worker.sh ;; "2") - . ~/speed-worker.sh + . ~/speed-test-worker.sh ;; *) echo "Mode $MODE has not been implemented" From cbd1a6165c5ecfa7dfc44900b7cf9c9d46464578 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Tue, 9 Jul 2019 19:58:15 -0400 Subject: [PATCH 30/42] Additions to readme about CI developments --- ci/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/README.md b/ci/README.md index a84513c52..b30a867ac 100644 --- a/ci/README.md +++ b/ci/README.md @@ -83,11 +83,14 @@ The CI process can be broken into multiple steps: Use paramiko to ssh and run `run-workload.py` -9. Acquire results from the worker nodes +9. Run modes are supplied to tell the worker which applications to test + `worker-files/worker.sh` handles installation, builds, and setting up manager for performance tests + +10. Acquire results from the worker nodes Use scp to copy the result stat file from worker -10. Submit results as a comment on github +11. Submit results as a comment on github Uses the `post-msg.py` script @@ -106,6 +109,9 @@ ProxyPassReverse /onvm-ci/ http://nimbnode44:8080/ ``` (Also need to setup github webhook to post to **http://nimbus.seas.gwu.edu/onvm-ci/github-webhook**) +### Public and Private CI Runs +CI is now able to accept requests from unauthenticated users. There is a list of Github users in the public project allowed to create a full run. Anyone who is able to view the private `-dev` repository is able to run CI there as well. In `openNetVM`, if a user is not in our list, the linter and branch checks will be executed, ignoring statistics calculations from the worker nodes. + ### Checking if Online If you are worried if the CI build is offline or want to make sure it is listening for events, you can check the following url: curl http://nimbus.seas.gwu.edu/onvm-ci/status. If that URL returns 404, CI is offline. Otherwise it will display a message saying it is online. From eb39bc126f2b949860ed5d2b3f9db35b27df28f4 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 10 Jul 2019 19:46:16 -0400 Subject: [PATCH 31/42] Add a section about Pktgen helper nodes --- ci/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/README.md b/ci/README.md index b30a867ac..ef725b9ca 100644 --- a/ci/README.md +++ b/ci/README.md @@ -84,7 +84,8 @@ The CI process can be broken into multiple steps: Use paramiko to ssh and run `run-workload.py` 9. Run modes are supplied to tell the worker which applications to test - `worker-files/worker.sh` handles installation, builds, and setting up manager for performance tests + + Handle installation with `worker-files/worker.sh` for builds, and setting up manager for performance tests 10. Acquire results from the worker nodes @@ -110,8 +111,20 @@ ProxyPassReverse /onvm-ci/ http://nimbnode44:8080/ (Also need to setup github webhook to post to **http://nimbus.seas.gwu.edu/onvm-ci/github-webhook**) ### Public and Private CI Runs + CI is now able to accept requests from unauthenticated users. There is a list of Github users in the public project allowed to create a full run. Anyone who is able to view the private `-dev` repository is able to run CI there as well. In `openNetVM`, if a user is not in our list, the linter and branch checks will be executed, ignoring statistics calculations from the worker nodes. +### Setting Up a Connected Worker + +Connecting two nodes is useful for measuring statistics with tools like Pktgen and the MTCP stack. There is a bit of setup required to get working connection working. Firstly, an SFP+ 10Gb Intel cable will be required to connect the Network Interface Cards in the two machines. Once this is done, attempt to bring up the correct interfaces for a stable connection. Some debugging might be required: +- If you don't know which `ifconfig -a` interface is correct, use `ethtool -p 120` + - This will blink a light on the interface (you have to be next to the machine for this to help) +- Do this on both machines, to find the name of the interfaces that are linked +- Run `sudo ifconfig 11.0.0.1/24 up` on the first machine and `sudo ifconfig 11.0.0.2/24 up` + - This will ensure `ping` understands what IP address it is supposed to talk to +- If `ping -I 11.0.0.2>` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` +Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install-pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! + ### Checking if Online If you are worried if the CI build is offline or want to make sure it is listening for events, you can check the following url: curl http://nimbus.seas.gwu.edu/onvm-ci/status. If that URL returns 404, CI is offline. Otherwise it will display a message saying it is online. From 93b94eea7042b5ac8fe289e085a6df05510f9583 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Wed, 10 Jul 2019 19:53:15 -0400 Subject: [PATCH 32/42] Fix missed line --- ci/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/README.md b/ci/README.md index ef725b9ca..556d0656b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -122,7 +122,8 @@ Connecting two nodes is useful for measuring statistics with tools like Pktgen a - Do this on both machines, to find the name of the interfaces that are linked - Run `sudo ifconfig 11.0.0.1/24 up` on the first machine and `sudo ifconfig 11.0.0.2/24 up` - This will ensure `ping` understands what IP address it is supposed to talk to -- If `ping -I 11.0.0.2>` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` +- If `ping -I 11.0.0.2>` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` + Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install-pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! ### Checking if Online From 5a652cbee9b92197e9420c540ffe3cee68944eb6 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sat, 13 Jul 2019 18:28:54 -0400 Subject: [PATCH 33/42] Add compatability for different interfaces --- ci/manager.sh | 2 -- ci/worker-files/helper-functions.sh | 25 +++++++++++++++++++++---- ci/worker-files/run-pktgen.py | 6 ++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 00da996f0..20b49b64b 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -113,8 +113,6 @@ check_exit_code "ERROR: Failed to fetch and checkout pull request" print_header "Running Linter" cd repository -# ensure we can copy correct files -chmod -R +rw .git rm -f ../linter-output.txt run_linter ../linter-output.txt cd .. diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index c48d4c026..f882cc731 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -54,11 +54,28 @@ install_env() { then . ./scripts/install.sh else + # we're running Pktgen, so dpdk interfaces are necessary # make sure interfaces are accesible by dpdk - sudo ifconfig p2p1 down - sudo ifconfig p2p2 down - # we're running Pktgen - python3 ~/install.py + pci_addresses="" + interfaces=$(python3 $RTE_SDK/usertools/dpdk-devbind.py --status | grep -P "10[-G](igabit)?") + # bring down correct interfaces and set them for binding + for id in $interfaces + do + if [[ $id =~ 0000* ]]; + then + # setup_environment (line 108) uses this to bind interfaces without user input + pci_addresses="$pci_addresses $id " + elif [[ $id =~ if=* ]]; + then + # name of interface to bring down (dpdk needs it inactive) + sudo ifconfig $(echo $id | cut -f 2 -d "=") down + fi + done + + echo export ONVM_NIC_PCI="\"$pci_addresses\"" >> ~/.bashrc + export ONVM_NIC_PCI="\"$pci_addresses\"" + + . ./scripts/install.sh # disable flow table lookup for faster results sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile fi diff --git a/ci/worker-files/run-pktgen.py b/ci/worker-files/run-pktgen.py index 63bdcdaed..70484cb57 100644 --- a/ci/worker-files/run-pktgen.py +++ b/ci/worker-files/run-pktgen.py @@ -2,6 +2,7 @@ from paramiko import SSHClient from paramiko import AutoAddPolicy import sys +import datetime if len(sys.argv) != 4: print("ERROR! Incorrect number of arguments") @@ -21,6 +22,11 @@ # block until script finishes exit_status = stdout.channel.recv_exit_status() +# write to file for debugging +with open('paramiko_pktgen_out.log', 'a+') as paramiko_out: + paramiko_out.write(datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y") + "\n") + paramiko_out.write(stdout.read().decode('ascii') + "\n") + print("Successfully ran pktgen".format(worker_ip)) client.close() From d1e83bde6d1e1934abc03829d29e6be118dccc73 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sun, 14 Jul 2019 11:40:13 -0400 Subject: [PATCH 34/42] Fixes for user input and debugging --- ci/worker-files/helper-functions.sh | 4 ++-- ci/worker-files/run-pktgen.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index f882cc731..5f2d06fe2 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -72,8 +72,8 @@ install_env() { fi done - echo export ONVM_NIC_PCI="\"$pci_addresses\"" >> ~/.bashrc - export ONVM_NIC_PCI="\"$pci_addresses\"" + echo export ONVM_NIC_PCI=\"$pci_addresses\" >> ~/.bashrc + export ONVM_NIC_PCI="$pci_addresses" . ./scripts/install.sh # disable flow table lookup for faster results diff --git a/ci/worker-files/run-pktgen.py b/ci/worker-files/run-pktgen.py index 70484cb57..98268e9bd 100644 --- a/ci/worker-files/run-pktgen.py +++ b/ci/worker-files/run-pktgen.py @@ -23,7 +23,7 @@ exit_status = stdout.channel.recv_exit_status() # write to file for debugging -with open('paramiko_pktgen_out.log', 'a+') as paramiko_out: +with open('/home/' + worker_user + 'paramiko_pktgen_out.log', 'a+') as paramiko_out: paramiko_out.write(datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y") + "\n") paramiko_out.write(stdout.read().decode('ascii') + "\n") From 4f8fc06049402051065c2e117496b5105399505c Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Thu, 25 Jul 2019 19:02:45 -0400 Subject: [PATCH 35/42] Fixes for Pktgen and binding interface --- ci/manager.sh | 6 ++-- ci/run-workload.py | 1 + ci/worker-files/helper-functions.sh | 56 ++++++++++++++--------------- ci/worker-files/pktgen-worker.sh | 31 +++++++--------- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 20b49b64b..328a5a1aa 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -157,7 +157,7 @@ do worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" # make sure the config file is updated with the correct run mode - sed -i "/MODES*/c\\MODES=\"${RUN_MODE}\"" worker-files/worker-config + sed -i "/WORKER_MODE*/c\\WORKER_MODE=\"${RUN_MODE}\"" worker-files/worker-config # put all files in one temporary folder for one scp cp -r ./$worker_ip temp cp -r ./repository temp/ @@ -187,7 +187,7 @@ do # get the benchmarks for each node (some servers are faster) . ./$worker_ip/benchmarks # TODO: this will overwrite results if we have more than 1 worker, investigate this case - if [[ "$RUN_MODE" -eq "0" ]] + if [[ "$RUN_MODE" -eq "3" ]] then # fetch pktgen stats fetch_files $worker_key_file $worker_ip pktgen_stats @@ -200,7 +200,7 @@ do else # only fetch speed tester stats if mode is not 0 fetch_files $worker_key_file $worker_ip speed_stats - python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats + python3 speed-tester-analysis.py ./$worker_ip.speed_stats $worker_ip speed_summary.stats $AVG_SPEED_TESTER_SPEED check_exit_code "Failed to parse Speed Tester stats" fi check_exit_code "ERROR: Failed to analyze results from $worker_ip" diff --git a/ci/run-workload.py b/ci/run-workload.py index 31a880da0..4b81d8cf4 100644 --- a/ci/run-workload.py +++ b/ci/run-workload.py @@ -4,6 +4,7 @@ from paramiko import SSHClient from paramiko import AutoAddPolicy import sys +import datetime """ get buffered stream of input from the client channel while the worker.sh script executes diff --git a/ci/worker-files/helper-functions.sh b/ci/worker-files/helper-functions.sh index 5f2d06fe2..8c9917c1e 100644 --- a/ci/worker-files/helper-functions.sh +++ b/ci/worker-files/helper-functions.sh @@ -3,6 +3,7 @@ LOGFILE=worker.log # remove previous log if it's over 300 lines RETAIN_NUM_LINES=300 +DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py # print to stdout and the logfile logsetup() { @@ -25,6 +26,13 @@ log() { # set up our log file logsetup +# given a 10G NIC interface, bring down and bind to dpdk +bind_nic_from_iface() { + sudo ifconfig $1 down + id=$($DPDK_DEVBIND --status | grep -e "if=$1" | cut -f 1 -d " ") + sudo $DPDK_DEVBIND -b igb_uio $id +} + # sets up dpdk, sets env variables, and runs the install script install_env() { git submodule sync @@ -44,38 +52,30 @@ install_env() { echo export ONVM_NUM_HUGEPAGES=1024 >> ~/.bashrc export ONVM_NUM_HUGEPAGES=1024 - echo $RTE_SDK - sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - - cd ../ - # check if we need to bind an interface - if [[ -z $1 || ! $1 ]] + + if [[ ! -z $MTCP_IFACE ]] then - . ./scripts/install.sh - else - # we're running Pktgen, so dpdk interfaces are necessary - # make sure interfaces are accesible by dpdk - pci_addresses="" - interfaces=$(python3 $RTE_SDK/usertools/dpdk-devbind.py --status | grep -P "10[-G](igabit)?") - # bring down correct interfaces and set them for binding - for id in $interfaces - do - if [[ $id =~ 0000* ]]; - then - # setup_environment (line 108) uses this to bind interfaces without user input - pci_addresses="$pci_addresses $id " - elif [[ $id =~ if=* ]]; - then - # name of interface to bring down (dpdk needs it inactive) - sudo ifconfig $(echo $id | cut -f 2 -d "=") down - fi - done + # bring iface up so onvm install can't bind it + sudo ifconfig $MTCP_IFACE 11.0.0.17 up + fi + + if [[ ! -z $PKT_IFACE ]] + then + # dummy so setup_environment binds nothing to dpdk + sudo ifconfig $PKT_IFACE 11.0.0.17 up + fi + + cd ../ + . ./scripts/install.sh - echo export ONVM_NIC_PCI=\"$pci_addresses\" >> ~/.bashrc - export ONVM_NIC_PCI="$pci_addresses" + # helper for binding interfaces + export DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py - . ./scripts/install.sh + if [[ ! -z $PKT_IFACE ]] + then + # bring pktgen interface down, and set it up + bind_nic_from_iface $PKT_IFACE # disable flow table lookup for faster results sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile fi diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker-files/pktgen-worker.sh index 4bbdd2243..05dcbd9f7 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker-files/pktgen-worker.sh @@ -1,18 +1,10 @@ #!/bin/bash -if [[ -z $1 ]] -then - echo "ERROR: Missing first argument, path to pktgen config file" - exit 1 -fi - -if [[ ! -f $1 ]] -then - echo "ERROR: Could not find config file at given path!" - exit 1 -fi - -. $1 # source the variables from pktgen config file +run_pktgen() { + python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER + # get Pktgen stats from server + scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats +} log "Running ONVM Manager" cd ~/repository/onvm @@ -43,15 +35,18 @@ sleep 10 # run pktgen log "Collecting Pktgen Statistics" -python3 ~/run-pktgen.py $PKT_WORKER_IP $PKT_WORKER_KEY_FILE $WORKER_USER -# get Pktgen stats from server -scp -i $PKT_WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$PKT_WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats +run_pktgen +# check if pktgen returned results (non-zero) +grep -v "^0$" ~/pktgen_stats +if [ $? -eq 1 ] +then + log "Running Pktgen again" + run_pktgen +fi log "Killing Basic Monitor" sudo pkill -f /basic_monitor log "Exiting ONVM" - -echo "Manager pid: ${mgr_pid}" sudo pkill -f onvm_mgr check_exit_code "ERROR: Killing manager failed" From fcd3b1a541dd8af944f4407a2e6d2992aa3ceaaf Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Thu, 1 Aug 2019 21:22:13 -0400 Subject: [PATCH 36/42] Renaming a ton of files --- ci/README.md | 6 ++-- ci/ci_busy.sh | 2 +- ...nctions.sh => helper-manager-functions.sh} | 0 .../helper-install-functions.sh} | 0 .../install-pktgen.sh | 2 +- .../pktgen-timed-config.lua | 0 ci/manager.sh | 8 ++--- ci/run_ci.sh | 2 +- ci/worker-files/install.py | 15 -------- ci/worker-files/worker-config | 2 -- .../helper-worker-functions.sh} | 0 .../pktgen-worker.sh | 3 +- .../run-pktgen.py | 0 .../speed-tester-worker.sh} | 0 ci/worker_files/worker-config | 8 +++++ ci/{worker-files => worker_files}/worker.sh | 36 +++++++++---------- 16 files changed, 35 insertions(+), 49 deletions(-) rename ci/{helper-functions.sh => helper-manager-functions.sh} (100%) rename ci/{install-pktgen/helper-functions.sh => install_pktgen/helper-install-functions.sh} (100%) rename ci/{install-pktgen => install_pktgen}/install-pktgen.sh (97%) rename ci/{install-pktgen => install_pktgen}/pktgen-timed-config.lua (100%) delete mode 100644 ci/worker-files/install.py delete mode 100755 ci/worker-files/worker-config rename ci/{worker-files/helper-functions.sh => worker_files/helper-worker-functions.sh} (100%) rename ci/{worker-files => worker_files}/pktgen-worker.sh (95%) rename ci/{worker-files => worker_files}/run-pktgen.py (100%) rename ci/{worker-files/speed-test-worker.sh => worker_files/speed-tester-worker.sh} (100%) create mode 100755 ci/worker_files/worker-config rename ci/{worker-files => worker_files}/worker.sh (65%) diff --git a/ci/README.md b/ci/README.md index 556d0656b..5b9d74c71 100644 --- a/ci/README.md +++ b/ci/README.md @@ -69,7 +69,7 @@ The CI process can be broken into multiple steps: 5. Run linter on the checked out code - Runs the `run_linter` function in `helper-functions.sh` + Runs the `run_linter` function in `helper-manager-functions.sh` 6. Clean up and restart all worker nodes @@ -85,7 +85,7 @@ The CI process can be broken into multiple steps: 9. Run modes are supplied to tell the worker which applications to test - Handle installation with `worker-files/worker.sh` for builds, and setting up manager for performance tests + Handle installation with `worker_files/worker.sh` for builds, and setting up manager for performance tests 10. Acquire results from the worker nodes @@ -124,7 +124,7 @@ Connecting two nodes is useful for measuring statistics with tools like Pktgen a - This will ensure `ping` understands what IP address it is supposed to talk to - If `ping -I 11.0.0.2>` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` -Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install-pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! +Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install_pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! ### Checking if Online diff --git a/ci/ci_busy.sh b/ci/ci_busy.sh index 59d510f39..66038c6de 100755 --- a/ci/ci_busy.sh +++ b/ci/ci_busy.sh @@ -2,7 +2,7 @@ set -e -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" diff --git a/ci/helper-functions.sh b/ci/helper-manager-functions.sh similarity index 100% rename from ci/helper-functions.sh rename to ci/helper-manager-functions.sh diff --git a/ci/install-pktgen/helper-functions.sh b/ci/install_pktgen/helper-install-functions.sh similarity index 100% rename from ci/install-pktgen/helper-functions.sh rename to ci/install_pktgen/helper-install-functions.sh diff --git a/ci/install-pktgen/install-pktgen.sh b/ci/install_pktgen/install-pktgen.sh similarity index 97% rename from ci/install-pktgen/install-pktgen.sh rename to ci/install_pktgen/install-pktgen.sh index b345c798f..05e158839 100755 --- a/ci/install-pktgen/install-pktgen.sh +++ b/ci/install_pktgen/install-pktgen.sh @@ -1,6 +1,6 @@ #!/bin/bash -. helper-functions.sh +. helper-install-functions.sh set -e diff --git a/ci/install-pktgen/pktgen-timed-config.lua b/ci/install_pktgen/pktgen-timed-config.lua similarity index 100% rename from ci/install-pktgen/pktgen-timed-config.lua rename to ci/install_pktgen/pktgen-timed-config.lua diff --git a/ci/manager.sh b/ci/manager.sh index 328a5a1aa..6a9f3e9c0 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -4,7 +4,7 @@ set -e # source helper functions file -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Config File and Sourcing Variables" @@ -157,11 +157,11 @@ do worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" # make sure the config file is updated with the correct run mode - sed -i "/WORKER_MODE*/c\\WORKER_MODE=\"${RUN_MODE}\"" worker-files/worker-config + sed -i "/WORKER_MODE*/c\\WORKER_MODE=\"${RUN_MODE}\"" worker_files/worker-config # put all files in one temporary folder for one scp cp -r ./$worker_ip temp cp -r ./repository temp/ - cp -r ./worker-files/* temp/ + cp -r ./worker_files/* temp/ scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./temp/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" # get rid of the temp folder now for next worker @@ -187,7 +187,7 @@ do # get the benchmarks for each node (some servers are faster) . ./$worker_ip/benchmarks # TODO: this will overwrite results if we have more than 1 worker, investigate this case - if [[ "$RUN_MODE" -eq "3" ]] + if [[ "$RUN_MODE" -eq "0" ]] then # fetch pktgen stats fetch_files $worker_key_file $worker_ip pktgen_stats diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 4b05c4e3d..b19f32ba1 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -4,7 +4,7 @@ set -e # source helper functions file -. helper-functions.sh +. helper-manager-functions.sh SCRIPT_LOC=$(pwd) print_header "Validating Input Variables" diff --git a/ci/worker-files/install.py b/ci/worker-files/install.py deleted file mode 100644 index 335c1f210..000000000 --- a/ci/worker-files/install.py +++ /dev/null @@ -1,15 +0,0 @@ -import pexpect -import os - -cwd = os.path.dirname(os.path.realpath(__file__)) -cwd += "/repository/scripts" -child = pexpect.spawn(cwd + "/install.sh", cwd=cwd) - -# script takes a while, increase timeout -child.timeout = 120 -# bind correct interface to DPDK -child.expect("Bind interface 0000:07:00.0 to DPDK.*") -child.sendline("N") -child.expect("Bind interface 0000:07:00.1 to DPDK.*") -child.sendline("Y\n") -child.interact() diff --git a/ci/worker-files/worker-config b/ci/worker-files/worker-config deleted file mode 100755 index ac716c1bb..000000000 --- a/ci/worker-files/worker-config +++ /dev/null @@ -1,2 +0,0 @@ -MODES="0" -PKT_CONFIG=~/pkt-config diff --git a/ci/worker-files/helper-functions.sh b/ci/worker_files/helper-worker-functions.sh similarity index 100% rename from ci/worker-files/helper-functions.sh rename to ci/worker_files/helper-worker-functions.sh diff --git a/ci/worker-files/pktgen-worker.sh b/ci/worker_files/pktgen-worker.sh similarity index 95% rename from ci/worker-files/pktgen-worker.sh rename to ci/worker_files/pktgen-worker.sh index 05dcbd9f7..404650d2b 100755 --- a/ci/worker-files/pktgen-worker.sh +++ b/ci/worker_files/pktgen-worker.sh @@ -37,8 +37,7 @@ sleep 10 log "Collecting Pktgen Statistics" run_pktgen # check if pktgen returned results (non-zero) -grep -v "^0$" ~/pktgen_stats -if [ $? -eq 1 ] +if [ -z "$(grep -v "^0$" ~/pktgen_stats | cat)" ] then log "Running Pktgen again" run_pktgen diff --git a/ci/worker-files/run-pktgen.py b/ci/worker_files/run-pktgen.py similarity index 100% rename from ci/worker-files/run-pktgen.py rename to ci/worker_files/run-pktgen.py diff --git a/ci/worker-files/speed-test-worker.sh b/ci/worker_files/speed-tester-worker.sh similarity index 100% rename from ci/worker-files/speed-test-worker.sh rename to ci/worker_files/speed-tester-worker.sh diff --git a/ci/worker_files/worker-config b/ci/worker_files/worker-config new file mode 100755 index 000000000..723d8192a --- /dev/null +++ b/ci/worker_files/worker-config @@ -0,0 +1,8 @@ +WORKER_MODE="0" +PKT_IFACE="p2p1" +MTCP_IFACE="p2p2" +MTCP_IP_ADDR="11.0.0.17" +MTCP_RUN_TIME="15" +WORKER_IP="nimbnode18" +WORKER_KEY_FILE=~/ci18key +WORKER_USER="kdeems" diff --git a/ci/worker-files/worker.sh b/ci/worker_files/worker.sh similarity index 65% rename from ci/worker-files/worker.sh rename to ci/worker_files/worker.sh index 771738a7f..878632041 100755 --- a/ci/worker-files/worker.sh +++ b/ci/worker_files/worker.sh @@ -14,55 +14,51 @@ fi . $1 # source config file -if [[ -z $MODES ]] +if [[ -z $WORKER_MODE ]] then echo "ERROR: Missing mode argument of config!" exit 1 fi -RUN_PKT=false -if [[ $MODES == *0* ]] -then - # will be running pktgen - if [[ ! -f $PKT_CONFIG ]] - then - echo "ERROR: Mode 0 (Pktgen) must have a config file" - exit 1 - fi - RUN_PKT=true -fi - # source helper functions file -. helper-functions.sh +. helper-worker-functions.sh sudo apt-get install -y build-essential linux-headers-$(uname -r) git sudo apt-get install -y libnuma1 sudo apt-get install -y libnuma-dev +sudo apt-get install -y python3 cd repository log "Beginning Execution of Workload" log "Installing Environment" -install_env $RUN_PKT +install_env check_exit_code "ERROR: Installing environment failed" log "Building ONVM" build_onvm check_exit_code "ERROR: Building ONVM failed" -for mode in $MODES +for mode in $WORKER_MODE do # run functionality for each mode case "$mode" in "0") - . ~/pktgen-worker.sh $PKT_CONFIG - . ~/speed-test-worker.sh + . ~/speed-tester-worker.sh + . ~/pktgen-worker.sh + . ~/mtcp-worker.sh + ;; + "1") + . ~/speed-tester-worker.sh ;; "2") - . ~/speed-test-worker.sh + . ~/pktgen-worker.sh + ;; + "3") + . ~/mtcp-worker.sh ;; *) - echo "Mode $MODE has not been implemented" + echo "Worker mode $mode has not been implemented" ;; esac done From ee976516fbd801564d94122c3c1cf9477bdb286b Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Thu, 1 Aug 2019 21:57:19 -0400 Subject: [PATCH 37/42] Fixed symbolic link --- ci/install_pktgen/helper-install-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_pktgen/helper-install-functions.sh b/ci/install_pktgen/helper-install-functions.sh index d61897ac9..ab2dbde82 120000 --- a/ci/install_pktgen/helper-install-functions.sh +++ b/ci/install_pktgen/helper-install-functions.sh @@ -1 +1 @@ -../worker-files/helper-functions.sh \ No newline at end of file +../worker_files/helper-worker-functions.sh \ No newline at end of file From 2d305f7a6323d061a306cb78d14033aa2b461a91 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Sat, 3 Aug 2019 12:47:35 -0400 Subject: [PATCH 38/42] Added benchmark info to README and renamed file --- ci/README.md | 15 +++++++++++++-- ...ll-functions.sh => helper-worker-functions.sh} | 0 2 files changed, 13 insertions(+), 2 deletions(-) rename ci/install_pktgen/{helper-install-functions.sh => helper-worker-functions.sh} (100%) diff --git a/ci/README.md b/ci/README.md index 5b9d74c71..2caf627a9 100644 --- a/ci/README.md +++ b/ci/README.md @@ -122,9 +122,20 @@ Connecting two nodes is useful for measuring statistics with tools like Pktgen a - Do this on both machines, to find the name of the interfaces that are linked - Run `sudo ifconfig 11.0.0.1/24 up` on the first machine and `sudo ifconfig 11.0.0.2/24 up` - This will ensure `ping` understands what IP address it is supposed to talk to -- If `ping -I 11.0.0.2>` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` +- If `ping -I 11.0.0.2` on the first machine works, great, if not, try changing the IP addresses or viewing `dmesg` -Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install_pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! +Now that the interfaces are connected, choose which machine will be the CI worker, and which is a helper (Pktgen for example). Install Pktgen on this node by sending the `ci/install_pktgen` files to that machines' home folder. *Remember public keys must be created for all new machines*. Store these public keys in a folder with the server name, see the next section on statistics for more information. Run `chmod +x install-pktgen.sh` if it's not already an executable and run `./install-pktgen.sh` to install everything. If there are dependency errors, the machine might be a different version, so try to install the necessary packages. Once everything is installed, test ONVM->Pktgen between the machines, and if a connection is established, CI should work just fine with no more setup! + +### Advanced Statistics + +As CI continued to improve, with more programs to test with, benchmarks were made to track the average performance of a worker. In the future, CI will be able to handle multiple workers running many different tests. Since server configurations are not all the same, some with different hardware (Intel x710 vs. x520 NIC for example), performance of the nodes will not be the same. All that matters with CI is that the result of a run is the same or better, not globally across all nodes, but based on the specific server it ran on. For each worker, create a folder in the ci directory with the name of the worker IP. For example if `nimbnode17` is the current worker, a folder with path `/ci/nimbnode17/` should exist. In this folder, 3 files should be there at least. Firstly, a `benchmarks` file (used by the manager) should look similar to this: + +``` +AVG_SPEED_TESTER_SPEED=40000000 +AVG_PKTGEN_SPEED=10000000 +AVG_MTCP_SPEED=.230 +``` +This is a configuration file, sourced by the manager to keep track of `nimbnode17`'s average performance for each test (currently Speed Tester, Pktgen, and mTCP). The other two files in the folder should be the two public keys, one for the worker, and the second for the worker's client server. Check the previous section on setting up a connection for more information. ### Checking if Online diff --git a/ci/install_pktgen/helper-install-functions.sh b/ci/install_pktgen/helper-worker-functions.sh similarity index 100% rename from ci/install_pktgen/helper-install-functions.sh rename to ci/install_pktgen/helper-worker-functions.sh From 4f9928ba754e60f5752b152e4e25c3d66fa3f59b Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Mon, 5 Aug 2019 19:20:11 -0400 Subject: [PATCH 39/42] Fix Pktgen install requested changes --- ci/install_pktgen/install-pktgen.sh | 4 +--- ci/install_pktgen/pktgen-timed-config.lua | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/install_pktgen/install-pktgen.sh b/ci/install_pktgen/install-pktgen.sh index 05e158839..7811ef947 100755 --- a/ci/install_pktgen/install-pktgen.sh +++ b/ci/install_pktgen/install-pktgen.sh @@ -7,7 +7,7 @@ set -e sudo rm -rf repository git clone https://github.com/sdnfv/openNetVM.git repository -check_exit_code "ERROR: Failed installing onvm" +check_exit_code "ERROR: Failed cloning" print_header "Installing Dependencies" sudo apt-get update @@ -27,8 +27,6 @@ sudo make install cd repository -print_header "Beginning Execution of Workload" - print_header "Installing Environment" install_env $RUN_PKT check_exit_code "ERROR: Installing environment failed" diff --git a/ci/install_pktgen/pktgen-timed-config.lua b/ci/install_pktgen/pktgen-timed-config.lua index cd0f5b7d5..d5b3cac9d 100644 --- a/ci/install_pktgen/pktgen-timed-config.lua +++ b/ci/install_pktgen/pktgen-timed-config.lua @@ -59,8 +59,7 @@ local function doWait(port, waitTime) -- Try to wait for the total number of packets to be sent. local idx = 0; while( idx < waitTime ) do - - -- Write port stats to output file separated by line + -- Write port stats to output file separated by line pkt_rate_file:write(pktgen.portStats("all", "rate")[0]["pkts_rx"] .. "\n"); idx = idx + 1; From a9168f9a8a42e227969622a5f7577d78ec2b18b4 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Mon, 5 Aug 2019 20:03:31 -0400 Subject: [PATCH 40/42] Address Github change requests --- ci/manager.sh | 6 +++--- ci/worker_files/pktgen-worker.sh | 11 ++++++++--- ci/worker_files/run-pktgen.py | 1 + 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 6a9f3e9c0..2a4d3da33 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -158,10 +158,10 @@ do worker_key_file="${tuple_arr[1]}" # make sure the config file is updated with the correct run mode sed -i "/WORKER_MODE*/c\\WORKER_MODE=\"${RUN_MODE}\"" worker_files/worker-config + # create directory for scp + mkdir temp # put all files in one temporary folder for one scp - cp -r ./$worker_ip temp - cp -r ./repository temp/ - cp -r ./worker_files/* temp/ + sudo cp -r ./nimbnode17/* ./repository ./worker_files/* temp scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./temp/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" # get rid of the temp folder now for next worker diff --git a/ci/worker_files/pktgen-worker.sh b/ci/worker_files/pktgen-worker.sh index 404650d2b..2f561a4a3 100755 --- a/ci/worker_files/pktgen-worker.sh +++ b/ci/worker_files/pktgen-worker.sh @@ -1,6 +1,6 @@ #!/bin/bash -run_pktgen() { +run_and_fetch_pktgen_stats() { python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER # get Pktgen stats from server scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats @@ -35,12 +35,17 @@ sleep 10 # run pktgen log "Collecting Pktgen Statistics" -run_pktgen +python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER +# get Pktgen stats from server +scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats + # check if pktgen returned results (non-zero) if [ -z "$(grep -v "^0$" ~/pktgen_stats | cat)" ] then log "Running Pktgen again" - run_pktgen + python3 ~/run-pktgen.py $WORKER_IP $WORKER_KEY_FILE $WORKER_USER + # get Pktgen stats from server + scp -i $WORKER_KEY_FILE -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $WORKER_USER@$WORKER_IP:~/repository/tools/Pktgen/pktgen-dpdk/port_stats ~/pktgen_stats fi log "Killing Basic Monitor" diff --git a/ci/worker_files/run-pktgen.py b/ci/worker_files/run-pktgen.py index 98268e9bd..5aa2e2fd9 100644 --- a/ci/worker_files/run-pktgen.py +++ b/ci/worker_files/run-pktgen.py @@ -18,6 +18,7 @@ client.set_missing_host_key_policy(AutoAddPolicy()) client.connect(worker_ip, timeout = 30, username=worker_user, pkey = key) +# Pktgen requires a pseudoterminal, created by get_pty (stdin, stdout, stderr) = client.exec_command("sudo ~/repository/tools/Pktgen/openNetVM-Scripts/run-pktgen.sh 1", get_pty=True) # block until script finishes exit_status = stdout.channel.recv_exit_status() From 0f3dfd2c389fe7b915af6cc29e875c49973d3fbe Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Mon, 5 Aug 2019 21:50:31 -0400 Subject: [PATCH 41/42] Adjustments to interface configuration --- ci/manager.sh | 6 ++-- ci/worker_files/helper-worker-functions.sh | 33 ++++++++-------------- ci/worker_files/worker-config | 5 ++-- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index 2a4d3da33..fd1dacd06 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -90,7 +90,7 @@ then fi print_header "Posting on Github that CI is starting" -python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "Your results will arrive shortly" +#python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "Your results will arrive shortly" check_exit_code "ERROR: Failed to post initial message to GitHub" for worker_tuple in "${WORKER_LIST[@]}" @@ -132,7 +132,7 @@ do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - python3 prepare-worker.py $worker_ip $worker_key_file + #python3 prepare-worker.py $worker_ip $worker_key_file done sleep 10 # wait 10 seconds for reboot to take effect @@ -161,7 +161,7 @@ do # create directory for scp mkdir temp # put all files in one temporary folder for one scp - sudo cp -r ./nimbnode17/* ./repository ./worker_files/* temp + cp -r ./$worker_ip/* ./repository ./worker_files/* temp scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r ./temp/* $worker_ip: check_exit_code "ERROR: Failed to copy ONVM files to $worker_ip" # get rid of the temp folder now for next worker diff --git a/ci/worker_files/helper-worker-functions.sh b/ci/worker_files/helper-worker-functions.sh index 8c9917c1e..e6f093426 100644 --- a/ci/worker_files/helper-worker-functions.sh +++ b/ci/worker_files/helper-worker-functions.sh @@ -29,7 +29,7 @@ logsetup # given a 10G NIC interface, bring down and bind to dpdk bind_nic_from_iface() { sudo ifconfig $1 down - id=$($DPDK_DEVBIND --status | grep -e "if=$1" | cut -f 1 -d " ") + id=$($DPDK_DEVBIND --status | grep -e "if=$1 drv=ixgbe" | cut -f 1 -d " ") sudo $DPDK_DEVBIND -b igb_uio $id } @@ -54,31 +54,22 @@ install_env() { sudo sh -c "echo 0 > /proc/sys/kernel/randomize_va_space" - if [[ ! -z $MTCP_IFACE ]] - then - # bring iface up so onvm install can't bind it - sudo ifconfig $MTCP_IFACE 11.0.0.17 up - fi + # helper for binding interfaces + export DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py - if [[ ! -z $PKT_IFACE ]] - then - # dummy so setup_environment binds nothing to dpdk - sudo ifconfig $PKT_IFACE 11.0.0.17 up - fi + for iface in $($DPDK_DEVBIND --status | grep -oP "if=\K(\w+)\sdrv=ixgbe" | cut -f 1 -d " ") + do + # bring all ixgbe interfaces up so install script can't bind + sudo ifconfig $iface 11.0.0.1 up + done cd ../ . ./scripts/install.sh - - # helper for binding interfaces - export DPDK_DEVBIND=$RTE_SDK/usertools/dpdk-devbind.py - if [[ ! -z $PKT_IFACE ]] - then - # bring pktgen interface down, and set it up - bind_nic_from_iface $PKT_IFACE - # disable flow table lookup for faster results - sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile - fi + # bring client facing interface down for dpdk + bind_nic_from_iface $CLIENT_IFACE + # disable flow table lookup for faster results + sed -i "/ENABLE_FLOW_LOOKUP\=1/c\\ENABLE_FLOW_LOOKUP=0" ~/repository/onvm/onvm_mgr/Makefile } # makes all onvm code diff --git a/ci/worker_files/worker-config b/ci/worker_files/worker-config index 723d8192a..297204a29 100755 --- a/ci/worker_files/worker-config +++ b/ci/worker_files/worker-config @@ -1,7 +1,6 @@ WORKER_MODE="0" -PKT_IFACE="p2p1" -MTCP_IFACE="p2p2" -MTCP_IP_ADDR="11.0.0.17" +CLIENT_IFACE="p2p1" +SERVER_IP_ADDR="11.0.0.17" MTCP_RUN_TIME="15" WORKER_IP="nimbnode18" WORKER_KEY_FILE=~/ci18key From 381f27d5a09bfaf54f0e019f565b6224b2bf2708 Mon Sep 17 00:00:00 2001 From: Kevin Deems Date: Mon, 5 Aug 2019 22:00:26 -0400 Subject: [PATCH 42/42] Fixed manager debug --- ci/manager.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/manager.sh b/ci/manager.sh index fd1dacd06..2b7005f17 100755 --- a/ci/manager.sh +++ b/ci/manager.sh @@ -90,7 +90,7 @@ then fi print_header "Posting on Github that CI is starting" -#python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "Your results will arrive shortly" +python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "Your results will arrive shortly" check_exit_code "ERROR: Failed to post initial message to GitHub" for worker_tuple in "${WORKER_LIST[@]}" @@ -132,7 +132,7 @@ do tuple_arr=($worker_tuple) worker_ip="${tuple_arr[0]}" worker_key_file="${tuple_arr[1]}" - #python3 prepare-worker.py $worker_ip $worker_key_file + python3 prepare-worker.py $worker_ip $worker_key_file done sleep 10 # wait 10 seconds for reboot to take effect