From 3e19eb386cb5b2545d16d1ffde61cc7b81e69be9 Mon Sep 17 00:00:00 2001 From: Aleksandr Kutsan Date: Thu, 9 Apr 2020 12:21:00 +0300 Subject: [PATCH 01/12] Fix boost compilation for RPi --- src/remote_adapter/remote_adapter_server/BOOST.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/remote_adapter/remote_adapter_server/BOOST.cmake b/src/remote_adapter/remote_adapter_server/BOOST.cmake index 1b2a190c..b15edd6e 100644 --- a/src/remote_adapter/remote_adapter_server/BOOST.cmake +++ b/src/remote_adapter/remote_adapter_server/BOOST.cmake @@ -24,7 +24,7 @@ if(QNXNTO) set(CMAKE_FIND_ROOT_PATH "${CMAKE_FIND_ROOT_PATH}" "${BOOST_ROOT}") endif() -find_package(Boost 1.68.0 COMPONENTS system filesystem) +find_package(Boost 1.68.0 COMPONENTS filesystem system) if (NOT ${Boost_FOUND}) message(STATUS "Did not find boost. Downloading and installing boost 1.68") @@ -70,8 +70,8 @@ if (NOT ${Boost_FOUND}) endif() set(BOOST_LIBRARIES - "${BOOST_LIBRARY_DIRS}/libboost_system.a" - "${BOOST_LIBRARY_DIRS}/libboost_filesystem.a") + "${BOOST_LIBRARY_DIRS}/libboost_filesystem.a" + "${BOOST_LIBRARY_DIRS}/libboost_system.a") include_directories("${BOOST_INCLUDE_DIRS}") From d68e4984ff45c89e667951ee21c3d49c8c3ec06b Mon Sep 17 00:00:00 2001 From: Aleksandr Kutsan Date: Thu, 9 Apr 2020 12:21:21 +0300 Subject: [PATCH 02/12] Add support of Background taks --- .../plugins/utils/utils_manager.cc | 14 +++++++++++--- .../plugins/utils/utils_manager.h | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.cc b/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.cc index 65480d87..fa7a4ba1 100644 --- a/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.cc +++ b/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include "../../common/constants.h" #include "../../common/custom_types.h" @@ -394,11 +395,11 @@ void UtilsManager::Bind(rpc::server &server) { LOG_ERROR("{0}: {1}", __func__, error_msg::kBadTypeValue); return response_type(error_msg::kBadTypeValue, error_codes::FAILED); } - auto receive_result = UtilsManager::ExecuteCommand(bash_command); - - return receive_result; + return ('&' == bash_command.back()) ? UtilsManager::ExecuteCommandBg(bash_command): + UtilsManager::ExecuteCommand(bash_command); }); } + std::string UtilsManager::PluginName() { return "RemoteUtilsManager"; } int UtilsManager::StartApp(const std::string &app_path, const std::string &app_name) { @@ -685,6 +686,13 @@ UtilsManager::ExecuteCommand(const std::string &bash_command) { return std::make_pair(command_output, term_status); } +std::pair +UtilsManager::ExecuteCommandBg(const std::string &bash_command) { + LOG_INFO("{0}: {1}", __func__, bash_command); + system(bash_command.c_str()); + return std::make_pair("Background process", error_codes::SUCCESS); +} + std::vector UtilsManager::GetAppPids(const std::string &app_name) { struct dirent *dirent; diff --git a/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.h b/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.h index 73b50357..da087ac4 100644 --- a/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.h +++ b/src/remote_adapter/remote_adapter_server/plugins/utils/utils_manager.h @@ -143,6 +143,11 @@ class UtilsManager : public remote_adapter::UtilsPlugin { static std::pair ExecuteCommand(const std::string &bash_command); + static + std::pair + ExecuteCommandBg(const std::string &bash_command); + + private: static std::vector GetAppPids(const std::string &app_name); static std::string GetAppStatus(int pid, int *num_threads = 0); From 281890ff6078d148263ca69b70767112d99126b6 Mon Sep 17 00:00:00 2001 From: Aleksandr Kutsan Date: Thu, 9 Apr 2020 12:21:58 +0300 Subject: [PATCH 03/12] add remote adapter server --- CMakeLists.txt | 6 ++++ remote_atf_docker/Dockerfile | 29 +++++++++++++++++++ remote_atf_docker/build_docker.sh | 1 + remote_atf_docker/entrypoint.sh | 36 ++++++++++++++++++++++++ remote_atf_docker/start_remote_server.sh | 22 +++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 remote_atf_docker/Dockerfile create mode 100755 remote_atf_docker/build_docker.sh create mode 100644 remote_atf_docker/entrypoint.sh create mode 100755 remote_atf_docker/start_remote_server.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index ad4b7e62..5aec9ba4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,12 @@ install(TARGETS ${PROJECT_NAME} PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ COMPONENT sdl_atf) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/remote_atf_docker" + "${CMAKE_SOURCE_DIR}/data" + DESTINATION "${CMAKE_INSTALL_PREFIX}" + FILE_PERMISSIONS OWNER_WRITE OWNER_READ + COMPONENT sdl_atf) + install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules" "${CMAKE_SOURCE_DIR}/data" DESTINATION "${CMAKE_INSTALL_PREFIX}" diff --git a/remote_atf_docker/Dockerfile b/remote_atf_docker/Dockerfile new file mode 100644 index 00000000..994e0b0b --- /dev/null +++ b/remote_atf_docker/Dockerfile @@ -0,0 +1,29 @@ +from ubuntu:latest + +ENV TZ=Europe/Minsk +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && apt-get -q -y install locales sudo + +RUN apt-get update && apt-get -q -y install libssl1.0.0 libusb-1.0-0 libbluetooth3 \ + libssl-doc- libusb-1.0-doc- manpages- manpages-dev- + +RUN apt-get update && apt-get -q -y install openssl liblua5.2-0 psmisc \ + autotools-dev- binutils- build-essential- bzip2- cpp- cpp-5- dpkg-dev- fakeroot- \ + manpages- manpages-dev- + +RUN apt-get update && apt-get -q -y install sqlite3 libqt5websockets5 net-tools iproute2 \ + qttranslations5-l10n- xdg-user-dirs- xml-core- dbus- + +RUN apt-get update && apt install -y netcat cpipe gnuplot sysstat python3-pip docker.io +RUN apt-get update && apt install -y python3-numpy python3-scipy python3-matplotlib + +RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 + +ENV LC_ALL en_US.UTF-8 + +RUN chmod u+s /sbin/ifconfig /sbin/ip && mkdir /home/developer + +COPY entrypoint.sh /usr/bin/ + +ENTRYPOINT ["/bin/bash", "-e", "/usr/bin/entrypoint.sh"] diff --git a/remote_atf_docker/build_docker.sh b/remote_atf_docker/build_docker.sh new file mode 100755 index 00000000..7ba5abd0 --- /dev/null +++ b/remote_atf_docker/build_docker.sh @@ -0,0 +1 @@ +docker build . -t remote_atf_server diff --git a/remote_atf_docker/entrypoint.sh b/remote_atf_docker/entrypoint.sh new file mode 100644 index 00000000..dedd48d5 --- /dev/null +++ b/remote_atf_docker/entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Add local user +# Either use the LOCAL_USER_ID if passed in at runtime or +# fallback + +USER_ID=${LOCAL_USER_ID:-9001} + +useradd --shell /bin/bash -u $USER_ID -o -c "" developer +usermod -aG docker developer +sudo chmod 666 /var/run/docker.sock +echo "developer ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +chown developer /home/developer +chgrp developer /home/developer + +export HOME=/home/developer +export LANG=en_US.UTF-8 + +echo "Remote adapter server available by following ip adresses : " +ip addr | grep -E "inet " | tr -s " " | cut -d ' ' -f 3| cut -d '/' -f 1 + +ls /home/developer + +echo "SDL bin dir : /home/developer/sdlbin"; ls /home/developer/sdlbin +echo "ATF bin dir : /home/developer/atfbin"; ls /home/developer/atfbin +echo "Third party /home/developer/thirdparty"; ls /home/developer/thirdparty + +cd /home/developer/atfbin/RemoteTestingAdapterServer + +echo "Third party contents" +LIB="/home/developer/thirdparty/lib/" +LIB64="/home/developer/thirdparty/x86_64/lib/" + +sudo LD_LIBRARY_PATH="$LIB:$LIB64:." -u developer "./RemoteTestingAdapterServer" + diff --git a/remote_atf_docker/start_remote_server.sh b/remote_atf_docker/start_remote_server.sh new file mode 100755 index 00000000..58d8d54d --- /dev/null +++ b/remote_atf_docker/start_remote_server.sh @@ -0,0 +1,22 @@ +PATH_TO_SDLBIN=$1 +PATH_TO_ATFBIN=$2 +PATH_TO_THIRDPARTY=$3 + +CONTAINER_NAME=remote_sdl +IMAGE_NAME=remote_atf_server +MEMORY_CONSTRAINS=4G +CPUS=2 +docker rm $CONTAINER_NAME + +echo $1 $2 $3 +docker run -e LOCAL_USER_ID=$UID \ + -e CONTAINER_NAME=$CONTAINER_NAME\ + -e LOCAL_USER_ID=$(id -u)\ + --net=host\ + --name=$CONTAINER_NAME\ + -v /var/run/docker.sock:/var/run/docker.sock\ + -v $PATH_TO_ATFBIN:/home/developer/atfbin\ + -v $PATH_TO_SDLBIN:/home/developer/sdlbin\ + -v $PATH_TO_THIRDPARTY:/home/developer/thirdparty\ + -m=$MEMORY_CONSTRAINS\ + -it $IMAGE_NAME From 0bd5e7acbf656a5ae41baac93da82c44731d056b Mon Sep 17 00:00:00 2001 From: Aleksandr Kutsan Date: Thu, 9 Apr 2020 12:22:21 +0300 Subject: [PATCH 04/12] Add script for collecting SDL RAM\CPU --- tools/measure_sdl.sh | 63 ++++++++++++++ tools/sdl_graphs.py | 191 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100755 tools/measure_sdl.sh create mode 100755 tools/sdl_graphs.py diff --git a/tools/measure_sdl.sh b/tools/measure_sdl.sh new file mode 100755 index 00000000..a5e6481a --- /dev/null +++ b/tools/measure_sdl.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# apt-get install sysstat linux-tools-common + +SDL_PID=$(pidof smartDeviceLinkCore) +REMOTE=false +OUTPUT_DIR="measure"/$1 +OUTPUT_PIDSTAT_FILE=$OUTPUT_DIR/pidstat.log +OUTPUT_PS_FILE=$OUTPUT_DIR/ps.log +OUTPUT_DOCKER_FILE=$OUTPUT_DIR/docker.log + +STEP=1 + +if [ -n $2 ] && [ "$2" = "--remote" ]; then REMOTE=true; fi + +rm -rf $OUTPUT_DIR +mkdir -p $OUTPUT_DIR +pwd +echo $OUTPUT_DIR +function ps_formated() { + ps --no-headers --format "start %cpu cp cputime %mem sz thcount " -p $SDL_PID +} + +function measure_ps() { + PS_OUTPUT=$(ps_formated) + TIME=$(date +'%T:%N') + echo $TIME $PS_OUTPUT >> $OUTPUT_PS_FILE +} + +function measure_pid_stat() { + export S_TIME_FORMAT=ISO + STAT_OUTPUT=$(pidstat -urdlv -h -p $SDL_PID | grep $SDL_PID) + echo $STAT_OUTPUT >> $OUTPUT_PIDSTAT_FILE +} + +function measure_docker() { + RECORD=$(docker stats --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}" --no-stream | grep remote_sdl) + echo $RECORD + TIME=$(date +'%T:%N') + echo $TIME $RECORD >> $OUTPUT_DOCKER_FILE +} + +echo "Wait for SDL start" +until SDL_PID=$(pidof smartDeviceLinkCore) +do + sleep $STEP + printf "." +done + +while SDL_PID=$(pidof smartDeviceLinkCore) +do + measure_pid_stat + measure_ps + if [ $REMOTE = true ]; then measure_docker; fi + sleep $STEP +done + +PARAMS="--pidstat_file=$OUTPUT_PIDSTAT_FILE --ps_file=$OUTPUT_PS_FILE --output_dir=$OUTPUT_DIR --title=$1" +if [ $REMOTE = true ]; then PARAMS="$PARAMS --docker_file=$OUTPUT_DOCKER_FILE"; fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR +python3 ./sdl_graphs.py $PARAMS diff --git a/tools/sdl_graphs.py b/tools/sdl_graphs.py new file mode 100755 index 00000000..2feaaf9c --- /dev/null +++ b/tools/sdl_graphs.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# coding: utf-8 + +import matplotlib +matplotlib.use('Agg') +from scipy.stats import norm +from matplotlib import pyplot as plt +import matplotlib.dates as mdates +import numpy as np +from datetime import datetime +import datetime as dt +import argparse +import os +import re + + +def generator(line): + line = line.split() + for x in line: + yield x + +def sdl_start_time_ps(lines): + process_start = datetime.strptime(lines[0].split()[1], "%H:%M:%S") + return process_start + +def parse_ps(line): + # time + ps --no-headers --format "start %cpu cp cputime %mem sz thcount" + line = line.split() + record = {} + record["time"] = datetime.strptime(line[0][:-3], "%H:%M:%S:%f") + record["start"] = datetime.strptime(line[1], "%H:%M:%S") + record["%cpu"] = float(line[2]) + record["cp"] = float(line[3]) + record["cputime"] = datetime.strptime(line[4], "%H:%M:%S") + record["%mem"] = float(line[5]) + record["sz"] = float(line[6]) + record["thcount"] = int(line[7]) + return record + +def sdl_start_time_docker(lines): + line = lines[0].split() + + process_start = datetime.strptime(line[0][:-3], "%H:%M:%S:%f") + return process_start + +def parse_docker(line): + # docker stats --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}" --no-stream + columns = generator(line) + + next_col = lambda : next(columns) + record = {} + record["time"] = datetime.strptime(next_col()[:-3], "%H:%M:%S:%f") + record["Name"] = next_col() + record["CPUPerc"] = float(next_col().replace("%", "")) + record["MemUsage"] = float(re.sub(r'[A-z]', '', next_col())) + slash = next_col() + record["MemLimit"] = next_col() + record["PIDs"] = int(next_col()) + + return record + +def graphs(record_type): + if record_type == "ps": + return ["%cpu", "%mem", "sz", "thcount"] + if record_type == "pidstat": + return ["%CPU", "VSZ", "RSS", "%MEM", "threads"] + if record_type == "docker": + return ["CPUPerc", "MemUsage", "PIDs"] + raise "Unknown record_type {}".format(record_type) + +def sdl_start_time_pidstat(lines): + process_start = datetime.strptime(lines[0].split()[0], "%H:%M:%S") + return process_start + +def parse_pidstat(line): + # pidstat -urdlv -h + columns = generator(line) + next_col = lambda : next(columns) + record = {} + record["time"] = datetime.strptime(next_col(), "%H:%M:%S") + uid = next_col() + pid = next_col() + record["%usr"] = float(next_col()) + record["%system"] = float(next_col()) + record["%guest"] = float(next_col()) + record["%wait"] = float(next_col()) + record["%CPU"] = float(next_col()) + record["CPU"] = int(next_col()) + record["minflt/s"] = float(next_col()) + record["majflt/s"] = float(next_col()) + record["VSZ"] = int(next_col()) + record["RSS"] = int(next_col()) + record["%MEM"] = float(next_col()) + record["kB_rd/s"] = float(next_col()) + record["kB_wr/s"] = float(next_col()) + record["kB_ccwr/s"] = float(next_col()) + record["iodelay"] = float(next_col()) + record["threads"] = float(next_col()) + record["fd-nr"] = float(next_col()) + record["Command"] = next_col() + return record + +def read_records(filename, parsing_func, parsing_proc_stat): + f=open(filename) + lines = f.readlines() + process_start = parsing_proc_stat(lines) + print("Started at ", process_start) + records = [] + for line in lines: + try: + record = parsing_func(line) + delta = record["time"] - process_start + delta = datetime(1970,1,1) + delta + record["delta"] = delta + records.append(record) + except Exception as e: + print(line) + raise e + return records + +def plot(x, y ,records, title, xlable, ylable, file_pattern): + assert(len(x) == len(y)) + assert(len(x) != 0) + fig, ax = plt.subplots(figsize=(20,5)) + ax.plot(x,y) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%M:%S')) + fig.autofmt_xdate() + plt.title(title) + plt.ylabel(ylable) + plt.xlabel(xlable) + plt.savefig("{}_{}.png".format(file_pattern, ylable), format='png') + # plt.show() + +def plot_record(key, records, title = None, xlable="Time", ylable=None, file_pattern = "output"): + if ylable is None: + ylable = key + if title is None: + title = key + x = [record["delta"] for record in records] + y = [record[key] for record in records] + plot(x, y, records, title, xlable, ylable, file_pattern) + +def process_records(records, record_type, title, output_dir): + if len(records) == 0: + print("empty records :".format(record_type)) + for key in graphs(record_type): + plot_record(key, records, title = title, + file_pattern = output_dir + "_" + record_type) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--pidstat_file", help='''File with pidstat output in format : + pidstat -urdlv -h ''', + type=str) + parser.add_argument("--ps_file", help='''File with ps output in format : + time + ps --no-headers --format "start %cpu cp cputime %mem sz thcount"''', + type=str) + parser.add_argument("--docker_file", help='''File with docker stats output in format : + docker stats --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}"''', + type=str) + parser.add_argument("--output_dir", help='''Output Directory''', + type=str) + parser.add_argument("--title", help='''Graph title''', + type=str) + args = parser.parse_args() + + if args.pidstat_file: + print("pidstat log file : {}".format(args.pidstat_file)) + records = read_records(args.pidstat_file, parse_pidstat, sdl_start_time_pidstat) + process_records(records, "pidstat", + args.title + "\npidstat -urdlv -h", + os.path.join(args.output_dir , "pidstat")) + + if args.ps_file: + print("ps log file : {}".format(args.ps_file)) + records = read_records(args.ps_file, parse_ps, sdl_start_time_ps) + process_records(records, "ps", + args.title + '\nps --no-headers --format "start %cpu cp cputime %mem sz thcount"', + os.path.join(args.output_dir , "ps")) + if args.docker_file: + print("docker log file : {}".format(args.docker_file)) + records = read_records(args.docker_file, parse_docker, sdl_start_time_docker) + process_records(records, "docker", + args.title + '\n{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}', + os.path.join(args.output_dir , "docker")) + + + print("Done") + +if __name__ == "__main__": + main() From 58f7051aef0a6e8bdd18551d1b252eda77856869 Mon Sep 17 00:00:00 2001 From: Aleksandr Kutsan Date: Mon, 13 Apr 2020 13:51:12 +0300 Subject: [PATCH 05/12] Minor foxes --- remote_atf_docker/entrypoint.sh | 7 ++++--- remote_atf_docker/start_remote_server.sh | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/remote_atf_docker/entrypoint.sh b/remote_atf_docker/entrypoint.sh index dedd48d5..e7a4f3e8 100644 --- a/remote_atf_docker/entrypoint.sh +++ b/remote_atf_docker/entrypoint.sh @@ -20,11 +20,12 @@ export LANG=en_US.UTF-8 echo "Remote adapter server available by following ip adresses : " ip addr | grep -E "inet " | tr -s " " | cut -d ' ' -f 3| cut -d '/' -f 1 +echo " ls /home/developer " ls /home/developer -echo "SDL bin dir : /home/developer/sdlbin"; ls /home/developer/sdlbin -echo "ATF bin dir : /home/developer/atfbin"; ls /home/developer/atfbin -echo "Third party /home/developer/thirdparty"; ls /home/developer/thirdparty +echo "SDL bin dir : /home/developer/sdlbin"; ls -la /home/developer/sdlbin +echo "ATF bin dir : /home/developer/atfbin"; ls -la /home/developer/atfbin +echo "Third party /home/developer/thirdparty"; ls -la /home/developer/thirdparty cd /home/developer/atfbin/RemoteTestingAdapterServer diff --git a/remote_atf_docker/start_remote_server.sh b/remote_atf_docker/start_remote_server.sh index 58d8d54d..4ba69191 100755 --- a/remote_atf_docker/start_remote_server.sh +++ b/remote_atf_docker/start_remote_server.sh @@ -15,8 +15,8 @@ docker run -e LOCAL_USER_ID=$UID \ --net=host\ --name=$CONTAINER_NAME\ -v /var/run/docker.sock:/var/run/docker.sock\ - -v $PATH_TO_ATFBIN:/home/developer/atfbin\ -v $PATH_TO_SDLBIN:/home/developer/sdlbin\ + -v $PATH_TO_ATFBIN:/home/developer/atfbin\ -v $PATH_TO_THIRDPARTY:/home/developer/thirdparty\ -m=$MEMORY_CONSTRAINS\ -it $IMAGE_NAME From f53dbb03b766db533cf1a4a419894c0be7ec5180 Mon Sep 17 00:00:00 2001 From: a-kutsan Date: Mon, 13 Apr 2020 17:47:38 +0300 Subject: [PATCH 06/12] Update measure_sdl.sh --- tools/measure_sdl.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/measure_sdl.sh b/tools/measure_sdl.sh index a5e6481a..1f141204 100755 --- a/tools/measure_sdl.sh +++ b/tools/measure_sdl.sh @@ -58,6 +58,4 @@ done PARAMS="--pidstat_file=$OUTPUT_PIDSTAT_FILE --ps_file=$OUTPUT_PS_FILE --output_dir=$OUTPUT_DIR --title=$1" if [ $REMOTE = true ]; then PARAMS="$PARAMS --docker_file=$OUTPUT_DOCKER_FILE"; fi -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd $DIR -python3 ./sdl_graphs.py $PARAMS +python3 $DIR/sdl_graphs.py $PARAMS From ffd93f51d47b8214a42894495e45e1ed2df007f4 Mon Sep 17 00:00:00 2001 From: Mykyta Nikora Date: Mon, 13 Apr 2020 20:07:49 +0300 Subject: [PATCH 07/12] Update Dockerfile --- remote_atf_docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote_atf_docker/Dockerfile b/remote_atf_docker/Dockerfile index 994e0b0b..139594e9 100644 --- a/remote_atf_docker/Dockerfile +++ b/remote_atf_docker/Dockerfile @@ -1,4 +1,4 @@ -from ubuntu:latest +FROM ubuntu:18.04 ENV TZ=Europe/Minsk RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone From f4e6511bf3d4ed66f73ff08d565aff79e8272363 Mon Sep 17 00:00:00 2001 From: a-kutsan Date: Wed, 15 Apr 2020 17:53:10 +0300 Subject: [PATCH 08/12] Update measure_sdl.sh --- tools/measure_sdl.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/measure_sdl.sh b/tools/measure_sdl.sh index 1f141204..d4cb5cab 100755 --- a/tools/measure_sdl.sh +++ b/tools/measure_sdl.sh @@ -58,4 +58,5 @@ done PARAMS="--pidstat_file=$OUTPUT_PIDSTAT_FILE --ps_file=$OUTPUT_PS_FILE --output_dir=$OUTPUT_DIR --title=$1" if [ $REMOTE = true ]; then PARAMS="$PARAMS --docker_file=$OUTPUT_DOCKER_FILE"; fi +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" python3 $DIR/sdl_graphs.py $PARAMS From 64758164043fb226495a464db4a71ce5cf47fe8e Mon Sep 17 00:00:00 2001 From: Dmitriy Boltovskiy Date: Thu, 16 Apr 2020 15:27:07 -0400 Subject: [PATCH 09/12] Minor fixes --- CMakeLists.txt | 6 +++--- remote_atf_docker/Dockerfile | 2 +- remote_atf_docker/entrypoint.sh | 0 remote_atf_docker/start_remote_server.sh | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) mode change 100644 => 100755 remote_atf_docker/entrypoint.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 5aec9ba4..451daef2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,18 +74,18 @@ install(TARGETS ${PROJECT_NAME} install(DIRECTORY "${CMAKE_SOURCE_DIR}/remote_atf_docker" "${CMAKE_SOURCE_DIR}/data" DESTINATION "${CMAKE_INSTALL_PREFIX}" - FILE_PERMISSIONS OWNER_WRITE OWNER_READ + USE_SOURCE_PERMISSIONS COMPONENT sdl_atf) install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules" "${CMAKE_SOURCE_DIR}/data" DESTINATION "${CMAKE_INSTALL_PREFIX}" - FILE_PERMISSIONS OWNER_WRITE OWNER_READ + USE_SOURCE_PERMISSIONS COMPONENT sdl_atf) install(DIRECTORY "${CMAKE_SOURCE_DIR}/tools" DESTINATION "${CMAKE_INSTALL_PREFIX}" - FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ + USE_SOURCE_PERMISSIONS COMPONENT sdl_atf) install(FILES "${CMAKE_SOURCE_DIR}/start.sh" diff --git a/remote_atf_docker/Dockerfile b/remote_atf_docker/Dockerfile index 139594e9..478341c0 100644 --- a/remote_atf_docker/Dockerfile +++ b/remote_atf_docker/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && apt-get -q -y install sqlite3 libqt5websockets5 net-tools qttranslations5-l10n- xdg-user-dirs- xml-core- dbus- RUN apt-get update && apt install -y netcat cpipe gnuplot sysstat python3-pip docker.io -RUN apt-get update && apt install -y python3-numpy python3-scipy python3-matplotlib +RUN apt-get update && apt install -y python3-numpy python3-scipy python3-matplotlib RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 diff --git a/remote_atf_docker/entrypoint.sh b/remote_atf_docker/entrypoint.sh old mode 100644 new mode 100755 diff --git a/remote_atf_docker/start_remote_server.sh b/remote_atf_docker/start_remote_server.sh index 4ba69191..320ea855 100755 --- a/remote_atf_docker/start_remote_server.sh +++ b/remote_atf_docker/start_remote_server.sh @@ -2,6 +2,21 @@ PATH_TO_SDLBIN=$1 PATH_TO_ATFBIN=$2 PATH_TO_THIRDPARTY=$3 +if [ -z "$PATH_TO_SDLBIN" ]; then + echo "Path to SDL binaries is not defined as 1st argument" + exit 1; +fi + +if [ -z "$PATH_TO_ATFBIN" ]; then + echo "Path to ATF binaries is not defined as 2nd argument" + exit 1; +fi + +if [ -z "$PATH_TO_THIRDPARTY" ]; then + echo "Path to 3rd party libraries is not defined as 3rd argument" + exit 1; +fi + CONTAINER_NAME=remote_sdl IMAGE_NAME=remote_atf_server MEMORY_CONSTRAINS=4G From 1256c78793bad1ae5d74b54066fee74b3a9fbaff Mon Sep 17 00:00:00 2001 From: a-kutsan Date: Tue, 21 Apr 2020 13:27:03 +0300 Subject: [PATCH 10/12] Update measure_sdl.sh --- tools/measure_sdl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/measure_sdl.sh b/tools/measure_sdl.sh index d4cb5cab..a545521b 100755 --- a/tools/measure_sdl.sh +++ b/tools/measure_sdl.sh @@ -8,6 +8,7 @@ OUTPUT_DIR="measure"/$1 OUTPUT_PIDSTAT_FILE=$OUTPUT_DIR/pidstat.log OUTPUT_PS_FILE=$OUTPUT_DIR/ps.log OUTPUT_DOCKER_FILE=$OUTPUT_DIR/docker.log +CONTAINTER_NAME=$(docker inspect --format='{{.Name}}' $HOSTNAME | cut -c2-) STEP=1 @@ -34,7 +35,7 @@ function measure_pid_stat() { } function measure_docker() { - RECORD=$(docker stats --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}" --no-stream | grep remote_sdl) + RECORD=$(docker stats --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}" --no-stream | grep $CONTAINTER_NAME) echo $RECORD TIME=$(date +'%T:%N') echo $TIME $RECORD >> $OUTPUT_DOCKER_FILE From 46ea68493a323a682f128864e89ce96e864b6012 Mon Sep 17 00:00:00 2001 From: "Mykyta Nikora (GitHub)" <50582435+MNikora@users.noreply.github.com> Date: Mon, 27 Apr 2020 14:09:14 +0300 Subject: [PATCH 11/12] Update sdl_graphs.py --- tools/sdl_graphs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdl_graphs.py b/tools/sdl_graphs.py index 2feaaf9c..835fc4cd 100755 --- a/tools/sdl_graphs.py +++ b/tools/sdl_graphs.py @@ -169,7 +169,7 @@ def main(): records = read_records(args.pidstat_file, parse_pidstat, sdl_start_time_pidstat) process_records(records, "pidstat", args.title + "\npidstat -urdlv -h", - os.path.join(args.output_dir , "pidstat")) + os.path.join(args.output_dir , re.sub("\D", "", args.output_dir))) if args.ps_file: print("ps log file : {}".format(args.ps_file)) From 7be57c472895f43f134908dca358ec0d7c4fa74a Mon Sep 17 00:00:00 2001 From: "Mykyta Nikora (GitHub)" <50582435+MNikora@users.noreply.github.com> Date: Mon, 27 Apr 2020 15:46:30 +0300 Subject: [PATCH 12/12] Update sdl_graphs.py Remove percentage signs, replace extra word to number in file names. --- tools/sdl_graphs.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/sdl_graphs.py b/tools/sdl_graphs.py index 835fc4cd..526d56da 100755 --- a/tools/sdl_graphs.py +++ b/tools/sdl_graphs.py @@ -29,10 +29,10 @@ def parse_ps(line): record = {} record["time"] = datetime.strptime(line[0][:-3], "%H:%M:%S:%f") record["start"] = datetime.strptime(line[1], "%H:%M:%S") - record["%cpu"] = float(line[2]) + record["pcpu"] = float(line[2]) record["cp"] = float(line[3]) record["cputime"] = datetime.strptime(line[4], "%H:%M:%S") - record["%mem"] = float(line[5]) + record["pmem"] = float(line[5]) record["sz"] = float(line[6]) record["thcount"] = int(line[7]) return record @@ -61,9 +61,9 @@ def parse_docker(line): def graphs(record_type): if record_type == "ps": - return ["%cpu", "%mem", "sz", "thcount"] + return ["pcpu", "pmem", "sz", "thcount"] if record_type == "pidstat": - return ["%CPU", "VSZ", "RSS", "%MEM", "threads"] + return ["pCPU", "VSZ", "RSS", "pMEM", "threads"] if record_type == "docker": return ["CPUPerc", "MemUsage", "PIDs"] raise "Unknown record_type {}".format(record_type) @@ -80,17 +80,17 @@ def parse_pidstat(line): record["time"] = datetime.strptime(next_col(), "%H:%M:%S") uid = next_col() pid = next_col() - record["%usr"] = float(next_col()) - record["%system"] = float(next_col()) - record["%guest"] = float(next_col()) - record["%wait"] = float(next_col()) - record["%CPU"] = float(next_col()) + record["pusr"] = float(next_col()) + record["psystem"] = float(next_col()) + record["pguest"] = float(next_col()) + record["pwait"] = float(next_col()) + record["pCPU"] = float(next_col()) record["CPU"] = int(next_col()) record["minflt/s"] = float(next_col()) record["majflt/s"] = float(next_col()) record["VSZ"] = int(next_col()) record["RSS"] = int(next_col()) - record["%MEM"] = float(next_col()) + record["pMEM"] = float(next_col()) record["kB_rd/s"] = float(next_col()) record["kB_wr/s"] = float(next_col()) record["kB_ccwr/s"] = float(next_col()) @@ -176,13 +176,13 @@ def main(): records = read_records(args.ps_file, parse_ps, sdl_start_time_ps) process_records(records, "ps", args.title + '\nps --no-headers --format "start %cpu cp cputime %mem sz thcount"', - os.path.join(args.output_dir , "ps")) + os.path.join(args.output_dir , re.sub("\D", "", args.output_dir))) if args.docker_file: print("docker log file : {}".format(args.docker_file)) records = read_records(args.docker_file, parse_docker, sdl_start_time_docker) process_records(records, "docker", args.title + '\n{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.PIDs}}', - os.path.join(args.output_dir , "docker")) + os.path.join(args.output_dir , re.sub("\D", "", args.output_dir))) print("Done")