From a9522708fcb6dba9f0f0708d47e4415f54abc1be Mon Sep 17 00:00:00 2001 From: Vasiliev Evgenii Date: Mon, 13 Nov 2023 14:19:40 +0300 Subject: [PATCH 1/3] Create jenkins_pipeline.groovy --- src/deployment/jenkins_pipeline.groovy | 76 ++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/deployment/jenkins_pipeline.groovy diff --git a/src/deployment/jenkins_pipeline.groovy b/src/deployment/jenkins_pipeline.groovy new file mode 100644 index 000000000..7309f125e --- /dev/null +++ b/src/deployment/jenkins_pipeline.groovy @@ -0,0 +1,76 @@ +node { + + def workersIP = ['ip_1','ip_2'] + def deploy_config = ''' + + + ip1 + user_login + user_password + Linux + /home/user_login/dli-jenkins-worker + /mnt/datasets + /mnt/models + + + ip2 + user_login + user_password + Linux + /home/user_login/dli-jenkins-worker + /mnt/datasets + /mnt/models + +''' + + + + stage('Clone repository') { + git url: 'https://github.com/itlab-vision/dl-benchmark.git', branch: 'master' + } + stage('Build docker') { + sh 'docker build -t ubuntu_for_dli --build-arg DATASET_DOWNLOAD_LINK="https://:@github.com/.git" docker/' + sh 'docker build -t openvino:2022.3 docker/OpenVINO_DLDT' + sh 'docker save openvino:2022.3 > openvino_2022.3.tar' + } + stage('Prepare workers') { + workersIP.each { item -> + def remote = [:] + remote.name = '' + remote.host = item + remote.user = '' + remote.password = '' + remote.allowAnyHosts = true + remote.pty = true + sshCommand remote: remote, command: "hostname -I" + sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker && cd /home//dli-jenkins-worker" + sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"dl-benchmark" pull || git clone https://github.com/itlab-vision/dl-benchmark.git --depth 1 dl-benchmark' + sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"open_model_zoo" pull || git clone https://github.com/openvinotoolkit/open_model_zoo.git --recursive --branch 2022.3.0 --single-branch --depth 1 open_model_zoo' + sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker/results" + + //Mount shared folder with models and datasets + sshCommand remote: remote, command: 'mountpoint -q /mnt && echo "Directory already mounted" || sudo mount -t cifs -o username=,password= ///linuxshare /mnt', sudo: true, pty:true + } + } + stage('Deploy docker on workers') { + sh 'touch deploy_config.xml' + sh 'echo ${deploy_config} >> deploy_config.xml' + sh 'python3 src/deployment/deploy.py -s -l -p -i ./openvino_2022.3.tar -d /home//ftp -n OpenVINO_DLDT --machine_list ./deploy_config.xml --project_folder /home//dli-jenkins-worker/dl-benchmark' + + } + stage('Remote start') { + + def remote = [:] + remote.name = 'ftp_login' + remote.host = 'ftp_ip' + remote.user = 'ftp_login' + remote.password = 'ftp_password' + remote.allowAnyHosts = true + remote.pty = true + + sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" + sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" + } + stage('Convert results') { + } +} From 1960372e2b632b118e38e00f6d17976f67d2f999 Mon Sep 17 00:00:00 2001 From: Vasiliev Evgenii Date: Wed, 22 Nov 2023 13:57:28 +0300 Subject: [PATCH 2/3] Move script to jenkins dir --- src/deployment/{ => jenkins}/jenkins_pipeline.groovy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/deployment/{ => jenkins}/jenkins_pipeline.groovy (100%) diff --git a/src/deployment/jenkins_pipeline.groovy b/src/deployment/jenkins/jenkins_pipeline.groovy similarity index 100% rename from src/deployment/jenkins_pipeline.groovy rename to src/deployment/jenkins/jenkins_pipeline.groovy From 57524118ec4172d0c31711d86870382b7bbb3970 Mon Sep 17 00:00:00 2001 From: Vasiliev Evgenii Date: Sun, 24 Dec 2023 17:21:40 +0300 Subject: [PATCH 3/3] Fix code formatting --- .../jenkins/jenkins_pipeline.groovy | 136 +++++++++--------- 1 file changed, 66 insertions(+), 70 deletions(-) diff --git a/src/deployment/jenkins/jenkins_pipeline.groovy b/src/deployment/jenkins/jenkins_pipeline.groovy index 7309f125e..1cc3059e8 100644 --- a/src/deployment/jenkins/jenkins_pipeline.groovy +++ b/src/deployment/jenkins/jenkins_pipeline.groovy @@ -1,76 +1,72 @@ node { - - def workersIP = ['ip_1','ip_2'] - def deploy_config = ''' + def workersIP = ['ip_1', 'ip_2'] + def deploy_config = '''\ + - - ip1 - user_login - user_password - Linux - /home/user_login/dli-jenkins-worker - /mnt/datasets - /mnt/models - - ip2 - user_login - user_password - Linux - /home/user_login/dli-jenkins-worker - /mnt/datasets - /mnt/models - + ip1 + user_login + user_password + Linux + /home/user_login/dli-jenkins-worker + /mnt/datasets + /mnt/models + + + ip2 + user_login + user_password + Linux + /home/user_login/dli-jenkins-worker + /mnt/datasets + /mnt/models + ''' - - - - stage('Clone repository') { - git url: 'https://github.com/itlab-vision/dl-benchmark.git', branch: 'master' - } - stage('Build docker') { - sh 'docker build -t ubuntu_for_dli --build-arg DATASET_DOWNLOAD_LINK="https://:@github.com/.git" docker/' - sh 'docker build -t openvino:2022.3 docker/OpenVINO_DLDT' - sh 'docker save openvino:2022.3 > openvino_2022.3.tar' - } - stage('Prepare workers') { - workersIP.each { item -> - def remote = [:] - remote.name = '' - remote.host = item - remote.user = '' - remote.password = '' - remote.allowAnyHosts = true - remote.pty = true - sshCommand remote: remote, command: "hostname -I" - sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker && cd /home//dli-jenkins-worker" - sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"dl-benchmark" pull || git clone https://github.com/itlab-vision/dl-benchmark.git --depth 1 dl-benchmark' - sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"open_model_zoo" pull || git clone https://github.com/openvinotoolkit/open_model_zoo.git --recursive --branch 2022.3.0 --single-branch --depth 1 open_model_zoo' - sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker/results" - - //Mount shared folder with models and datasets - sshCommand remote: remote, command: 'mountpoint -q /mnt && echo "Directory already mounted" || sudo mount -t cifs -o username=,password= ///linuxshare /mnt', sudo: true, pty:true - } - } - stage('Deploy docker on workers') { - sh 'touch deploy_config.xml' - sh 'echo ${deploy_config} >> deploy_config.xml' - sh 'python3 src/deployment/deploy.py -s -l -p -i ./openvino_2022.3.tar -d /home//ftp -n OpenVINO_DLDT --machine_list ./deploy_config.xml --project_folder /home//dli-jenkins-worker/dl-benchmark' - } - stage('Remote start') { - - def remote = [:] - remote.name = 'ftp_login' - remote.host = 'ftp_ip' - remote.user = 'ftp_login' - remote.password = 'ftp_password' - remote.allowAnyHosts = true - remote.pty = true - - sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" - sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" - } - stage('Convert results') { - } + stage('Clone repository') { + git url: 'https://github.com/itlab-vision/dl-benchmark.git', branch: 'master' + } + stage('Build docker') { + sh 'docker build -t ubuntu_for_dli --build-arg DATASET_DOWNLOAD_LINK="https://:@github.com/.git" docker/' + sh 'docker build -t openvino:2022.3 docker/OpenVINO_DLDT' + sh 'docker save openvino:2022.3 > openvino_2022.3.tar' + } + stage('Prepare workers') { + workersIP.each { + item -> + def remote = [: ] + remote.name = '' + remote.host = item + remote.user = '' + remote.password = '' + remote.allowAnyHosts = true + remote.pty = true + sshCommand remote: remote, command: "hostname -I" + sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker && cd /home//dli-jenkins-worker" + sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"dl-benchmark" pull || git clone https://github.com/itlab-vision/dl-benchmark.git --depth 1 dl-benchmark' + sshCommand remote: remote, command: 'cd /home//dli-jenkins-worker && git -C $"open_model_zoo" pull || git clone https://github.com/openvinotoolkit/open_model_zoo.git --recursive --branch 2022.3.0 --single-branch --depth 1 open_model_zoo' + sshCommand remote: remote, command: "mkdir -p /home//dli-jenkins-worker/results" + + //Mount shared folder with models and datasets + sshCommand remote: remote, command: 'mountpoint -q /mnt && echo "Directory already mounted" || sudo mount -t cifs -o username=,password= ///linuxshare /mnt', sudo: true, pty: true + } + } + stage('Deploy docker on workers') { + sh 'touch deploy_config.xml' + sh 'echo ${deploy_config} >> deploy_config.xml' + sh 'python3 src/deployment/deploy.py -s -l -p -i ./openvino_2022.3.tar -d /home//ftp -n OpenVINO_DLDT --machine_list ./deploy_config.xml --project_folder /home//dli-jenkins-worker/dl-benchmark' + } + stage('Remote start') { + def remote = [: ] + remote.name = 'ftp_login' + remote.host = 'ftp_ip' + remote.user = 'ftp_login' + remote.password = 'ftp_password' + remote.allowAnyHosts = true + remote.pty = true + + sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" + sshCommand remote: remote, command: "python3 /home//ftp/dl-benchmark/src/remote_control/remote_start.py -c /home//ftp/jenkins_remote_configs/openvino_2022.3/config.xml -s -l -p -acr accuracy_checker_results.csv -br benchmark_results.csv --ftp_dir /home//ftp/jenkins_results" + } + stage('Convert results') {} }