From 815c45cdcc230c5e23416d731b64724a00e2866d Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Thu, 31 Aug 2023 11:41:49 +0300 Subject: [PATCH 01/37] Use python 3 for Cloud-Pipeline-Distribution GitHub action --- .github/workflows/main.yml | 4 ++-- deploy/github_actions/gha_pack_dist.sh | 4 +--- pipe-cli/build_mac.sh | 14 +++++++------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd90c9641a..6ceae8a949 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '2.7' + python-version: '3.10' architecture: 'x64' - uses: aws-actions/configure-aws-credentials@v1 with: @@ -45,7 +45,7 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '2.7.18' + python-version: '3.10' architecture: 'x64' - uses: actions/setup-node@v3 with: diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index ad8d75cc64..4aa5885c5e 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -44,9 +44,7 @@ done # Fail script if any command will fail set -e -curl https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/pip/2.7/get-pip.py | python2 - - -python2 -m pip install mkdocs +python -m pip install mkdocs API_STATIC_PATH=api/src/main/resources/static rm -rf ${API_STATIC_PATH}/* diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index efeecf5366..bf09155324 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -21,21 +21,21 @@ mkdir -p $PYINSTALLER_PATH cd $PYINSTALLER_PATH git clone --branch resolve_tmpdir https://github.com/mzueva/pyinstaller.git cd pyinstaller/bootloader/ -python2 ./waf all +python ./waf all cd - ### # Setup common dependencies ### -python2 -m pip install macholib -python2 -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt +python -m pip install macholib +python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt ### # Build pipe fuse ### -python2 -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt +python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt cd $PIPE_MOUNT_SOURCES_DIR && \ -python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ +python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ --paths "${PIPE_CLI_SOURCES_DIR}" \ --hidden-import=UserList \ --hidden-import=UserString \ @@ -70,7 +70,7 @@ git clone https://github.com/sidoruka/ntlmaps.git && \ cd ntlmaps && \ git checkout 5f798a88369eddbe732364b98fbd445aacc809d0 -python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ +python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ main.py -y \ --clean \ --distpath /tmp/ntlmaps/dist \ @@ -101,7 +101,7 @@ function build_pipe { local pipe_commit_hash=$(git log --pretty=tformat:"%H" -n1 .) echo "__component_version__='$pipe_commit_hash'" >> $version_file - python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ + python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ --hidden-import=UserList \ --hidden-import=UserString \ From 733e35e18b808012ec68e1c157e3fef3b54a16fb Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 1 Sep 2023 13:52:09 +0300 Subject: [PATCH 02/37] Use python 3.9 for Cloud-Pipeline-Distribution GitHub action --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ceae8a949..025ddddc91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.9' architecture: 'x64' - uses: aws-actions/configure-aws-credentials@v1 with: @@ -45,7 +45,7 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.9' architecture: 'x64' - uses: actions/setup-node@v3 with: From 732b17bcef48fd1548d8b9dbba1b15a842965936 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Thu, 14 Sep 2023 11:05:37 +0300 Subject: [PATCH 03/37] Exclude ntlmaps from pipe macos build --- pipe-cli/build_mac.sh | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index bf09155324..d61470d29a 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -62,24 +62,6 @@ python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ chmod +x /tmp/mount/dist/pipe-fuse/pipe-fuse -### -# Build ntlm proxy -### -cd /tmp -git clone https://github.com/sidoruka/ntlmaps.git && \ -cd ntlmaps && \ -git checkout 5f798a88369eddbe732364b98fbd445aacc809d0 - -python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ - main.py -y \ - --clean \ - --distpath /tmp/ntlmaps/dist \ - -p /tmp/ntlmaps/lib \ - --add-data ./server.cfg:./ \ - --name ntlmaps - -chmod +x /tmp/ntlmaps/dist/ntlmaps/ntlmaps - ### # Build pipe ### @@ -124,7 +106,6 @@ function build_pipe { -y \ --clean \ --distpath $distpath \ - --add-data /tmp/ntlmaps/dist/ntlmaps:ntlmaps \ --add-data /tmp/mount/dist/pipe-fuse:mount \ ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile } From dd6fa2406d39aca91b37d1170349a0e2872e7f48 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Thu, 14 Sep 2023 12:04:17 +0300 Subject: [PATCH 04/37] Use python 3 for pipe macos build --- pipe-cli/build_mac.sh | 25 +++++++++++------------- pipe-cli/hooks/hook-prettytable.py | 17 ++++++++++++++++ pipe-cli/mount/hooks/hook-prettytable.py | 17 ++++++++++++++++ 3 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 pipe-cli/hooks/hook-prettytable.py create mode 100644 pipe-cli/mount/hooks/hook-prettytable.py diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index d61470d29a..101bad462c 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -17,17 +17,12 @@ ### # Setup Pyinstaller ### -mkdir -p $PYINSTALLER_PATH -cd $PYINSTALLER_PATH -git clone --branch resolve_tmpdir https://github.com/mzueva/pyinstaller.git -cd pyinstaller/bootloader/ -python ./waf all -cd - +python -m pip install pyinstaller==5.13.2 ### # Setup common dependencies ### -python -m pip install macholib +python -m pip install macholib==1.16.2 python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt ### @@ -35,8 +30,9 @@ python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt ### python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt cd $PIPE_MOUNT_SOURCES_DIR && \ -python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ - --paths "${PIPE_CLI_SOURCES_DIR}" \ +pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ + --paths "$PIPE_MOUNT_SOURCES_DIR" \ --hidden-import=UserList \ --hidden-import=UserString \ --hidden-import=commands \ @@ -54,7 +50,7 @@ python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -70,7 +66,7 @@ function build_pipe { local onefile="$2" version_file="${PIPE_CLI_SOURCES_DIR}/src/version.py" - sed -i.bkp '/__bundle_info__/d' $version_file + sed -i'.bkp' '/__bundle_info__/d' $version_file bundle_type="one-folder" [ "$onefile" ] && bundle_type="one-file" @@ -79,12 +75,12 @@ function build_pipe { echo "__bundle_info__ = { 'bundle_type': '$bundle_type', 'build_os_id': 'macos', 'build_os_version_id': '$build_os_version_id' }" >> $version_file cd $PIPE_CLI_SOURCES_DIR - sed -i '/__component_version__/d' $version_file + sed -i'.bkp' '/__component_version__/d' $version_file local pipe_commit_hash=$(git log --pretty=tformat:"%H" -n1 .) echo "__component_version__='$pipe_commit_hash'" >> $version_file - python $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ - --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ + pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ --hidden-import=UserList \ --hidden-import=UserString \ --hidden-import=commands \ @@ -107,6 +103,7 @@ function build_pipe { --clean \ --distpath $distpath \ --add-data /tmp/mount/dist/pipe-fuse:mount \ + --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile } build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file --onefile diff --git a/pipe-cli/hooks/hook-prettytable.py b/pipe-cli/hooks/hook-prettytable.py new file mode 100644 index 0000000000..3082c779d0 --- /dev/null +++ b/pipe-cli/hooks/hook-prettytable.py @@ -0,0 +1,17 @@ +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from PyInstaller.utils.hooks import collect_all + +datas, binaries, hiddenimports = collect_all('PTable') diff --git a/pipe-cli/mount/hooks/hook-prettytable.py b/pipe-cli/mount/hooks/hook-prettytable.py new file mode 100644 index 0000000000..3082c779d0 --- /dev/null +++ b/pipe-cli/mount/hooks/hook-prettytable.py @@ -0,0 +1,17 @@ +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from PyInstaller.utils.hooks import collect_all + +datas, binaries, hiddenimports = collect_all('PTable') From ca748baa00891a33d46bc38e18504e2aacf4c4e0 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Thu, 14 Sep 2023 12:05:30 +0300 Subject: [PATCH 05/37] Clean pyinstaller resources in pipe cli module clean task --- pipe-cli/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pipe-cli/build.gradle b/pipe-cli/build.gradle index e61335455e..9e9edc30db 100644 --- a/pipe-cli/build.gradle +++ b/pipe-cli/build.gradle @@ -1,7 +1,13 @@ task clean(type: Delete) { group = 'build' + delete("$project.rootDir/pipe-cli/build") + delete("$project.rootDir/pipe-cli/__pycache__") + delete("$project.rootDir/pipe-cli/pipe.spec") delete("$project.rootDir/pipe-cli/dist") + delete("$project.rootDir/pipe-cli/mount/build") + delete("$project.rootDir/pipe-cli/mount/__pycache__") + delete("$project.rootDir/pipe-cli/mount/pipe-fuse.spec") } task build(type: Exec, dependsOn: ':pipe-cli:test') { From 6f873727316e1c1d6d43aabebace70d66f26997a Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Thu, 14 Sep 2023 12:23:02 +0300 Subject: [PATCH 06/37] Publish all pipe macos builds --- deploy/github_actions/gha_build_macos.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy/github_actions/gha_build_macos.sh b/deploy/github_actions/gha_build_macos.sh index a4aec8b877..e87934ceb4 100644 --- a/deploy/github_actions/gha_build_macos.sh +++ b/deploy/github_actions/gha_build_macos.sh @@ -28,7 +28,5 @@ cd pipe-cli DIST_TGZ_NAME=pipe-osx-full.$CLOUD_PIPELINE_BUILD_NUMBER.tar.gz tar -zcf $DIST_TGZ_NAME dist if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - if [ "$GITHUB_REF_NAME" == "develop" ] || [ "$GITHUB_REF_NAME" == "master" ] || [[ "$GITHUB_REF_NAME" == "release/"* ]] || [[ "$GITHUB_REF_NAME" == "stage/"* ]]; then - aws s3 cp $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/temp/ - fi + aws s3 cp $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/temp/ fi From 7a0c6837c25e06a46e5f21e603c7b63df1b334a2 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 10:46:27 +0300 Subject: [PATCH 07/37] Use prettytable hooks for macos build only --- pipe-cli/build_linux.sh | 4 ++-- pipe-cli/build_mac.sh | 6 ++++-- pipe-cli/build_windows.sh | 4 ++-- .../{ => python}/hook-google.resumable_media.requests.py | 0 pipe-cli/hooks/{ => python39}/hook-prettytable.py | 0 .../{ => python}/hook-google.resumable_media.requests.py | 0 pipe-cli/mount/hooks/{ => python39}/hook-prettytable.py | 0 7 files changed, 8 insertions(+), 6 deletions(-) rename pipe-cli/hooks/{ => python}/hook-google.resumable_media.requests.py (100%) rename pipe-cli/hooks/{ => python39}/hook-prettytable.py (100%) rename pipe-cli/mount/hooks/{ => python}/hook-google.resumable_media.requests.py (100%) rename pipe-cli/mount/hooks/{ => python39}/hook-prettytable.py (100%) diff --git a/pipe-cli/build_linux.sh b/pipe-cli/build_linux.sh index ed8ed85dfa..0e776a4e1b 100644 --- a/pipe-cli/build_linux.sh +++ b/pipe-cli/build_linux.sh @@ -80,7 +80,7 @@ python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ --hidden-import=functools \ --hidden-import=re \ --hidden-import=subprocess \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks/python" \ -y \ --clean \ --runtime-tmpdir $PIPE_CLI_RUNTIME_TMP_DIR \ @@ -145,7 +145,7 @@ function build_pipe { --hidden-import=functools \ --hidden-import=re \ --hidden-import=subprocess \ - --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks" \ + --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python" \ -y \ --clean \ --runtime-tmpdir $PIPE_CLI_RUNTIME_TMP_DIR \ diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index 101bad462c..c35f7b821d 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -50,7 +50,8 @@ pyinstaller \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks" \ + --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks/python" \ + --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks/python39" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -98,7 +99,8 @@ function build_pipe { --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks" \ + --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python" \ + --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python39" \ -y \ --clean \ --distpath $distpath \ diff --git a/pipe-cli/build_windows.sh b/pipe-cli/build_windows.sh index 2d3a958689..8269a1e24e 100644 --- a/pipe-cli/build_windows.sh +++ b/pipe-cli/build_windows.sh @@ -115,7 +115,7 @@ pyinstaller --paths "/pipe-cli" \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=pkg_resources.py2_warn \ - --additional-hooks-dir="/pipe-cli/mount/hooks" \ + --additional-hooks-dir="/pipe-cli/mount/hooks/python" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -143,7 +143,7 @@ pyinstaller --add-data "/pipe-cli/res/effective_tld_names.dat.txt;tld/res/" \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=pkg_resources.py2_warn \ - --additional-hooks-dir="/pipe-cli/hooks" \ + --additional-hooks-dir="/pipe-cli/hooks/python" \ -y \ --clean \ --workpath /tmp \ diff --git a/pipe-cli/hooks/hook-google.resumable_media.requests.py b/pipe-cli/hooks/python/hook-google.resumable_media.requests.py similarity index 100% rename from pipe-cli/hooks/hook-google.resumable_media.requests.py rename to pipe-cli/hooks/python/hook-google.resumable_media.requests.py diff --git a/pipe-cli/hooks/hook-prettytable.py b/pipe-cli/hooks/python39/hook-prettytable.py similarity index 100% rename from pipe-cli/hooks/hook-prettytable.py rename to pipe-cli/hooks/python39/hook-prettytable.py diff --git a/pipe-cli/mount/hooks/hook-google.resumable_media.requests.py b/pipe-cli/mount/hooks/python/hook-google.resumable_media.requests.py similarity index 100% rename from pipe-cli/mount/hooks/hook-google.resumable_media.requests.py rename to pipe-cli/mount/hooks/python/hook-google.resumable_media.requests.py diff --git a/pipe-cli/mount/hooks/hook-prettytable.py b/pipe-cli/mount/hooks/python39/hook-prettytable.py similarity index 100% rename from pipe-cli/mount/hooks/hook-prettytable.py rename to pipe-cli/mount/hooks/python39/hook-prettytable.py From 5eb61b4591973130ffab221a66cc92624e07ce09 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 11:55:23 +0300 Subject: [PATCH 08/37] Publish all cloud pipeline distributions --- deploy/github_actions/gha_pack_dist.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 4aa5885c5e..cd1d82efbc 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -73,8 +73,5 @@ if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) # Publish repackaged distribution tgz to S3 into builds/ prefix - # Only if it is one of the allowed branches and it is a push (not PR) - if [ "$GITHUB_REF_NAME" == "develop" ] || [ "$GITHUB_REF_NAME" == "master" ] || [[ "$GITHUB_REF_NAME" == "release/"* ]] || [[ "$GITHUB_REF_NAME" == "stage/"* ]]; then - aws s3 cp --quiet $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ - fi + aws s3 cp --quiet $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ fi From 4cc5ee2812e32105f45f77dcd1043c2fe677eea5 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 12:03:36 +0300 Subject: [PATCH 09/37] Add temporary debug logs --- build.gradle | 7 +++++++ deploy/github_actions/gha_pack_dist.sh | 3 +++ 2 files changed, 10 insertions(+) diff --git a/build.gradle b/build.gradle index ca015b2f0b..6223d7edb5 100644 --- a/build.gradle +++ b/build.gradle @@ -173,6 +173,13 @@ task installDist() { distFiles.each { distFile -> + def command = "ls -la ${distFile.toString()}" + def proc = command.execute() + proc.waitFor() + println "$command" + println "exit code: ${proc.exitValue()}" + println "stderr: ${proc.err.text}" + println "stdout: ${proc.in.text}" copy { from distFile into distFolder diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index cd1d82efbc..92898da62d 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -63,6 +63,9 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz +ls -la "/home/runner/work/cloud-pipeline" +ls -la "/home/runner/work/cloud-pipeline/cloud-pipeline" + ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ From ce4ee83cd9de9c5690ebba1fd8b0d6a33a849d3c Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 15:46:50 +0300 Subject: [PATCH 10/37] Add temporary debug logs --- build.gradle | 2 +- deploy/github_actions/gha_pack_dist.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 6223d7edb5..fa4c8e62c6 100644 --- a/build.gradle +++ b/build.gradle @@ -173,7 +173,7 @@ task installDist() { distFiles.each { distFile -> - def command = "ls -la ${distFile.toString()}" + def command = "df -hT /home/runner/work/cloud-pipeline/cloud-pipeline" def proc = command.execute() proc.waitFor() println "$command" diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 92898da62d..ba7afc9007 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -63,14 +63,15 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz -ls -la "/home/runner/work/cloud-pipeline" -ls -la "/home/runner/work/cloud-pipeline/cloud-pipeline" +df -hT "/home/runner/work/cloud-pipeline/cloud-pipeline" ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ -Pfast \ - --no-daemon + --no-daemon \ + --debug \ + --stacktrace if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) From 4c8c6afb9c557a307501f6dbd374688b585c6a28 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 16:38:34 +0300 Subject: [PATCH 11/37] Add temporary debug logs --- build.gradle | 7 ------- deploy/github_actions/gha_pack_dist.sh | 6 ++---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index fa4c8e62c6..ca015b2f0b 100644 --- a/build.gradle +++ b/build.gradle @@ -173,13 +173,6 @@ task installDist() { distFiles.each { distFile -> - def command = "df -hT /home/runner/work/cloud-pipeline/cloud-pipeline" - def proc = command.execute() - proc.waitFor() - println "$command" - println "exit code: ${proc.exitValue()}" - println "stderr: ${proc.err.text}" - println "stdout: ${proc.in.text}" copy { from distFile into distFolder diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index ba7afc9007..1cd1921db3 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -63,15 +63,13 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz -df -hT "/home/runner/work/cloud-pipeline/cloud-pipeline" +du -sh "/home/runner/work/cloud-pipeline/cloud-pipeline" ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ -Pfast \ - --no-daemon \ - --debug \ - --stacktrace + --no-daemon if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) From 4deed8ecf5adfb3c141c3c63aff4dce8dff9e2da Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 15 Sep 2023 16:58:04 +0300 Subject: [PATCH 12/37] Add temporary debug logs --- deploy/github_actions/gha_pack_dist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 1cd1921db3..d2e026ea00 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -63,7 +63,7 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz -du -sh "/home/runner/work/cloud-pipeline/cloud-pipeline" +du -sh "/home/runner/work/cloud-pipeline/cloud-pipeline/*" ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ From 69cc2739db4124655327c08bd2f67313256c5ef8 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 10:20:58 +0300 Subject: [PATCH 13/37] Add temporary debug logs --- build.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.gradle b/build.gradle index ca015b2f0b..6bb500e51e 100644 --- a/build.gradle +++ b/build.gradle @@ -173,6 +173,13 @@ task installDist() { distFiles.each { distFile -> + def command = "ls -lah /home/runner/work/cloud-pipeline/cloud-pipeline" + def proc = command.execute() + proc.waitFor() + println "$command" + println "exit code: ${proc.exitValue()}" + println "stderr: ${proc.err.text}" + println "stdout: ${proc.in.text}" copy { from distFile into distFolder From dacfcc04db6803d4c674d2b00ee34748033eab58 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 10:40:26 +0300 Subject: [PATCH 14/37] Add temporary debug logs --- deploy/github_actions/gha_pack_dist.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index d2e026ea00..cd1d82efbc 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -63,8 +63,6 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz -du -sh "/home/runner/work/cloud-pipeline/cloud-pipeline/*" - ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ From 48c047d95e94355aa6a2cae3038c6100c5ccef56 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 10:43:06 +0300 Subject: [PATCH 15/37] Build pipe for each platform separately --- .github/workflows/main.yml | 58 ++++++++++++++++++- deploy/github_actions/gha_build_pipe_linux.sh | 30 ++++++++++ .../gha_build_pipe_linux_el6.sh | 31 ++++++++++ ...build_macos.sh => gha_build_pipe_macos.sh} | 6 +- .../github_actions/gha_build_pipe_windows.sh | 29 ++++++++++ deploy/github_actions/gha_pack_dist.sh | 21 ++++--- 6 files changed, 158 insertions(+), 17 deletions(-) create mode 100644 deploy/github_actions/gha_build_pipe_linux.sh create mode 100644 deploy/github_actions/gha_build_pipe_linux_el6.sh rename deploy/github_actions/{gha_build_macos.sh => gha_build_pipe_macos.sh} (79%) create mode 100644 deploy/github_actions/gha_build_pipe_windows.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 025ddddc91..a74b9c1cfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,42 @@ env: CLOUD_PIPELINE_BUILD_NUMBER_SEED: 30000 jobs: + CLI_Linux: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_linux.sh + CLI_Linux_el6: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_linux_el6.sh CLI_MacOS: runs-on: macos-11 steps: @@ -33,10 +69,28 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - run: bash deploy/github_actions/gha_build_macos.sh + - run: bash deploy/github_actions/gha_build_pipe_macos.sh + CLI_Windows: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_windows.sh Build_All: runs-on: ubuntu-20.04 - needs: CLI_MacOS + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/deploy/github_actions/gha_build_pipe_linux.sh b/deploy/github_actions/gha_build_pipe_linux.sh new file mode 100644 index 0000000000..2cfe417e68 --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildLinux \ + --no-daemon \ + -x :pipe-cli:test + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp pipe-cli/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe + aws s3 cp pipe-cli/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz +fi diff --git a/deploy/github_actions/gha_build_pipe_linux_el6.sh b/deploy/github_actions/gha_build_pipe_linux_el6.sh new file mode 100644 index 0000000000..6bd3b55265 --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_linux_el6.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +_BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-centos6" \ +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildLinux \ + --no-daemon \ + -x :pipe-cli:test + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp pipe-cli/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 + aws s3 cp pipe-cli/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz +fi diff --git a/deploy/github_actions/gha_build_macos.sh b/deploy/github_actions/gha_build_pipe_macos.sh similarity index 79% rename from deploy/github_actions/gha_build_macos.sh rename to deploy/github_actions/gha_build_pipe_macos.sh index e87934ceb4..238b6ef9e5 100644 --- a/deploy/github_actions/gha_build_macos.sh +++ b/deploy/github_actions/gha_build_pipe_macos.sh @@ -24,9 +24,7 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM --no-daemon \ -x :pipe-cli:test -cd pipe-cli -DIST_TGZ_NAME=pipe-osx-full.$CLOUD_PIPELINE_BUILD_NUMBER.tar.gz -tar -zcf $DIST_TGZ_NAME dist if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/temp/ + aws s3 cp pipe-cli/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx + aws s3 cp pipe-cli/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_windows.sh b/deploy/github_actions/gha_build_pipe_windows.sh new file mode 100644 index 0000000000..90c9615a3c --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_windows.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildWin \ + --no-daemon \ + -x :pipe-cli:test + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp pipe-cli/dist-folder/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index cd1d82efbc..ca8b9921cf 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -51,21 +51,20 @@ rm -rf ${API_STATIC_PATH}/* rm -rf build/install/dist/* mkdir -p ${API_STATIC_PATH} -_OSX_CLI_TAR_NAME=pipe-osx-full.$CLOUD_PIPELINE_BUILD_NUMBER.tar.gz -_OSX_CLI_PATH=$(mktemp -d) -aws s3 cp s3://cloud-pipeline-oss-builds/temp/${_OSX_CLI_TAR_NAME} ${_OSX_CLI_PATH}/ -tar -zxf $_OSX_CLI_PATH/$_OSX_CLI_TAR_NAME -C $_OSX_CLI_PATH - -mv $_OSX_CLI_PATH/dist/dist-file/pipe-osx ${API_STATIC_PATH}/pipe-osx -mv $_OSX_CLI_PATH/dist/dist-folder/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz - -_BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-centos6" ./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} -Pprofile=release pipe-cli:buildLinux --no-daemon -x :pipe-cli:test -mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 -mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe ${API_STATIC_PATH}/pipe +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz ${API_STATIC_PATH}/pipe.tar.gz +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 ${API_STATIC_PATH}/pipe-el6 +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx ${API_STATIC_PATH}/pipe-osx +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz +aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip ${API_STATIC_PATH}/pipe.zip ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildWin \ -Pfast \ --no-daemon From 9ae8e92313b787655f307dc2ea598f43cd5b2123 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 11:02:17 +0300 Subject: [PATCH 16/37] Fix pipe build paths --- deploy/github_actions/gha_build_pipe_linux.sh | 4 ++-- deploy/github_actions/gha_build_pipe_linux_el6.sh | 4 ++-- deploy/github_actions/gha_build_pipe_macos.sh | 4 ++-- deploy/github_actions/gha_build_pipe_windows.sh | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/github_actions/gha_build_pipe_linux.sh b/deploy/github_actions/gha_build_pipe_linux.sh index 2cfe417e68..7de4b4e4c4 100644 --- a/deploy/github_actions/gha_build_pipe_linux.sh +++ b/deploy/github_actions/gha_build_pipe_linux.sh @@ -25,6 +25,6 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM -x :pipe-cli:test if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe - aws s3 cp pipe-cli/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz + aws s3 cp pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe + aws s3 cp pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_linux_el6.sh b/deploy/github_actions/gha_build_pipe_linux_el6.sh index 6bd3b55265..dd62d2660b 100644 --- a/deploy/github_actions/gha_build_pipe_linux_el6.sh +++ b/deploy/github_actions/gha_build_pipe_linux_el6.sh @@ -26,6 +26,6 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c -x :pipe-cli:test if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 - aws s3 cp pipe-cli/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz + aws s3 cp pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 + aws s3 cp pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_macos.sh b/deploy/github_actions/gha_build_pipe_macos.sh index 238b6ef9e5..3319bfd70d 100644 --- a/deploy/github_actions/gha_build_pipe_macos.sh +++ b/deploy/github_actions/gha_build_pipe_macos.sh @@ -25,6 +25,6 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM -x :pipe-cli:test if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx - aws s3 cp pipe-cli/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz + aws s3 cp pipe-cli/dist/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx + aws s3 cp pipe-cli/dist/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_windows.sh b/deploy/github_actions/gha_build_pipe_windows.sh index 90c9615a3c..f89e86d645 100644 --- a/deploy/github_actions/gha_build_pipe_windows.sh +++ b/deploy/github_actions/gha_build_pipe_windows.sh @@ -25,5 +25,5 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM -x :pipe-cli:test if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist-folder/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip + aws s3 cp pipe-cli/dist/dist-folder/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip fi From 04201bb46736b85f56074d79e011f5f89dc9b282 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 11:54:51 +0300 Subject: [PATCH 17/37] Fix pipe build paths --- deploy/github_actions/gha_build_pipe_windows.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/github_actions/gha_build_pipe_windows.sh b/deploy/github_actions/gha_build_pipe_windows.sh index f89e86d645..8bd44340e6 100644 --- a/deploy/github_actions/gha_build_pipe_windows.sh +++ b/deploy/github_actions/gha_build_pipe_windows.sh @@ -25,5 +25,5 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM -x :pipe-cli:test if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist/dist-folder/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip + aws s3 cp pipe-cli/dist/win/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip fi From 20d0943214f3658fb44769a61f233a1725a37f5d Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 18 Sep 2023 12:24:23 +0300 Subject: [PATCH 18/37] Remove temporary debug logs --- build.gradle | 7 ------- 1 file changed, 7 deletions(-) diff --git a/build.gradle b/build.gradle index 6bb500e51e..ca015b2f0b 100644 --- a/build.gradle +++ b/build.gradle @@ -173,13 +173,6 @@ task installDist() { distFiles.each { distFile -> - def command = "ls -lah /home/runner/work/cloud-pipeline/cloud-pipeline" - def proc = command.execute() - proc.waitFor() - println "$command" - println "exit code: ${proc.exitValue()}" - println "stderr: ${proc.err.text}" - println "stdout: ${proc.in.text}" copy { from distFile into distFolder From f9cba961fab11ff26d4b64897ae5a50ef2543058 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 22 Sep 2023 12:25:02 +0300 Subject: [PATCH 19/37] Revert unnecessary changes --- pipe-cli/build.gradle | 6 ------ pipe-cli/build_linux.sh | 4 ++-- pipe-cli/build_mac.sh | 8 ++++---- pipe-cli/build_windows.sh | 4 ++-- .../{hooks/python39 => hooks-py39}/hook-prettytable.py | 0 .../{python => }/hook-google.resumable_media.requests.py | 0 .../{hooks/python39 => hooks-py3}/hook-prettytable.py | 0 .../{python => }/hook-google.resumable_media.requests.py | 0 8 files changed, 8 insertions(+), 14 deletions(-) rename pipe-cli/{hooks/python39 => hooks-py39}/hook-prettytable.py (100%) rename pipe-cli/hooks/{python => }/hook-google.resumable_media.requests.py (100%) rename pipe-cli/mount/{hooks/python39 => hooks-py3}/hook-prettytable.py (100%) rename pipe-cli/mount/hooks/{python => }/hook-google.resumable_media.requests.py (100%) diff --git a/pipe-cli/build.gradle b/pipe-cli/build.gradle index 9e9edc30db..e61335455e 100644 --- a/pipe-cli/build.gradle +++ b/pipe-cli/build.gradle @@ -1,13 +1,7 @@ task clean(type: Delete) { group = 'build' - delete("$project.rootDir/pipe-cli/build") - delete("$project.rootDir/pipe-cli/__pycache__") - delete("$project.rootDir/pipe-cli/pipe.spec") delete("$project.rootDir/pipe-cli/dist") - delete("$project.rootDir/pipe-cli/mount/build") - delete("$project.rootDir/pipe-cli/mount/__pycache__") - delete("$project.rootDir/pipe-cli/mount/pipe-fuse.spec") } task build(type: Exec, dependsOn: ':pipe-cli:test') { diff --git a/pipe-cli/build_linux.sh b/pipe-cli/build_linux.sh index 0e776a4e1b..ed8ed85dfa 100644 --- a/pipe-cli/build_linux.sh +++ b/pipe-cli/build_linux.sh @@ -80,7 +80,7 @@ python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ --hidden-import=functools \ --hidden-import=re \ --hidden-import=subprocess \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks/python" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ -y \ --clean \ --runtime-tmpdir $PIPE_CLI_RUNTIME_TMP_DIR \ @@ -145,7 +145,7 @@ function build_pipe { --hidden-import=functools \ --hidden-import=re \ --hidden-import=subprocess \ - --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python" \ + --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks" \ -y \ --clean \ --runtime-tmpdir $PIPE_CLI_RUNTIME_TMP_DIR \ diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index c35f7b821d..b631bb088c 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -50,8 +50,8 @@ pyinstaller \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks/python" \ - --additional-hooks-dir="$PIPE_MOUNT_SOURCES_DIR/hooks/python39" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -99,8 +99,8 @@ function build_pipe { --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python" \ - --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks/python39" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ -y \ --clean \ --distpath $distpath \ diff --git a/pipe-cli/build_windows.sh b/pipe-cli/build_windows.sh index 8269a1e24e..2d3a958689 100644 --- a/pipe-cli/build_windows.sh +++ b/pipe-cli/build_windows.sh @@ -115,7 +115,7 @@ pyinstaller --paths "/pipe-cli" \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=pkg_resources.py2_warn \ - --additional-hooks-dir="/pipe-cli/mount/hooks/python" \ + --additional-hooks-dir="/pipe-cli/mount/hooks" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -143,7 +143,7 @@ pyinstaller --add-data "/pipe-cli/res/effective_tld_names.dat.txt;tld/res/" \ --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=pkg_resources.py2_warn \ - --additional-hooks-dir="/pipe-cli/hooks/python" \ + --additional-hooks-dir="/pipe-cli/hooks" \ -y \ --clean \ --workpath /tmp \ diff --git a/pipe-cli/hooks/python39/hook-prettytable.py b/pipe-cli/hooks-py39/hook-prettytable.py similarity index 100% rename from pipe-cli/hooks/python39/hook-prettytable.py rename to pipe-cli/hooks-py39/hook-prettytable.py diff --git a/pipe-cli/hooks/python/hook-google.resumable_media.requests.py b/pipe-cli/hooks/hook-google.resumable_media.requests.py similarity index 100% rename from pipe-cli/hooks/python/hook-google.resumable_media.requests.py rename to pipe-cli/hooks/hook-google.resumable_media.requests.py diff --git a/pipe-cli/mount/hooks/python39/hook-prettytable.py b/pipe-cli/mount/hooks-py3/hook-prettytable.py similarity index 100% rename from pipe-cli/mount/hooks/python39/hook-prettytable.py rename to pipe-cli/mount/hooks-py3/hook-prettytable.py diff --git a/pipe-cli/mount/hooks/python/hook-google.resumable_media.requests.py b/pipe-cli/mount/hooks/hook-google.resumable_media.requests.py similarity index 100% rename from pipe-cli/mount/hooks/python/hook-google.resumable_media.requests.py rename to pipe-cli/mount/hooks/hook-google.resumable_media.requests.py From 07c50c23b08ab586fead4197975b2587ff27c05e Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 22 Sep 2023 12:29:16 +0300 Subject: [PATCH 20/37] Use separate scripts to build pipe macos --- deploy/github_actions/gha_build_pipe_macos.sh | 2 +- deploy/github_actions/gha_pack_dist.sh | 1 + pipe-cli/build.gradle | 25 ++++ pipe-cli/build_mac.sh | 50 +++++--- pipe-cli/build_mac_py39.sh | 117 ++++++++++++++++++ 5 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 pipe-cli/build_mac_py39.sh diff --git a/deploy/github_actions/gha_build_pipe_macos.sh b/deploy/github_actions/gha_build_pipe_macos.sh index 3319bfd70d..9ff579e662 100644 --- a/deploy/github_actions/gha_build_pipe_macos.sh +++ b/deploy/github_actions/gha_build_pipe_macos.sh @@ -20,7 +20,7 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM ./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ - pipe-cli:buildMac \ + pipe-cli:buildMacPy39 \ --no-daemon \ -x :pipe-cli:test diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index ca8b9921cf..c0fe648951 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -64,6 +64,7 @@ aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe. -x test \ -x :pipe-cli:buildLinux \ -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ -x :pipe-cli:buildWin \ -Pfast \ --no-daemon diff --git a/pipe-cli/build.gradle b/pipe-cli/build.gradle index e61335455e..3576210ee2 100644 --- a/pipe-cli/build.gradle +++ b/pipe-cli/build.gradle @@ -100,6 +100,31 @@ task buildMac(type: Exec, dependsOn: ':pipe-cli:test') { } } +task buildMacPy39(type: Exec) { + group = 'build' + doFirst { + setCLIVersion() + } + environment "PIPE_CLI_SOURCES_DIR": "$project.rootDir/pipe-cli" + environment "PIPE_CLI_LINUX_DIST_DIR": "$project.rootDir/pipe-cli" + environment "PIPE_MOUNT_SOURCES_DIR": "$project.rootDir/pipe-cli/mount" + environment "PYINSTALLER_PATH": "/tmp/pyinstaller" + + commandLine "bash", "$project.rootDir/pipe-cli/build_mac_py39.sh" + doLast { + cleanCLIVersion() + delete("$project.rootDir/api/src/main/resources/static/pipe_osx") + copy { + from("$project.rootDir/pipe-cli/dist/dist-file/pipe_osx") + into("$project.rootDir/api/src/main/resources/static/") + } + copy { + from("$project.rootDir/pipe-cli/dist/dist-folder/pipe_osx.tar.gz") + into("$project.rootDir/api/src/main/resources/static/") + } + } +} + task test(type: Exec, dependsOn: ':pipe-cli:installPackages') { group = 'test' commandLine "pytest", "-s", "-vvv", "--cov=$project.rootDir/pipe-cli", "--cov-config=$project.rootDir/pipe-cli/tests/cov_config" diff --git a/pipe-cli/build_mac.sh b/pipe-cli/build_mac.sh index b631bb088c..efeecf5366 100644 --- a/pipe-cli/build_mac.sh +++ b/pipe-cli/build_mac.sh @@ -17,22 +17,26 @@ ### # Setup Pyinstaller ### -python -m pip install pyinstaller==5.13.2 +mkdir -p $PYINSTALLER_PATH +cd $PYINSTALLER_PATH +git clone --branch resolve_tmpdir https://github.com/mzueva/pyinstaller.git +cd pyinstaller/bootloader/ +python2 ./waf all +cd - ### # Setup common dependencies ### -python -m pip install macholib==1.16.2 -python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt +python2 -m pip install macholib +python2 -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt ### # Build pipe fuse ### -python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt +python2 -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt cd $PIPE_MOUNT_SOURCES_DIR && \ -pyinstaller \ - --paths "$PIPE_CLI_SOURCES_DIR" \ - --paths "$PIPE_MOUNT_SOURCES_DIR" \ +python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ + --paths "${PIPE_CLI_SOURCES_DIR}" \ --hidden-import=UserList \ --hidden-import=UserString \ --hidden-import=commands \ @@ -51,7 +55,6 @@ pyinstaller \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ -y \ --clean \ --distpath /tmp/mount/dist \ @@ -59,6 +62,24 @@ pyinstaller \ chmod +x /tmp/mount/dist/pipe-fuse/pipe-fuse +### +# Build ntlm proxy +### +cd /tmp +git clone https://github.com/sidoruka/ntlmaps.git && \ +cd ntlmaps && \ +git checkout 5f798a88369eddbe732364b98fbd445aacc809d0 + +python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ + main.py -y \ + --clean \ + --distpath /tmp/ntlmaps/dist \ + -p /tmp/ntlmaps/lib \ + --add-data ./server.cfg:./ \ + --name ntlmaps + +chmod +x /tmp/ntlmaps/dist/ntlmaps/ntlmaps + ### # Build pipe ### @@ -67,7 +88,7 @@ function build_pipe { local onefile="$2" version_file="${PIPE_CLI_SOURCES_DIR}/src/version.py" - sed -i'.bkp' '/__bundle_info__/d' $version_file + sed -i.bkp '/__bundle_info__/d' $version_file bundle_type="one-folder" [ "$onefile" ] && bundle_type="one-file" @@ -76,12 +97,12 @@ function build_pipe { echo "__bundle_info__ = { 'bundle_type': '$bundle_type', 'build_os_id': 'macos', 'build_os_version_id': '$build_os_version_id' }" >> $version_file cd $PIPE_CLI_SOURCES_DIR - sed -i'.bkp' '/__component_version__/d' $version_file + sed -i '/__component_version__/d' $version_file local pipe_commit_hash=$(git log --pretty=tformat:"%H" -n1 .) echo "__component_version__='$pipe_commit_hash'" >> $version_file - pyinstaller \ - --paths "$PIPE_CLI_SOURCES_DIR" \ + python2 $PYINSTALLER_PATH/pyinstaller/pyinstaller.py \ + --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ --hidden-import=UserList \ --hidden-import=UserString \ --hidden-import=commands \ @@ -99,13 +120,12 @@ function build_pipe { --hidden-import=re \ --hidden-import=subprocess \ --hidden-import=_sysconfigdata \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ - --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ + --additional-hooks-dir="$PIPE_CLI_SOURCES_DIR/hooks" \ -y \ --clean \ --distpath $distpath \ + --add-data /tmp/ntlmaps/dist/ntlmaps:ntlmaps \ --add-data /tmp/mount/dist/pipe-fuse:mount \ - --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile } build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file --onefile diff --git a/pipe-cli/build_mac_py39.sh b/pipe-cli/build_mac_py39.sh new file mode 100644 index 0000000000..b631bb088c --- /dev/null +++ b/pipe-cli/build_mac_py39.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Copyright 2017-2021 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### +# Setup Pyinstaller +### +python -m pip install pyinstaller==5.13.2 + +### +# Setup common dependencies +### +python -m pip install macholib==1.16.2 +python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt + +### +# Build pipe fuse +### +python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt +cd $PIPE_MOUNT_SOURCES_DIR && \ +pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ + --paths "$PIPE_MOUNT_SOURCES_DIR" \ + --hidden-import=UserList \ + --hidden-import=UserString \ + --hidden-import=commands \ + --hidden-import=ConfigParser \ + --hidden-import=UserDict \ + --hidden-import=itertools \ + --hidden-import=collections \ + --hidden-import=future.backports.misc \ + --hidden-import=commands \ + --hidden-import=base64 \ + --hidden-import=__builtin__ \ + --hidden-import=math \ + --hidden-import=reprlib \ + --hidden-import=functools \ + --hidden-import=re \ + --hidden-import=subprocess \ + --hidden-import=_sysconfigdata \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ + -y \ + --clean \ + --distpath /tmp/mount/dist \ + ${PIPE_MOUNT_SOURCES_DIR}/pipe-fuse.py + +chmod +x /tmp/mount/dist/pipe-fuse/pipe-fuse + +### +# Build pipe +### +function build_pipe { + local distpath="$1" + local onefile="$2" + + version_file="${PIPE_CLI_SOURCES_DIR}/src/version.py" + sed -i'.bkp' '/__bundle_info__/d' $version_file + + bundle_type="one-folder" + [ "$onefile" ] && bundle_type="one-file" + + build_os_version_id=$(sw_vers -productVersion) + echo "__bundle_info__ = { 'bundle_type': '$bundle_type', 'build_os_id': 'macos', 'build_os_version_id': '$build_os_version_id' }" >> $version_file + + cd $PIPE_CLI_SOURCES_DIR + sed -i'.bkp' '/__component_version__/d' $version_file + local pipe_commit_hash=$(git log --pretty=tformat:"%H" -n1 .) + echo "__component_version__='$pipe_commit_hash'" >> $version_file + + pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ + --hidden-import=UserList \ + --hidden-import=UserString \ + --hidden-import=commands \ + --hidden-import=ConfigParser \ + --hidden-import=UserDict \ + --hidden-import=itertools \ + --hidden-import=collections \ + --hidden-import=future.backports.misc \ + --hidden-import=commands \ + --hidden-import=base64 \ + --hidden-import=__builtin__ \ + --hidden-import=math \ + --hidden-import=reprlib \ + --hidden-import=functools \ + --hidden-import=re \ + --hidden-import=subprocess \ + --hidden-import=_sysconfigdata \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ + -y \ + --clean \ + --distpath $distpath \ + --add-data /tmp/mount/dist/pipe-fuse:mount \ + --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ + ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile +} +build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file --onefile +mv $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe-osx + +build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder +tar -zcf $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder/pipe-osx.tar.gz \ + -C $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder \ + pipe From 5e7e5a889a58092cf3d94cbb11e085af1c24122d Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 22 Sep 2023 13:06:41 +0300 Subject: [PATCH 21/37] Build client separately --- .github/workflows/main.yml | 20 ++++++++++++++- deploy/github_actions/gha_build_client.sh | 30 +++++++++++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 22 +++++++++++------ 3 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 deploy/github_actions/gha_build_client.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a74b9c1cfc..2661a6c4a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,9 +88,27 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_pipe_windows.sh + Build_Client: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_client.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/deploy/github_actions/gha_build_client.sh b/deploy/github_actions/gha_build_client.sh new file mode 100644 index 0000000000..3063d3520c --- /dev/null +++ b/deploy/github_actions/gha_build_client.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + client:buildUI \ + --no-daemon + +tar -czf $PWD/client.tar.gz -C client/build . + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp client.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index c0fe648951..02c660a1ab 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -51,13 +51,20 @@ rm -rf ${API_STATIC_PATH}/* rm -rf build/install/dist/* mkdir -p ${API_STATIC_PATH} -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe ${API_STATIC_PATH}/pipe -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz ${API_STATIC_PATH}/pipe.tar.gz -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 ${API_STATIC_PATH}/pipe-el6 -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx ${API_STATIC_PATH}/pipe-osx -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz -aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip ${API_STATIC_PATH}/pipe.zip +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe ${API_STATIC_PATH}/pipe +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz ${API_STATIC_PATH}/pipe.tar.gz + +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 ${API_STATIC_PATH}/pipe-el6 +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz + +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx ${API_STATIC_PATH}/pipe-osx +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz + +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip ${API_STATIC_PATH}/pipe.zip + +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz ${API_STATIC_PATH}/client.tar.gz +tar -xzf ${API_STATIC_PATH}/client.tar.gz -C ${API_STATIC_PATH} +rm -f ${API_STATIC_PATH}/client.tar.gz ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ @@ -66,6 +73,7 @@ aws s3 cp s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe. -x :pipe-cli:buildMac \ -x :pipe-cli:buildMacPy3 \ -x :pipe-cli:buildWin \ + -x :client:buildUI \ -Pfast \ --no-daemon From 5582b9ce420af9fbbf4a2633b4c3d520c1f08147 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 22 Sep 2023 13:38:33 +0300 Subject: [PATCH 22/37] Fix typo --- pipe-cli/mount/{hooks-py3 => hooks-py39}/hook-prettytable.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pipe-cli/mount/{hooks-py3 => hooks-py39}/hook-prettytable.py (100%) diff --git a/pipe-cli/mount/hooks-py3/hook-prettytable.py b/pipe-cli/mount/hooks-py39/hook-prettytable.py similarity index 100% rename from pipe-cli/mount/hooks-py3/hook-prettytable.py rename to pipe-cli/mount/hooks-py39/hook-prettytable.py From eec14bb0e5b2d9a0823fcdec90fd326a02fbff54 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Fri, 22 Sep 2023 13:40:45 +0300 Subject: [PATCH 23/37] Install node --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2661a6c4a8..7259714769 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,6 +100,9 @@ jobs: with: python-version: '3.9' architecture: 'x64' + - uses: actions/setup-node@v3 + with: + node-version: 14 - uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} From dd17858d4797f7db4026af019b90aeaed954e1f8 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Wed, 25 Oct 2023 14:52:36 +0300 Subject: [PATCH 24/37] Build cloud data separately --- .github/workflows/main.yml | 38 ++++++++++++++++++- .../gha_build_cloud_data_linux.sh | 28 ++++++++++++++ .../gha_build_cloud_data_windows.sh | 28 ++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 6 +++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 deploy/github_actions/gha_build_cloud_data_linux.sh create mode 100644 deploy/github_actions/gha_build_cloud_data_windows.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7259714769..41fdc602a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,9 +109,45 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_client.sh + Build_CloudData_Linux: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_cloud_data_linux.sh + Build_CloudData_Windows: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_cloud_data_windows.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/deploy/github_actions/gha_build_cloud_data_linux.sh b/deploy/github_actions/gha_build_cloud_data_linux.sh new file mode 100644 index 0000000000..8bafadfcf3 --- /dev/null +++ b/deploy/github_actions/gha_build_cloud_data_linux.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + cloud-pipeline-webdav-client:buildLinux \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz +fi diff --git a/deploy/github_actions/gha_build_cloud_data_windows.sh b/deploy/github_actions/gha_build_cloud_data_windows.sh new file mode 100644 index 0000000000..ece5502ff2 --- /dev/null +++ b/deploy/github_actions/gha_build_cloud_data_windows.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + cloud-pipeline-webdav-client:buildWin \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp cloud-pipeline-webdav-client/out/cloud-data-win64.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 02c660a1ab..bf2b67aabb 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -66,6 +66,10 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB tar -xzf ${API_STATIC_PATH}/client.tar.gz -C ${API_STATIC_PATH} rm -f ${API_STATIC_PATH}/client.tar.gz +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz ${API_STATIC_PATH}/cloud-data-linux.tar.gz + +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip ${API_STATIC_PATH}/cloud-data-win64.zip + ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ @@ -74,6 +78,8 @@ rm -f ${API_STATIC_PATH}/client.tar.gz -x :pipe-cli:buildMacPy3 \ -x :pipe-cli:buildWin \ -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ -Pfast \ --no-daemon From c788a5b2b12bb71ad11db4740db458252f50c097 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Wed, 25 Oct 2023 15:00:19 +0300 Subject: [PATCH 25/37] Build fsbrowser separately --- .github/workflows/main.yml | 23 ++++++++++++++- deploy/github_actions/gha_build_fs_browser.sh | 28 +++++++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 3 ++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 deploy/github_actions/gha_build_fs_browser.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41fdc602a6..84ed0b666e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -145,9 +145,30 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_cloud_data_windows.sh + Build_FSBrowser: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: actions/setup-node@v3 + with: + node-version: 14 + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_fs_browser.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/deploy/github_actions/gha_build_fs_browser.sh b/deploy/github_actions/gha_build_fs_browser.sh new file mode 100644 index 0000000000..bcf9170de0 --- /dev/null +++ b/deploy/github_actions/gha_build_fs_browser.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + fs-browser:build \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp fs-browser/dist/dist-folder/fsbrowser.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index bf2b67aabb..5b4ee19cb8 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -70,6 +70,8 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip ${API_STATIC_PATH}/cloud-data-win64.zip +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz ${API_STATIC_PATH}/fsbrowser.tar.gz + ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ @@ -80,6 +82,7 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :client:buildUI \ -x :cloud-pipeline-webdav-client:buildLinux \ -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ -Pfast \ --no-daemon From c849dee78a0eeec0f08a251f02c4744d40804196 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Wed, 25 Oct 2023 15:11:11 +0300 Subject: [PATCH 26/37] Build data sharing service separately --- .github/workflows/main.yml | 21 ++++++++++++++ .../gha_build_data_sharing_service.sh | 28 +++++++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 4 +++ 3 files changed, 53 insertions(+) create mode 100644 deploy/github_actions/gha_build_data_sharing_service.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 84ed0b666e..a122df502a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -166,6 +166,27 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_fs_browser.sh + Build_DataSharingService: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: actions/setup-node@v3 + with: + node-version: 14 + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_data_sharing_service.sh Build_All: runs-on: ubuntu-20.04 needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser] diff --git a/deploy/github_actions/gha_build_data_sharing_service.sh b/deploy/github_actions/gha_build_data_sharing_service.sh new file mode 100644 index 0000000000..e5a50cf9c2 --- /dev/null +++ b/deploy/github_actions/gha_build_data_sharing_service.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + data-sharing-service:buildFast \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp data-sharing-service/api/build/libs/data-sharing-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 5b4ee19cb8..f1f2a21989 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -72,6 +72,9 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz ${API_STATIC_PATH}/fsbrowser.tar.gz +mkdir -p data-sharing-service/api/build/libs +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar + ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ @@ -83,6 +86,7 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :cloud-pipeline-webdav-client:buildLinux \ -x :cloud-pipeline-webdav-client:buildWin \ -x :fs-browser:build \ + -x :data-sharing-service:buildFast \ -Pfast \ --no-daemon From d351bfa14a07f9b5217d8a3624256f96586aeda3 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Wed, 25 Oct 2023 16:12:42 +0300 Subject: [PATCH 27/37] Add data sharing service to distribution --- deploy/github_actions/gha_pack_dist.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index f1f2a21989..8eee96687f 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -87,6 +87,8 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :cloud-pipeline-webdav-client:buildWin \ -x :fs-browser:build \ -x :data-sharing-service:buildFast \ + -x :data-sharing-service:api:clean \ + -x :data-sharing-service:client:clean \ -Pfast \ --no-daemon From eff10b6917b424825880f5e8d1a002cc033c8135 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Wed, 25 Oct 2023 16:16:26 +0300 Subject: [PATCH 28/37] Build data sharing service beforehand --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a122df502a..4edacc83ac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -189,7 +189,7 @@ jobs: - run: bash deploy/github_actions/gha_build_data_sharing_service.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser, Build_DataSharingService] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 From 148d43c9d9c335e4f8a809f82b6d88544539b99b Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 13:00:35 +0200 Subject: [PATCH 29/37] Skip extra builds --- deploy/github_actions/gha_pack_dist.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 8eee96687f..ea7aed7309 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -19,7 +19,21 @@ CLOUD_PIPELINE_BUILD_RETRY_TIMES=${CLOUD_PIPELINE_BUILD_RETRY_TIMES:-5} # pre-fetch gradle dependency to get rid of gradle timeouts in the distTar step function download_gradle_dependencies() { - ./gradlew clean buildDependents -Pfast -x test --no-daemon + ./gradlew clean buildDependents \ + -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ + -x :pipe-cli:buildWin \ + -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ + -x :data-sharing-service:buildFast \ + -x :data-sharing-service:api:clean \ + -x :data-sharing-service:client:clean \ + -Pfast \ + --no-daemon if [ "$?" != 0 ]; then echo "Problem with resolving gradle dependencies..." From bf2373bf03c449842e176816a741cb0785abc583 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 13:11:54 +0200 Subject: [PATCH 30/37] Build gpu stat separately --- .github/workflows/main.yml | 20 ++++++++++++++- deploy/github_actions/gha_build_gpu_stat.sh | 28 +++++++++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 4 +++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 deploy/github_actions/gha_build_gpu_stat.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4edacc83ac..f491dc0e7f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,9 +187,27 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_data_sharing_service.sh + Build_GpuStat: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_gpu_stat.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser, Build_DataSharingService] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser, Build_DataSharingService, Build_GpuStat] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/deploy/github_actions/gha_build_gpu_stat.sh b/deploy/github_actions/gha_build_gpu_stat.sh new file mode 100644 index 0000000000..f525d5c47d --- /dev/null +++ b/deploy/github_actions/gha_build_gpu_stat.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + workflows:buildGpuStat \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp api/src/main/resources/static/gpustat.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index ea7aed7309..e548c2d9eb 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -32,6 +32,7 @@ function download_gradle_dependencies() { -x :data-sharing-service:buildFast \ -x :data-sharing-service:api:clean \ -x :data-sharing-service:client:clean \ + -x :workflows:buildGpuStat \ -Pfast \ --no-daemon @@ -89,6 +90,8 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB mkdir -p data-sharing-service/api/build/libs aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz ${API_STATIC_PATH}/gpustat.tar.gz + ./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ -x test \ @@ -103,6 +106,7 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :data-sharing-service:buildFast \ -x :data-sharing-service:api:clean \ -x :data-sharing-service:client:clean \ + -x :workflows:buildGpuStat \ -Pfast \ --no-daemon From 05c46e5bca7bcd59b9b3323117c10dd9ebde8459 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 15:36:39 +0200 Subject: [PATCH 31/37] Build necessary modules only once --- deploy/github_actions/gha_pack_dist.sh | 69 ++++++++++++++------------ 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index e548c2d9eb..4ada33848d 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -19,22 +19,23 @@ CLOUD_PIPELINE_BUILD_RETRY_TIMES=${CLOUD_PIPELINE_BUILD_RETRY_TIMES:-5} # pre-fetch gradle dependency to get rid of gradle timeouts in the distTar step function download_gradle_dependencies() { - ./gradlew clean buildDependents \ - -x test \ - -x :pipe-cli:buildLinux \ - -x :pipe-cli:buildMac \ - -x :pipe-cli:buildMacPy3 \ - -x :pipe-cli:buildWin \ - -x :client:buildUI \ - -x :cloud-pipeline-webdav-client:buildLinux \ - -x :cloud-pipeline-webdav-client:buildWin \ - -x :fs-browser:build \ - -x :data-sharing-service:buildFast \ - -x :data-sharing-service:api:clean \ - -x :data-sharing-service:client:clean \ - -x :workflows:buildGpuStat \ - -Pfast \ - --no-daemon + ./gradlew buildDependents \ + -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ + -x :pipe-cli:buildWin \ + -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ + -x :data-sharing-service:client:buildUI \ + -x :data-sharing-service:api:build \ + -x :data-sharing-service:buildAll \ + -x :data-sharing-service:buildFast \ + -x :workflows:buildGpuStat \ + -Pfast \ + --no-daemon if [ "$?" != 0 ]; then echo "Problem with resolving gradle dependencies..." @@ -92,23 +93,25 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz ${API_STATIC_PATH}/gpustat.tar.gz -./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ - -Pprofile=release \ - -x test \ - -x :pipe-cli:buildLinux \ - -x :pipe-cli:buildMac \ - -x :pipe-cli:buildMacPy3 \ - -x :pipe-cli:buildWin \ - -x :client:buildUI \ - -x :cloud-pipeline-webdav-client:buildLinux \ - -x :cloud-pipeline-webdav-client:buildWin \ - -x :fs-browser:build \ - -x :data-sharing-service:buildFast \ - -x :data-sharing-service:api:clean \ - -x :data-sharing-service:client:clean \ - -x :workflows:buildGpuStat \ - -Pfast \ - --no-daemon +./gradlew distTar \ + -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ + -x :pipe-cli:buildWin \ + -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ + -x :data-sharing-service:client:buildUI \ + -x :data-sharing-service:api:build \ + -x :data-sharing-service:buildAll \ + -x :data-sharing-service:buildFast \ + -x :workflows:buildGpuStat \ + -Pfast \ + --no-daemon if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) From 1844857123e5665c9e53b7531e050dd3a0490b42 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 15:41:21 +0200 Subject: [PATCH 32/37] Refactor main workflow --- .github/workflows/main.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f491dc0e7f..c6efb11cd3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -145,7 +145,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_cloud_data_windows.sh - Build_FSBrowser: + Build_DataSharingService: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -165,8 +165,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - run: bash deploy/github_actions/gha_build_fs_browser.sh - Build_DataSharingService: + - run: bash deploy/github_actions/gha_build_data_sharing_service.sh + Build_FSBrowser: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -186,7 +186,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - run: bash deploy/github_actions/gha_build_data_sharing_service.sh + - run: bash deploy/github_actions/gha_build_fs_browser.sh Build_GpuStat: runs-on: ubuntu-20.04 steps: @@ -207,7 +207,11 @@ jobs: - run: bash deploy/github_actions/gha_build_gpu_stat.sh Build_All: runs-on: ubuntu-20.04 - needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, Build_FSBrowser, Build_DataSharingService, Build_GpuStat] + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, + Build_Client, + Build_CloudData_Linux, Build_CloudData_Windows, + Build_DataSharingService, + Build_FSBrowser, Build_GpuStat] steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 From 887d86f974b6809475996c0f99c3f911885ecbbf Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 15:50:20 +0200 Subject: [PATCH 33/37] Remove node install step from build all job --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6efb11cd3..215105c5e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -222,9 +222,6 @@ jobs: with: python-version: '3.9' architecture: 'x64' - - uses: actions/setup-node@v3 - with: - node-version: 14 - uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 51fa734ed6471307981778e23405a93862b43e6d Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Mon, 6 Nov 2023 16:42:57 +0200 Subject: [PATCH 34/37] Build data transfer service separately --- .github/workflows/main.yml | 16 +++++++++- .../gha_build_data_transfer_service.sh | 32 +++++++++++++++++++ deploy/github_actions/gha_pack_dist.sh | 12 +++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 deploy/github_actions/gha_build_data_transfer_service.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 215105c5e0..2e862e3539 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -166,6 +166,20 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: bash deploy/github_actions/gha_build_data_sharing_service.sh + Build_DataTransferService: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_data_transfer_service.sh Build_FSBrowser: runs-on: ubuntu-20.04 steps: @@ -210,7 +224,7 @@ jobs: needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, Build_Client, Build_CloudData_Linux, Build_CloudData_Windows, - Build_DataSharingService, + Build_DataSharingService, Build_DataTransferService, Build_FSBrowser, Build_GpuStat] steps: - uses: actions/checkout@v2 diff --git a/deploy/github_actions/gha_build_data_transfer_service.sh b/deploy/github_actions/gha_build_data_transfer_service.sh new file mode 100644 index 0000000000..ad0792a3b2 --- /dev/null +++ b/deploy/github_actions/gha_build_data_transfer_service.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + data-transfer-service:build \ + data-transfer-service:bundleWindows \ + data-transfer-service:bundleLinux \ + --no-daemon + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp data-transfer-service/build/libs/data-transfer-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar + aws s3 cp data-transfer-service/build/distributions/data-transfer-service-windows.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip + aws s3 cp data-transfer-service/build/distributions/data-transfer-service-linux.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 4ada33848d..36769ee9f9 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -33,6 +33,9 @@ function download_gradle_dependencies() { -x :data-sharing-service:api:build \ -x :data-sharing-service:buildAll \ -x :data-sharing-service:buildFast \ + -x :data-transfer-service:build \ + -x :data-transfer-service:bundleWindows \ + -x :data-transfer-service:bundleLinux \ -x :workflows:buildGpuStat \ -Pfast \ --no-daemon @@ -91,6 +94,12 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB mkdir -p data-sharing-service/api/build/libs aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar +mkdir -p data-transfer-service/build/libs \ + data-transfer-service/build/distributions +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar data-transfer-service/build/libs/data-transfer-service.jar +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip data-transfer-service/build/distributions/data-transfer-service-windows.zip +aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip data-transfer-service/build/distributions/data-transfer-service-linux.zip + aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz ${API_STATIC_PATH}/gpustat.tar.gz ./gradlew distTar \ @@ -109,6 +118,9 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :data-sharing-service:api:build \ -x :data-sharing-service:buildAll \ -x :data-sharing-service:buildFast \ + -x :data-transfer-service:build \ + -x :data-transfer-service:bundleWindows \ + -x :data-transfer-service:bundleLinux \ -x :workflows:buildGpuStat \ -Pfast \ --no-daemon From 00460e3e6e1e0dca210a99fa83265eb3f9e07019 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Tue, 7 Nov 2023 09:59:00 +0200 Subject: [PATCH 35/37] Build necessary modules only once --- deploy/github_actions/gha_pack_dist.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 36769ee9f9..9b57f175a9 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -33,6 +33,7 @@ function download_gradle_dependencies() { -x :data-sharing-service:api:build \ -x :data-sharing-service:buildAll \ -x :data-sharing-service:buildFast \ + -x :data-transfer-service:bootJar \ -x :data-transfer-service:build \ -x :data-transfer-service:bundleWindows \ -x :data-transfer-service:bundleLinux \ @@ -118,6 +119,7 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -x :data-sharing-service:api:build \ -x :data-sharing-service:buildAll \ -x :data-sharing-service:buildFast \ + -x :data-transfer-service:bootJar \ -x :data-transfer-service:build \ -x :data-transfer-service:bundleWindows \ -x :data-transfer-service:bundleLinux \ From 5ad5757237ae333ef1f5dd807a1bb018ef96adb1 Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Tue, 7 Nov 2023 10:37:22 +0200 Subject: [PATCH 36/37] Transfer s3 files more efficiently --- deploy/github_actions/gha_build_client.sh | 4 +- .../gha_build_cloud_data_linux.sh | 4 +- .../gha_build_cloud_data_windows.sh | 4 +- .../gha_build_data_sharing_service.sh | 4 +- .../gha_build_data_transfer_service.sh | 10 ++-- deploy/github_actions/gha_build_fs_browser.sh | 4 +- deploy/github_actions/gha_build_gpu_stat.sh | 4 +- deploy/github_actions/gha_build_pipe_linux.sh | 7 ++- .../gha_build_pipe_linux_el6.sh | 7 ++- deploy/github_actions/gha_build_pipe_macos.sh | 7 ++- .../github_actions/gha_build_pipe_windows.sh | 4 +- deploy/github_actions/gha_pack_dist.sh | 54 +++++++++---------- 12 files changed, 70 insertions(+), 43 deletions(-) diff --git a/deploy/github_actions/gha_build_client.sh b/deploy/github_actions/gha_build_client.sh index 3063d3520c..6225c18511 100644 --- a/deploy/github_actions/gha_build_client.sh +++ b/deploy/github_actions/gha_build_client.sh @@ -25,6 +25,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM tar -czf $PWD/client.tar.gz -C client/build . +ls -lh client.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp client.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz + aws s3 cp --no-progress client.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz fi diff --git a/deploy/github_actions/gha_build_cloud_data_linux.sh b/deploy/github_actions/gha_build_cloud_data_linux.sh index 8bafadfcf3..7e6ede69bf 100644 --- a/deploy/github_actions/gha_build_cloud_data_linux.sh +++ b/deploy/github_actions/gha_build_cloud_data_linux.sh @@ -23,6 +23,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM cloud-pipeline-webdav-client:buildLinux \ --no-daemon +ls -lh cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz + aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz fi diff --git a/deploy/github_actions/gha_build_cloud_data_windows.sh b/deploy/github_actions/gha_build_cloud_data_windows.sh index ece5502ff2..634eb71b48 100644 --- a/deploy/github_actions/gha_build_cloud_data_windows.sh +++ b/deploy/github_actions/gha_build_cloud_data_windows.sh @@ -23,6 +23,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM cloud-pipeline-webdav-client:buildWin \ --no-daemon +ls -lh cloud-pipeline-webdav-client/out/cloud-data-win64.zip + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp cloud-pipeline-webdav-client/out/cloud-data-win64.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip + aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-win64.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip fi diff --git a/deploy/github_actions/gha_build_data_sharing_service.sh b/deploy/github_actions/gha_build_data_sharing_service.sh index e5a50cf9c2..a8d2bf372a 100644 --- a/deploy/github_actions/gha_build_data_sharing_service.sh +++ b/deploy/github_actions/gha_build_data_sharing_service.sh @@ -23,6 +23,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM data-sharing-service:buildFast \ --no-daemon +ls -lh data-sharing-service/api/build/libs/data-sharing-service.jar + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp data-sharing-service/api/build/libs/data-sharing-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar + aws s3 cp --no-progress data-sharing-service/api/build/libs/data-sharing-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar fi diff --git a/deploy/github_actions/gha_build_data_transfer_service.sh b/deploy/github_actions/gha_build_data_transfer_service.sh index ad0792a3b2..96b1eaa02b 100644 --- a/deploy/github_actions/gha_build_data_transfer_service.sh +++ b/deploy/github_actions/gha_build_data_transfer_service.sh @@ -25,8 +25,12 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM data-transfer-service:bundleLinux \ --no-daemon +ls -lh data-transfer-service/build/libs/data-transfer-service.jar \ + data-transfer-service/build/distributions/data-transfer-service-windows.zip \ + data-transfer-service/build/distributions/data-transfer-service-linux.zip + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp data-transfer-service/build/libs/data-transfer-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar - aws s3 cp data-transfer-service/build/distributions/data-transfer-service-windows.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip - aws s3 cp data-transfer-service/build/distributions/data-transfer-service-linux.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip + aws s3 cp --no-progress data-transfer-service/build/libs/data-transfer-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar + aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-windows.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip + aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-linux.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip fi diff --git a/deploy/github_actions/gha_build_fs_browser.sh b/deploy/github_actions/gha_build_fs_browser.sh index bcf9170de0..5c9ae99ed3 100644 --- a/deploy/github_actions/gha_build_fs_browser.sh +++ b/deploy/github_actions/gha_build_fs_browser.sh @@ -23,6 +23,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM fs-browser:build \ --no-daemon +ls -lh fs-browser/dist/dist-folder/fsbrowser.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp fs-browser/dist/dist-folder/fsbrowser.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz + aws s3 cp --no-progress fs-browser/dist/dist-folder/fsbrowser.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz fi diff --git a/deploy/github_actions/gha_build_gpu_stat.sh b/deploy/github_actions/gha_build_gpu_stat.sh index f525d5c47d..c69ebc9e1f 100644 --- a/deploy/github_actions/gha_build_gpu_stat.sh +++ b/deploy/github_actions/gha_build_gpu_stat.sh @@ -23,6 +23,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM workflows:buildGpuStat \ --no-daemon +ls -lh api/src/main/resources/static/gpustat.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp api/src/main/resources/static/gpustat.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz + aws s3 cp --no-progress api/src/main/resources/static/gpustat.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_linux.sh b/deploy/github_actions/gha_build_pipe_linux.sh index 7de4b4e4c4..1bb26b3db3 100644 --- a/deploy/github_actions/gha_build_pipe_linux.sh +++ b/deploy/github_actions/gha_build_pipe_linux.sh @@ -24,7 +24,10 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM --no-daemon \ -x :pipe-cli:test +ls -lh pipe-cli/dist/dist-file/pipe \ + pipe-cli/dist/dist-folder/pipe.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe - aws s3 cp pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_linux_el6.sh b/deploy/github_actions/gha_build_pipe_linux_el6.sh index dd62d2660b..e99ea49ffd 100644 --- a/deploy/github_actions/gha_build_pipe_linux_el6.sh +++ b/deploy/github_actions/gha_build_pipe_linux_el6.sh @@ -25,7 +25,10 @@ _BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-c --no-daemon \ -x :pipe-cli:test +ls -lh pipe-cli/dist/dist-file/pipe \ + pipe-cli/dist/dist-folder/pipe.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 - aws s3 cp pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_macos.sh b/deploy/github_actions/gha_build_pipe_macos.sh index 9ff579e662..4c5ecf1449 100644 --- a/deploy/github_actions/gha_build_pipe_macos.sh +++ b/deploy/github_actions/gha_build_pipe_macos.sh @@ -24,7 +24,10 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM --no-daemon \ -x :pipe-cli:test +ls -lh pipe-cli/dist/dist-file/pipe-osx \ + pipe-cli/dist/dist-folder/pipe-osx.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx - aws s3 cp pipe-cli/dist/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_windows.sh b/deploy/github_actions/gha_build_pipe_windows.sh index 8bd44340e6..f10303ece0 100644 --- a/deploy/github_actions/gha_build_pipe_windows.sh +++ b/deploy/github_actions/gha_build_pipe_windows.sh @@ -24,6 +24,8 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM --no-daemon \ -x :pipe-cli:test +ls -lh pipe-cli/dist/win/pipe.zip + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp pipe-cli/dist/win/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip + aws s3 cp --no-progress pipe-cli/dist/win/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index 9b57f175a9..c50c00a9a4 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -71,37 +71,36 @@ rm -rf ${API_STATIC_PATH}/* rm -rf build/install/dist/* mkdir -p ${API_STATIC_PATH} -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe ${API_STATIC_PATH}/pipe -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz ${API_STATIC_PATH}/pipe.tar.gz +aws s3 cp --no-progress --recursive s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/ ${API_STATIC_PATH}/ + +ls -lh ${API_STATIC_PATH}/pipe \ + ${API_STATIC_PATH}/pipe.tar.gz \ + ${API_STATIC_PATH}/pipe-el6 \ + ${API_STATIC_PATH}/pipe-el6.tar.gz \ + ${API_STATIC_PATH}/pipe-osx \ + ${API_STATIC_PATH}/pipe-osx.tar.gz \ + ${API_STATIC_PATH}/pipe.zip \ + ${API_STATIC_PATH}/client.tar.gz \ + ${API_STATIC_PATH}/cloud-data-linux.tar.gz \ + ${API_STATIC_PATH}/cloud-data-win64.zip \ + ${API_STATIC_PATH}/fsbrowser.tar.gz \ + ${API_STATIC_PATH}/gpustat.tar.gz \ + ${API_STATIC_PATH}/data-sharing-service.jar \ + ${API_STATIC_PATH}/data-transfer-service.jar \ + ${API_STATIC_PATH}/data-transfer-service-windows.zip \ + ${API_STATIC_PATH}/data-transfer-service-linux.zip -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 ${API_STATIC_PATH}/pipe-el6 -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx ${API_STATIC_PATH}/pipe-osx -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip ${API_STATIC_PATH}/pipe.zip - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz ${API_STATIC_PATH}/client.tar.gz tar -xzf ${API_STATIC_PATH}/client.tar.gz -C ${API_STATIC_PATH} rm -f ${API_STATIC_PATH}/client.tar.gz -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz ${API_STATIC_PATH}/cloud-data-linux.tar.gz - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip ${API_STATIC_PATH}/cloud-data-win64.zip - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz ${API_STATIC_PATH}/fsbrowser.tar.gz - mkdir -p data-sharing-service/api/build/libs -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar +mv ${API_STATIC_PATH}/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar mkdir -p data-transfer-service/build/libs \ data-transfer-service/build/distributions -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar data-transfer-service/build/libs/data-transfer-service.jar -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip data-transfer-service/build/distributions/data-transfer-service-windows.zip -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip data-transfer-service/build/distributions/data-transfer-service-linux.zip - -aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz ${API_STATIC_PATH}/gpustat.tar.gz +mv ${API_STATIC_PATH}/data-transfer-service.jar data-transfer-service/build/libs/data-transfer-service.jar +mv ${API_STATIC_PATH}/data-transfer-service-windows.zip data-transfer-service/build/distributions/data-transfer-service-windows.zip +mv ${API_STATIC_PATH}/data-transfer-service-linux.zip data-transfer-service/build/distributions/data-transfer-service-linux.zip ./gradlew distTar \ -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ @@ -127,9 +126,10 @@ aws s3 cp --quiet s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMB -Pfast \ --no-daemon -if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) +DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) - # Publish repackaged distribution tgz to S3 into builds/ prefix - aws s3 cp --quiet $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ +ls -lh $DIST_TGZ_NAME + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ fi From 8121e620406bb748c3c75db24a5586b42a281d8a Mon Sep 17 00:00:00 2001 From: Andrey Tsibin Date: Tue, 7 Nov 2023 12:05:33 +0200 Subject: [PATCH 37/37] Upload gha built distributions to a separate directory in s3 --- deploy/github_actions/gha_pack_dist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index c50c00a9a4..ae94f387e7 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -131,5 +131,5 @@ DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) ls -lh $DIST_TGZ_NAME if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - aws s3 cp --no-progress $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ + aws s3 cp --no-progress $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/gha/${GITHUB_REF_NAME}/ fi