diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd90c9641a..2e862e3539 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,42 @@ env: CLOUD_PIPELINE_BUILD_NUMBER_SEED: 30000 jobs: + CLI_Linux: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_linux.sh + CLI_Linux_el6: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_linux_el6.sh CLI_MacOS: runs-on: macos-11 steps: @@ -26,17 +62,34 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '2.7' + python-version: '3.9' architecture: 'x64' - uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - run: bash deploy/github_actions/gha_build_macos.sh - Build_All: + - run: bash deploy/github_actions/gha_build_pipe_macos.sh + CLI_Windows: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_pipe_windows.sh + Build_Client: runs-on: ubuntu-20.04 - needs: CLI_MacOS steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 @@ -45,7 +98,7 @@ jobs: java-version: '8' - uses: actions/setup-python@v2 with: - python-version: '2.7.18' + python-version: '3.9' architecture: 'x64' - uses: actions/setup-node@v3 with: @@ -55,4 +108,137 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_client.sh + Build_CloudData_Linux: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_cloud_data_linux.sh + Build_CloudData_Windows: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_cloud_data_windows.sh + Build_DataSharingService: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: actions/setup-node@v3 + with: + node-version: 14 + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_data_sharing_service.sh + Build_DataTransferService: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_data_transfer_service.sh + Build_FSBrowser: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: actions/setup-node@v3 + with: + node-version: 14 + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_fs_browser.sh + Build_GpuStat: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - run: bash deploy/github_actions/gha_build_gpu_stat.sh + Build_All: + runs-on: ubuntu-20.04 + needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows, + Build_Client, + Build_CloudData_Linux, Build_CloudData_Windows, + Build_DataSharingService, Build_DataTransferService, + Build_FSBrowser, Build_GpuStat] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: '8' + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - run: sudo -E PATH=$PATH bash deploy/github_actions/gha_pack_dist.sh diff --git a/deploy/github_actions/gha_build_client.sh b/deploy/github_actions/gha_build_client.sh new file mode 100644 index 0000000000..6225c18511 --- /dev/null +++ b/deploy/github_actions/gha_build_client.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + client:buildUI \ + --no-daemon + +tar -czf $PWD/client.tar.gz -C client/build . + +ls -lh client.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress client.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz +fi diff --git a/deploy/github_actions/gha_build_cloud_data_linux.sh b/deploy/github_actions/gha_build_cloud_data_linux.sh new file mode 100644 index 0000000000..7e6ede69bf --- /dev/null +++ b/deploy/github_actions/gha_build_cloud_data_linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + cloud-pipeline-webdav-client:buildLinux \ + --no-daemon + +ls -lh cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz +fi diff --git a/deploy/github_actions/gha_build_cloud_data_windows.sh b/deploy/github_actions/gha_build_cloud_data_windows.sh new file mode 100644 index 0000000000..634eb71b48 --- /dev/null +++ b/deploy/github_actions/gha_build_cloud_data_windows.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + cloud-pipeline-webdav-client:buildWin \ + --no-daemon + +ls -lh cloud-pipeline-webdav-client/out/cloud-data-win64.zip + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-win64.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip +fi diff --git a/deploy/github_actions/gha_build_data_sharing_service.sh b/deploy/github_actions/gha_build_data_sharing_service.sh new file mode 100644 index 0000000000..a8d2bf372a --- /dev/null +++ b/deploy/github_actions/gha_build_data_sharing_service.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + data-sharing-service:buildFast \ + --no-daemon + +ls -lh data-sharing-service/api/build/libs/data-sharing-service.jar + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress data-sharing-service/api/build/libs/data-sharing-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar +fi diff --git a/deploy/github_actions/gha_build_data_transfer_service.sh b/deploy/github_actions/gha_build_data_transfer_service.sh new file mode 100644 index 0000000000..96b1eaa02b --- /dev/null +++ b/deploy/github_actions/gha_build_data_transfer_service.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + data-transfer-service:build \ + data-transfer-service:bundleWindows \ + data-transfer-service:bundleLinux \ + --no-daemon + +ls -lh data-transfer-service/build/libs/data-transfer-service.jar \ + data-transfer-service/build/distributions/data-transfer-service-windows.zip \ + data-transfer-service/build/distributions/data-transfer-service-linux.zip + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress data-transfer-service/build/libs/data-transfer-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar + aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-windows.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip + aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-linux.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip +fi diff --git a/deploy/github_actions/gha_build_fs_browser.sh b/deploy/github_actions/gha_build_fs_browser.sh new file mode 100644 index 0000000000..5c9ae99ed3 --- /dev/null +++ b/deploy/github_actions/gha_build_fs_browser.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + fs-browser:build \ + --no-daemon + +ls -lh fs-browser/dist/dist-folder/fsbrowser.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress fs-browser/dist/dist-folder/fsbrowser.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/fsbrowser.tar.gz +fi diff --git a/deploy/github_actions/gha_build_gpu_stat.sh b/deploy/github_actions/gha_build_gpu_stat.sh new file mode 100644 index 0000000000..c69ebc9e1f --- /dev/null +++ b/deploy/github_actions/gha_build_gpu_stat.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + workflows:buildGpuStat \ + --no-daemon + +ls -lh api/src/main/resources/static/gpustat.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress api/src/main/resources/static/gpustat.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/gpustat.tar.gz +fi diff --git a/deploy/github_actions/gha_build_pipe_linux.sh b/deploy/github_actions/gha_build_pipe_linux.sh new file mode 100644 index 0000000000..1bb26b3db3 --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_linux.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildLinux \ + --no-daemon \ + -x :pipe-cli:test + +ls -lh pipe-cli/dist/dist-file/pipe \ + pipe-cli/dist/dist-folder/pipe.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.tar.gz +fi diff --git a/deploy/github_actions/gha_build_pipe_linux_el6.sh b/deploy/github_actions/gha_build_pipe_linux_el6.sh new file mode 100644 index 0000000000..e99ea49ffd --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_linux_el6.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +_BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-centos6" \ +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildLinux \ + --no-daemon \ + -x :pipe-cli:test + +ls -lh pipe-cli/dist/dist-file/pipe \ + pipe-cli/dist/dist-folder/pipe.tar.gz + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6 + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-el6.tar.gz +fi diff --git a/deploy/github_actions/gha_build_macos.sh b/deploy/github_actions/gha_build_pipe_macos.sh similarity index 69% rename from deploy/github_actions/gha_build_macos.sh rename to deploy/github_actions/gha_build_pipe_macos.sh index a4aec8b877..4c5ecf1449 100644 --- a/deploy/github_actions/gha_build_macos.sh +++ b/deploy/github_actions/gha_build_pipe_macos.sh @@ -20,15 +20,14 @@ CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUM ./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ -Pprofile=release \ - pipe-cli:buildMac \ + pipe-cli:buildMacPy39 \ --no-daemon \ -x :pipe-cli:test -cd pipe-cli -DIST_TGZ_NAME=pipe-osx-full.$CLOUD_PIPELINE_BUILD_NUMBER.tar.gz -tar -zcf $DIST_TGZ_NAME dist +ls -lh pipe-cli/dist/dist-file/pipe-osx \ + pipe-cli/dist/dist-folder/pipe-osx.tar.gz + if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - if [ "$GITHUB_REF_NAME" == "develop" ] || [ "$GITHUB_REF_NAME" == "master" ] || [[ "$GITHUB_REF_NAME" == "release/"* ]] || [[ "$GITHUB_REF_NAME" == "stage/"* ]]; then - aws s3 cp $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/temp/ - fi + aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe-osx s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx + aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe-osx.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-osx.tar.gz fi diff --git a/deploy/github_actions/gha_build_pipe_windows.sh b/deploy/github_actions/gha_build_pipe_windows.sh new file mode 100644 index 0000000000..f10303ece0 --- /dev/null +++ b/deploy/github_actions/gha_build_pipe_windows.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER)) + +./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + pipe-cli:buildWin \ + --no-daemon \ + -x :pipe-cli:test + +ls -lh pipe-cli/dist/win/pipe.zip + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress pipe-cli/dist/win/pipe.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe.zip +fi diff --git a/deploy/github_actions/gha_pack_dist.sh b/deploy/github_actions/gha_pack_dist.sh index ad8d75cc64..ae94f387e7 100644 --- a/deploy/github_actions/gha_pack_dist.sh +++ b/deploy/github_actions/gha_pack_dist.sh @@ -19,7 +19,27 @@ CLOUD_PIPELINE_BUILD_RETRY_TIMES=${CLOUD_PIPELINE_BUILD_RETRY_TIMES:-5} # pre-fetch gradle dependency to get rid of gradle timeouts in the distTar step function download_gradle_dependencies() { - ./gradlew clean buildDependents -Pfast -x test --no-daemon + ./gradlew buildDependents \ + -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ + -x :pipe-cli:buildWin \ + -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ + -x :data-sharing-service:client:buildUI \ + -x :data-sharing-service:api:build \ + -x :data-sharing-service:buildAll \ + -x :data-sharing-service:buildFast \ + -x :data-transfer-service:bootJar \ + -x :data-transfer-service:build \ + -x :data-transfer-service:bundleWindows \ + -x :data-transfer-service:bundleLinux \ + -x :workflows:buildGpuStat \ + -Pfast \ + --no-daemon if [ "$?" != 0 ]; then echo "Problem with resolving gradle dependencies..." @@ -44,39 +64,72 @@ done # Fail script if any command will fail set -e -curl https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/pip/2.7/get-pip.py | python2 - - -python2 -m pip install mkdocs +python -m pip install mkdocs API_STATIC_PATH=api/src/main/resources/static rm -rf ${API_STATIC_PATH}/* rm -rf build/install/dist/* mkdir -p ${API_STATIC_PATH} -_OSX_CLI_TAR_NAME=pipe-osx-full.$CLOUD_PIPELINE_BUILD_NUMBER.tar.gz -_OSX_CLI_PATH=$(mktemp -d) -aws s3 cp s3://cloud-pipeline-oss-builds/temp/${_OSX_CLI_TAR_NAME} ${_OSX_CLI_PATH}/ -tar -zxf $_OSX_CLI_PATH/$_OSX_CLI_TAR_NAME -C $_OSX_CLI_PATH +aws s3 cp --no-progress --recursive s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/ ${API_STATIC_PATH}/ -mv $_OSX_CLI_PATH/dist/dist-file/pipe-osx ${API_STATIC_PATH}/pipe-osx -mv $_OSX_CLI_PATH/dist/dist-folder/pipe-osx.tar.gz ${API_STATIC_PATH}/pipe-osx.tar.gz +ls -lh ${API_STATIC_PATH}/pipe \ + ${API_STATIC_PATH}/pipe.tar.gz \ + ${API_STATIC_PATH}/pipe-el6 \ + ${API_STATIC_PATH}/pipe-el6.tar.gz \ + ${API_STATIC_PATH}/pipe-osx \ + ${API_STATIC_PATH}/pipe-osx.tar.gz \ + ${API_STATIC_PATH}/pipe.zip \ + ${API_STATIC_PATH}/client.tar.gz \ + ${API_STATIC_PATH}/cloud-data-linux.tar.gz \ + ${API_STATIC_PATH}/cloud-data-win64.zip \ + ${API_STATIC_PATH}/fsbrowser.tar.gz \ + ${API_STATIC_PATH}/gpustat.tar.gz \ + ${API_STATIC_PATH}/data-sharing-service.jar \ + ${API_STATIC_PATH}/data-transfer-service.jar \ + ${API_STATIC_PATH}/data-transfer-service-windows.zip \ + ${API_STATIC_PATH}/data-transfer-service-linux.zip -_BUILD_DOCKER_IMAGE="${CP_DOCKER_DIST_SRV}lifescience/cloud-pipeline:python2.7-centos6" ./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} -Pprofile=release pipe-cli:buildLinux --no-daemon -x :pipe-cli:test -mv pipe-cli/dist/dist-file/pipe ${API_STATIC_PATH}/pipe-el6 -mv pipe-cli/dist/dist-folder/pipe.tar.gz ${API_STATIC_PATH}/pipe-el6.tar.gz +tar -xzf ${API_STATIC_PATH}/client.tar.gz -C ${API_STATIC_PATH} +rm -f ${API_STATIC_PATH}/client.tar.gz -./gradlew clean distTar -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ - -Pprofile=release \ - -x test \ - -Pfast \ - --no-daemon +mkdir -p data-sharing-service/api/build/libs +mv ${API_STATIC_PATH}/data-sharing-service.jar data-sharing-service/api/build/libs/data-sharing-service.jar -if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then - DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) +mkdir -p data-transfer-service/build/libs \ + data-transfer-service/build/distributions +mv ${API_STATIC_PATH}/data-transfer-service.jar data-transfer-service/build/libs/data-transfer-service.jar +mv ${API_STATIC_PATH}/data-transfer-service-windows.zip data-transfer-service/build/distributions/data-transfer-service-windows.zip +mv ${API_STATIC_PATH}/data-transfer-service-linux.zip data-transfer-service/build/distributions/data-transfer-service-linux.zip - # Publish repackaged distribution tgz to S3 into builds/ prefix - # Only if it is one of the allowed branches and it is a push (not PR) - if [ "$GITHUB_REF_NAME" == "develop" ] || [ "$GITHUB_REF_NAME" == "master" ] || [[ "$GITHUB_REF_NAME" == "release/"* ]] || [[ "$GITHUB_REF_NAME" == "stage/"* ]]; then - aws s3 cp --quiet $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/${GITHUB_REF_NAME}/ - fi +./gradlew distTar \ + -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \ + -Pprofile=release \ + -x test \ + -x :pipe-cli:buildLinux \ + -x :pipe-cli:buildMac \ + -x :pipe-cli:buildMacPy3 \ + -x :pipe-cli:buildWin \ + -x :client:buildUI \ + -x :cloud-pipeline-webdav-client:buildLinux \ + -x :cloud-pipeline-webdav-client:buildWin \ + -x :fs-browser:build \ + -x :data-sharing-service:client:buildUI \ + -x :data-sharing-service:api:build \ + -x :data-sharing-service:buildAll \ + -x :data-sharing-service:buildFast \ + -x :data-transfer-service:bootJar \ + -x :data-transfer-service:build \ + -x :data-transfer-service:bundleWindows \ + -x :data-transfer-service:bundleLinux \ + -x :workflows:buildGpuStat \ + -Pfast \ + --no-daemon + +DIST_TGZ_NAME=$(echo build/install/dist/cloud-pipeline*) + +ls -lh $DIST_TGZ_NAME + +if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then + aws s3 cp --no-progress $DIST_TGZ_NAME s3://cloud-pipeline-oss-builds/builds/gha/${GITHUB_REF_NAME}/ fi diff --git a/pipe-cli/build.gradle b/pipe-cli/build.gradle index e61335455e..3576210ee2 100644 --- a/pipe-cli/build.gradle +++ b/pipe-cli/build.gradle @@ -100,6 +100,31 @@ task buildMac(type: Exec, dependsOn: ':pipe-cli:test') { } } +task buildMacPy39(type: Exec) { + group = 'build' + doFirst { + setCLIVersion() + } + environment "PIPE_CLI_SOURCES_DIR": "$project.rootDir/pipe-cli" + environment "PIPE_CLI_LINUX_DIST_DIR": "$project.rootDir/pipe-cli" + environment "PIPE_MOUNT_SOURCES_DIR": "$project.rootDir/pipe-cli/mount" + environment "PYINSTALLER_PATH": "/tmp/pyinstaller" + + commandLine "bash", "$project.rootDir/pipe-cli/build_mac_py39.sh" + doLast { + cleanCLIVersion() + delete("$project.rootDir/api/src/main/resources/static/pipe_osx") + copy { + from("$project.rootDir/pipe-cli/dist/dist-file/pipe_osx") + into("$project.rootDir/api/src/main/resources/static/") + } + copy { + from("$project.rootDir/pipe-cli/dist/dist-folder/pipe_osx.tar.gz") + into("$project.rootDir/api/src/main/resources/static/") + } + } +} + task test(type: Exec, dependsOn: ':pipe-cli:installPackages') { group = 'test' commandLine "pytest", "-s", "-vvv", "--cov=$project.rootDir/pipe-cli", "--cov-config=$project.rootDir/pipe-cli/tests/cov_config" diff --git a/pipe-cli/build_mac_py39.sh b/pipe-cli/build_mac_py39.sh new file mode 100644 index 0000000000..b631bb088c --- /dev/null +++ b/pipe-cli/build_mac_py39.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Copyright 2017-2021 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### +# Setup Pyinstaller +### +python -m pip install pyinstaller==5.13.2 + +### +# Setup common dependencies +### +python -m pip install macholib==1.16.2 +python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt + +### +# Build pipe fuse +### +python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt +cd $PIPE_MOUNT_SOURCES_DIR && \ +pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ + --paths "$PIPE_MOUNT_SOURCES_DIR" \ + --hidden-import=UserList \ + --hidden-import=UserString \ + --hidden-import=commands \ + --hidden-import=ConfigParser \ + --hidden-import=UserDict \ + --hidden-import=itertools \ + --hidden-import=collections \ + --hidden-import=future.backports.misc \ + --hidden-import=commands \ + --hidden-import=base64 \ + --hidden-import=__builtin__ \ + --hidden-import=math \ + --hidden-import=reprlib \ + --hidden-import=functools \ + --hidden-import=re \ + --hidden-import=subprocess \ + --hidden-import=_sysconfigdata \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ + -y \ + --clean \ + --distpath /tmp/mount/dist \ + ${PIPE_MOUNT_SOURCES_DIR}/pipe-fuse.py + +chmod +x /tmp/mount/dist/pipe-fuse/pipe-fuse + +### +# Build pipe +### +function build_pipe { + local distpath="$1" + local onefile="$2" + + version_file="${PIPE_CLI_SOURCES_DIR}/src/version.py" + sed -i'.bkp' '/__bundle_info__/d' $version_file + + bundle_type="one-folder" + [ "$onefile" ] && bundle_type="one-file" + + build_os_version_id=$(sw_vers -productVersion) + echo "__bundle_info__ = { 'bundle_type': '$bundle_type', 'build_os_id': 'macos', 'build_os_version_id': '$build_os_version_id' }" >> $version_file + + cd $PIPE_CLI_SOURCES_DIR + sed -i'.bkp' '/__component_version__/d' $version_file + local pipe_commit_hash=$(git log --pretty=tformat:"%H" -n1 .) + echo "__component_version__='$pipe_commit_hash'" >> $version_file + + pyinstaller \ + --paths "$PIPE_CLI_SOURCES_DIR" \ + --hidden-import=UserList \ + --hidden-import=UserString \ + --hidden-import=commands \ + --hidden-import=ConfigParser \ + --hidden-import=UserDict \ + --hidden-import=itertools \ + --hidden-import=collections \ + --hidden-import=future.backports.misc \ + --hidden-import=commands \ + --hidden-import=base64 \ + --hidden-import=__builtin__ \ + --hidden-import=math \ + --hidden-import=reprlib \ + --hidden-import=functools \ + --hidden-import=re \ + --hidden-import=subprocess \ + --hidden-import=_sysconfigdata \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \ + --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \ + -y \ + --clean \ + --distpath $distpath \ + --add-data /tmp/mount/dist/pipe-fuse:mount \ + --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \ + ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile +} +build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file --onefile +mv $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe-osx + +build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder +tar -zcf $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder/pipe-osx.tar.gz \ + -C $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder \ + pipe diff --git a/pipe-cli/hooks-py39/hook-prettytable.py b/pipe-cli/hooks-py39/hook-prettytable.py new file mode 100644 index 0000000000..3082c779d0 --- /dev/null +++ b/pipe-cli/hooks-py39/hook-prettytable.py @@ -0,0 +1,17 @@ +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from PyInstaller.utils.hooks import collect_all + +datas, binaries, hiddenimports = collect_all('PTable') diff --git a/pipe-cli/mount/hooks-py39/hook-prettytable.py b/pipe-cli/mount/hooks-py39/hook-prettytable.py new file mode 100644 index 0000000000..3082c779d0 --- /dev/null +++ b/pipe-cli/mount/hooks-py39/hook-prettytable.py @@ -0,0 +1,17 @@ +# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from PyInstaller.utils.hooks import collect_all + +datas, binaries, hiddenimports = collect_all('PTable')