From 95c356c3047bd58d6a0baacbe4ebc4f3c68a563d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Fri, 5 Jul 2024 18:07:30 +0300 Subject: [PATCH 1/6] install `paramiko` in CI container Signed-off-by: onur-ozkan --- .docker/Dockerfile.ci-container | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.docker/Dockerfile.ci-container b/.docker/Dockerfile.ci-container index d68cecd225..c56e686888 100644 --- a/.docker/Dockerfile.ci-container +++ b/.docker/Dockerfile.ci-container @@ -14,9 +14,14 @@ RUN apt-get install -y \ gnupg \ git \ zip \ + python3 \ + python3-pip \ + python-dev \ + libffi-dev \ sudo -RUN ln -s /usr/bin/python3 /bin/python +RUN pip3 install --upgrade pip +RUN pip3 install paramiko RUN apt install -y \ software-properties-common \ From aef430fc303f6b7e8f59eded44fd73b3ed36a55d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Fri, 5 Jul 2024 18:08:06 +0300 Subject: [PATCH 2/6] create artifact uploader python script Signed-off-by: onur-ozkan --- scripts/ci/upload_artifact.py | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 scripts/ci/upload_artifact.py diff --git a/scripts/ci/upload_artifact.py b/scripts/ci/upload_artifact.py new file mode 100755 index 0000000000..eb48194f98 --- /dev/null +++ b/scripts/ci/upload_artifact.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import os +import paramiko +import sys +from io import StringIO + +def upload_files(remote_host, remote_user, remote_port, ssh_private_key, source, target): + pk = StringIO(ssh_private_key) + ssh_key = paramiko.Ed25519Key.from_private_key(pk) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + ssh.connect(remote_host, port=remote_port, username=remote_user, pkey=ssh_key) + + sftp = ssh.open_sftp() + for root, dirs, files in os.walk(source): + for filename in files: + local_path = os.path.join(root, filename) + relative_path = os.path.relpath(local_path, source) + remote_path = os.path.join(target, relative_path).replace("\\", "/") + + remote_dir = os.path.dirname(remote_path) + try: + sftp.mkdir(remote_dir) + except OSError: + pass # Directory probably already exists + + sftp.put(local_path, remote_path) + + sftp.close() + print("Upload finished successfully") + + except Exception as e: + print("Upload failed: " + str(e)) + sys.exit(1) + + finally: + ssh.close() + +def main(): + if len(sys.argv) != 3: + print("Usage: `python upload_artifact.py `") + sys.exit(1) + + source = sys.argv[1] + target = sys.argv[2] + + remote_host = os.getenv('FILE_SERVER_HOST') + remote_user = os.getenv('FILE_SERVER_USERNAME') + remote_port = int(os.getenv('FILE_SERVER_PORT', 22)) + ssh_private_key = os.getenv('FILE_SERVER_KEY') + + if not (remote_host or remote_user or remote_port or ssh_private_key): + print("Missing environment variables for SSH connection") + sys.exit(1) + + upload_files(remote_host, remote_user, remote_port, ssh_private_key, source, target) + +if __name__ == "__main__": + main() From 8a6d1a9eb2815954d9249ee61e3fb378e4b8ca09 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Fri, 5 Jul 2024 18:08:25 +0300 Subject: [PATCH 3/6] use in-tree script for uploading artifacts Signed-off-by: onur-ozkan --- .github/workflows/dev-build.yml | 144 +++++++++++----------------- .github/workflows/release-build.yml | 136 +++++++++++++------------- 2 files changed, 120 insertions(+), 160 deletions(-) diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 18c500440b..9ae37d6295 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -74,18 +74,14 @@ jobs: zip $NAME target/release/kdf -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" - name: Login to dockerhub if: github.event_name != 'pull_request' && github.ref == 'refs/heads/dev' @@ -151,18 +147,14 @@ jobs: zip $NAME target/x86_64-apple-darwin/release/kdf -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" win-x86-64: timeout-minutes: 60 @@ -217,18 +209,14 @@ jobs: 7z a $NAME .\target\release\kdf.exe .\target\release\*.dll mv $NAME ./$Env:BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: garygrossgarten/github-action-scp@v0.8.0 - with: - host: ${{ secrets.FILE_SERVER_HOST }} - username: ${{ secrets.FILE_SERVER_USERNAME }} - port: ${{ secrets.FILE_SERVER_PORT }} - privateKey: ${{ secrets.FILE_SERVER_KEY }} - local: ${{ env.BRANCH_NAME }} - remote: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" mac-dylib-x86-64: timeout-minutes: 60 @@ -284,18 +272,14 @@ jobs: zip $NAME target/x86_64-apple-darwin/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" wasm: timeout-minutes: 60 @@ -352,18 +336,14 @@ jobs: mkdir $BRANCH_NAME mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" ios-aarch64: timeout-minutes: 60 @@ -419,18 +399,14 @@ jobs: zip $NAME target/aarch64-apple-ios/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" android-aarch64: timeout-minutes: 60 @@ -500,18 +476,14 @@ jobs: zip $NAME target/aarch64-linux-android/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" android-armv7: timeout-minutes: 60 @@ -581,18 +553,14 @@ jobs: zip $NAME target/armv7-linux-androideabi/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output + - name: Upload build artifact env: - AVAILABLE: ${{ secrets.FILE_SERVER_KEY }} - if: ${{ env.AVAILABLE != '' }} - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" deployment-commitment: if: github.event_name != 'pull_request' && github.ref == 'refs/heads/dev' diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 8d867bc31b..7097405464 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -68,15 +68,14 @@ jobs: zip $NAME target/release/kdf -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" - name: Login to dockerhub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} docker.io @@ -133,15 +132,14 @@ jobs: zip $NAME target/x86_64-apple-darwin/release/kdf -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" win-x86-64: timeout-minutes: 60 @@ -190,15 +188,14 @@ jobs: 7z a $NAME .\target\release\kdf.exe .\target\release\*.dll mv $NAME ./$Env:BRANCH_NAME/ - - name: Upload output - uses: garygrossgarten/github-action-scp@v0.8.0 - with: - host: ${{ secrets.FILE_SERVER_HOST }} - username: ${{ secrets.FILE_SERVER_USERNAME }} - port: ${{ secrets.FILE_SERVER_PORT }} - privateKey: ${{ secrets.FILE_SERVER_KEY }} - local: ${{ env.BRANCH_NAME }} - remote: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" mac-dylib-x86-64: timeout-minutes: 60 @@ -247,15 +244,14 @@ jobs: zip $NAME target/x86_64-apple-darwin/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" wasm: timeout-minutes: 60 @@ -309,15 +305,14 @@ jobs: mkdir $BRANCH_NAME mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" ios-aarch64: timeout-minutes: 60 @@ -367,15 +362,14 @@ jobs: zip $NAME target/aarch64-apple-ios/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" android-aarch64: timeout-minutes: 60 @@ -439,15 +433,14 @@ jobs: zip $NAME target/aarch64-linux-android/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" android-armv7: timeout-minutes: 60 @@ -511,12 +504,11 @@ jobs: zip $NAME target/armv7-linux-androideabi/release/libkdf.a -j mv $NAME ./$BRANCH_NAME/ - - name: Upload output - uses: easingthemes/ssh-deploy@v5.0.3 - with: - REMOTE_HOST: ${{ secrets.FILE_SERVER_HOST }} - REMOTE_USER: ${{ secrets.FILE_SERVER_USERNAME }} - REMOTE_PORT: ${{ secrets.FILE_SERVER_PORT }} - SSH_PRIVATE_KEY: ${{ secrets.FILE_SERVER_KEY }} - SOURCE: "${{ env.BRANCH_NAME }}/" - TARGET: "/uploads/${{ env.BRANCH_NAME }}" + - name: Upload build artifact + env: + FILE_SERVER_HOST: ${{ secrets.FILE_SERVER_HOST }} + FILE_SERVER_USERNAME: ${{ secrets.FILE_SERVER_USERNAME }} + FILE_SERVER_PORT: ${{ secrets.FILE_SERVER_PORT }} + FILE_SERVER_KEY: ${{ secrets.FILE_SERVER_KEY }} + if: ${{ env.FILE_SERVER_KEY != '' }} + run: python3 ./scripts/ci/upload_artifact.py "${{ env.BRANCH_NAME }}" "/uploads/${{ env.BRANCH_NAME }}" From 6b91e5d12d7447aec5a3800621c610da191e8ea4 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Fri, 5 Jul 2024 18:26:12 +0300 Subject: [PATCH 4/6] extend deps-install composite pipeline Signed-off-by: onur-ozkan --- .github/actions/deps-install/action.yml | 15 +++++++++++++++ .github/workflows/dev-build.yml | 8 ++++---- .github/workflows/release-build.yml | 8 ++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/actions/deps-install/action.yml b/.github/actions/deps-install/action.yml index bc87d7c418..042fba26bf 100644 --- a/.github/actions/deps-install/action.yml +++ b/.github/actions/deps-install/action.yml @@ -48,3 +48,18 @@ runs: run: | sudo apt-get update -y sudo apt-get install -y libudev-dev + + - name: Install python3 + uses: actions/setup-python@v5 + env: + TMP: ${{ inputs.temp || runner.temp }} + if: contains(inputs.deps, 'paramiko') + with: + python-version: 3 + + - name: Install `paramiko` pip package + env: + TMP: ${{ inputs.temp || runner.temp }} + if: contains(inputs.deps, 'paramiko') + shell: bash + run: pip install paramiko diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 9ae37d6295..e12d6385a5 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -109,7 +109,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -169,7 +169,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -232,7 +232,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -359,7 +359,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 7097405464..f93bb82135 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -100,7 +100,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -154,7 +154,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -210,7 +210,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -328,7 +328,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc') + deps: ('protoc', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' From aa32c064fb1c13601f33f52ef09331a6eb3edb52 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Mon, 8 Jul 2024 11:21:09 +0300 Subject: [PATCH 5/6] apply review notes Signed-off-by: onur-ozkan --- .github/actions/deps-install/action.yml | 6 +----- .github/workflows/dev-build.yml | 8 ++++---- .github/workflows/release-build.yml | 8 ++++---- scripts/ci/upload_artifact.py | 4 ++-- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/actions/deps-install/action.yml b/.github/actions/deps-install/action.yml index 042fba26bf..25ed15bf50 100644 --- a/.github/actions/deps-install/action.yml +++ b/.github/actions/deps-install/action.yml @@ -51,15 +51,11 @@ runs: - name: Install python3 uses: actions/setup-python@v5 - env: - TMP: ${{ inputs.temp || runner.temp }} - if: contains(inputs.deps, 'paramiko') + if: contains(inputs.deps, 'python3') with: python-version: 3 - name: Install `paramiko` pip package - env: - TMP: ${{ inputs.temp || runner.temp }} if: contains(inputs.deps, 'paramiko') shell: bash run: pip install paramiko diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index e12d6385a5..c060051f74 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -109,7 +109,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -169,7 +169,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -232,7 +232,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -359,7 +359,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index f93bb82135..259f9f8465 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -100,7 +100,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -154,7 +154,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -210,7 +210,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' @@ -328,7 +328,7 @@ jobs: - name: Install build deps uses: ./.github/actions/deps-install with: - deps: ('protoc', 'paramiko') + deps: ('protoc', 'python3', 'paramiko') - name: Calculate commit hash for PR commit if: github.event_name == 'pull_request' diff --git a/scripts/ci/upload_artifact.py b/scripts/ci/upload_artifact.py index eb48194f98..05dd9a7b9e 100755 --- a/scripts/ci/upload_artifact.py +++ b/scripts/ci/upload_artifact.py @@ -6,8 +6,8 @@ from io import StringIO def upload_files(remote_host, remote_user, remote_port, ssh_private_key, source, target): - pk = StringIO(ssh_private_key) - ssh_key = paramiko.Ed25519Key.from_private_key(pk) + privkey = StringIO(ssh_private_key) + ssh_key = paramiko.Ed25519Key.from_private_key(privkey) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) From bd962be27154c635e346a66cbf801d999745bfec Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Mon, 8 Jul 2024 11:26:18 +0300 Subject: [PATCH 6/6] apply review note Signed-off-by: onur-ozkan --- scripts/ci/upload_artifact.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/ci/upload_artifact.py b/scripts/ci/upload_artifact.py index 05dd9a7b9e..649e13acb0 100755 --- a/scripts/ci/upload_artifact.py +++ b/scripts/ci/upload_artifact.py @@ -16,29 +16,25 @@ def upload_files(remote_host, remote_user, remote_port, ssh_private_key, source, sftp = ssh.open_sftp() for root, dirs, files in os.walk(source): + relative_dir_path = os.path.relpath(root, source) + remote_dir_path = os.path.join(target, relative_dir_path).replace("\\", "/") + try: + sftp.mkdir(remote_dir_path) + except OSError: + pass # Directory probably already exists + for filename in files: local_path = os.path.join(root, filename) - relative_path = os.path.relpath(local_path, source) - remote_path = os.path.join(target, relative_path).replace("\\", "/") - - remote_dir = os.path.dirname(remote_path) - try: - sftp.mkdir(remote_dir) - except OSError: - pass # Directory probably already exists - + remote_path = os.path.join(target, relative_dir_path, filename).replace("\\", "/") sftp.put(local_path, remote_path) - sftp.close() print("Upload finished successfully") + ssh.close() except Exception as e: print("Upload failed: " + str(e)) sys.exit(1) - finally: - ssh.close() - def main(): if len(sys.argv) != 3: print("Usage: `python upload_artifact.py `")