Skip to content

Commit 3315454

Browse files
vishwajeet-ibmVishwajeet-Rajopadhye1
vishwajeet-ibm
and
Vishwajeet-Rajopadhye1
authored
GHA changes (#63)
* gha-new-changes * Update currency-build.yaml * Update currency-build.yaml * Update currency-build.yaml * Update upload_file.sh * Update upload_docker_image.sh * Update upload_wheel.sh * Update upload_docker_image.sh * Update currency-build.yaml * Update ca_config.py * Update ca_config.py * Update upload_file.sh * Update upload_file.sh * Update ca_config.py * Update upload_wheel.sh * Update upload_wheel.sh * Update ca_config.py * Update upload_docker_image.sh * Update COSWrapper.py * Update COSWrapper.py * Update currency-build.yaml * Update COSWrapper.py * Update upload_docker_image.sh * Update ca_config.py * Update COSWrapper.py * Update COSWrapper.py * Update ca_config.py --------- Co-authored-by: Vishwajeet-Rajopadhye1 <Vishwajeet-Rajopadhye1@ibm.com>
1 parent e5ce3ba commit 3315454

18 files changed

+1994
-25
lines changed

‎.github/workflows/currency-build.yaml‎

Lines changed: 76 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,8 @@ jobs:
127127

128128

129129
- name: Build Package
130-
#env:
131-
#CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.CURRENCY_SERVICE_ID_API_KEY }}
130+
env:
131+
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
132132
run: |
133133
echo "Sourcing variable.sh and scanner-env.sh..........................................."
134134
source package-cache/variable.sh
@@ -154,8 +154,8 @@ jobs:
154154
bash ../gha-script/pre_process.sh
155155
cd $GITHUB_WORKSPACE
156156
gzip build_log
157-
# chmod +x ./gha-script/upload-scripts/upload_file.sh
158-
# bash ./gha-script/upload-scripts/upload_file.sh build_log.gz
157+
chmod +x ./gha-script/upload-scripts/upload_file.sh
158+
bash ./gha-script/upload-scripts/upload_file.sh build_log.gz
159159
160160
161161
@@ -193,6 +193,7 @@ jobs:
193193
- python-version: "3.13"
194194
allow-failure: true
195195
env:
196+
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
196197
PYTHON_VERSION: ${{ matrix.python-version }}
197198
PACKAGE_NAME: ${{ inputs.package_name }}
198199
VERSION: ${{ inputs.version }}
@@ -231,14 +232,14 @@ jobs:
231232
232233
continue-on-error: ${{ matrix.allow-failure }}
233234

234-
# - name: Upload wheel artifact using custom script
235-
# run: |
236-
# WHEEL_FILE=*.whl
237-
# chmod +x ./gha-script/upload-scripts/upload_wheel.sh
238-
# bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE
239-
# continue-on-error: ${{ matrix.allow-failure }}
240-
241-
235+
- name: Upload wheel artifact using custom script
236+
run: |
237+
sudo chmod a+r *.whl
238+
WHEEL_FILE=*.whl
239+
chmod +x ./gha-script/upload-scripts/upload_wheel.sh
240+
bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE
241+
continue-on-error: ${{ matrix.allow-failure }}
242+
242243
source_scanner:
243244
needs: build
244245
if: ${{ inputs.validate_build_script == 'true' }}
@@ -306,11 +307,13 @@ jobs:
306307
307308
- name: Upload Scanner Results
308309
env:
309-
gha_currency_service_id_api_key_dev: ${{ secrets.gha_currency_service_id_api_key_dev }}
310+
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
310311
PACKAGE_NAME: ${{ inputs.package_name }}
311312
VERSION: ${{ inputs.version }}
312313
run: |
313314
mkdir source
315+
source package-cache/variable.sh
316+
source package-cache/scanner-env.sh
314317
find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true
315318
find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true
316319
find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true
@@ -343,9 +346,13 @@ jobs:
343346
docker images
344347
345348
- name: Upload Docker Image
349+
env:
350+
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
346351
run: |
347-
#chmod +x ./gha-script/upload-scripts/upload_docker_image.sh
348-
#bash ./gha-script/upload-scripts/upload_docker_image.sh
352+
source package-cache/variable.sh
353+
source package-cache/scanner-env.sh
354+
chmod +x ./gha-script/upload-scripts/upload_docker_image.sh
355+
bash ./gha-script/upload-scripts/upload_docker_image.sh
349356
350357
- name: Upload package-cache with image.tar
351358
uses: actions/upload-artifact@v4
@@ -408,11 +415,63 @@ jobs:
408415
mv grype_image_sbom_results.json grype_image_vulnerabilities_results.json image_grype
409416
410417
- name: Upload Image Scanner Results
418+
env:
419+
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
411420
run: |
412421
mkdir image
422+
source package-cache/variable.sh
423+
source package-cache/scanner-env.sh
413424
find image_trivy -mindepth 1 -exec mv -t image {} + 2>/dev/null || true
414425
find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true
415426
find image_grype -mindepth 1 -exec mv -t image {} + 2>/dev/null || true
416427
tar cvzf image_scanner.tar.gz image
417-
#chmod +x ./gha-script/upload-scripts/upload_file.sh
418-
#bash ./gha-script/upload-scripts/upload_file.sh image_scanner.tar.gz
428+
chmod +x ./gha-script/upload-scripts/upload_file.sh
429+
bash ./gha-script/upload-scripts/upload_file.sh image_scanner.tar.gz
430+
431+
final_summary:
432+
name: Final Summary Stage
433+
needs: [build, source_scanner]
434+
runs-on: ubuntu-24.04-ppc64le-p10
435+
436+
steps:
437+
- name: Checkout code
438+
uses: actions/checkout@v4
439+
- name: Download package-cache
440+
uses: actions/download-artifact@v4
441+
with:
442+
name: package-cache
443+
path: package-cache
444+
445+
- name: Create and activate venv
446+
run: |
447+
python -m venv venv
448+
source venv/bin/activate
449+
python -m pip install --upgrade pip
450+
pip install requests deprecated xlsxwriter packaging
451+
452+
# Export environment variables
453+
export PYTHONPATH=$(pwd):$(pwd)/process_bom/
454+
455+
- name: Summarize build results
456+
env :
457+
PACKAGE_NAME: ${{ inputs.package_name }}
458+
VERSION: ${{ inputs.version }}
459+
IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }}
460+
SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }}
461+
run: |
462+
echo "Sourcing variable.sh and scanner-env.sh..........................................."
463+
chmod +x package-cache/variable.sh
464+
chmod +x package-cache/scanner-env.sh
465+
source package-cache/variable.sh
466+
source package-cache/scanner-env.sh
467+
echo "===== variable.sh ====="
468+
cat package-cache/variable.sh
469+
echo "===== scanner-env.sh ====="
470+
cat package-cache/scanner-env.sh
471+
echo "-------------package name : ${PACKAGE_NAME}--------------------"
472+
echo "-------------version : ${VERSION}--------------------"
473+
pwd
474+
chmod +x process_bom/*.py || true
475+
# Export environment variables
476+
export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom"
477+
python -m process_bom.run_currency_processor

‎gha-script/scanner-scripts/scancode_wheel_scan.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash -e
2+
3+
validate_build_script=$VALIDATE_BUILD_SCRIPT
4+
cloned_package=$CLONED_PACKAGE
5+
6+
echo "----------Installing dependencies -----------------"
7+
sudo apt update -y && sudo apt install -y file git python3.12 python3.12-venv python3-pip python3.12-dev build-essential unzip patch wget tar libffi-dev zlib1g-dev libssl-dev libxml2-dev libxslt1-dev libicu-dev pkg-config
8+
9+
echo "----------Installed dependencies -----------------"
10+
git clone https://github.com/nexB/scancode-toolkit.git
11+
cd scancode-toolkit
12+
git checkout v32.4.0
13+
echo "-------------- Create venv ------------------"
14+
python --version
15+
echo "========================================="
16+
python3.12 -m venv venv
17+
source venv/bin/activate
18+
python3.12 -m pip install --upgrade pip setuptools wheel typecode pyahocorasick
19+
20+
echo "--------------- Apply changes ----------------"
21+
sed -i '/typecode\[full\] >= 30\.0\.1/s/^/ # /' setup.cfg
22+
sed -i '/extractcode\[full\] >= 31\.0\.0/s/^/ # /' setup.cfg
23+
sed -i '/typecode\[full\] >= 30\.0\.0/s/^/ # /' setup.cfg
24+
25+
echo "------------- Install scancode-toolkit ---------------"
26+
python3.12 -m pip install -e .
27+
python3.12 -m pip install click==8.0.4
28+
echo "------------- scancode version ---------------"
29+
scancode --version
30+
31+
echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
32+
cd ..
33+
ls
34+
echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
35+
cd package-cache/wheels
36+
37+
38+
for wheel in *.whl; do
39+
echo "Processing: $wheel"
40+
41+
base_name="${wheel%.whl}" # Strip .whl extension
42+
extract_dir="${base_name}_extract"
43+
output_json="${base_name}_output.json"
44+
45+
46+
echo "base name : $base_name"
47+
echo "extract_dir : $extract_dir"
48+
echo "output_json : $output_json"
49+
50+
51+
# Unzip the wheel
52+
unzip -q "$wheel" -d "$extract_dir"
53+
echo "------------- unzippied wheel ------------------------------"
54+
ls
55+
56+
57+
# Run scancode
58+
echo "------------------------------------------------------------"
59+
../../scancode-toolkit/venv/bin/scancode --license --package --json-pp "$output_json" "$extract_dir"
60+
61+
# Zip the result
62+
echo "------------------------- output files ---------------------"
63+
ls
64+
echo "------------------------------------------------------------"
65+
echo "Finished: $wheel"
66+
done

‎gha-script/upload-scripts/upload_docker_image.sh‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
#!/bin/bash -e
2-
3-
echo "$travis_currency_service_id_api_key" | docker login -u iamapikey --password-stdin icr.io
2+
echo "$GHA_CURRENCY_SERVICE_ID_API_KEY" | docker login -u iamapikey --password-stdin icr.io
43
if [ $? -ne 0 ]; then
54
echo "Docker login failed. Exiting script."
65
exit 1
76
fi
87
package_name=$(echo $PACKAGE_NAME | tr '[:upper:]' '[:lower:]')
9-
docker tag $IMAGE_NAME icr.io/ose4power-packages/$package_name-ppc64le:$VERSION
10-
docker push icr.io/ose4power-packages/$package_name-ppc64le:$VERSION
8+
docker tag $IMAGE_NAME icr.io/icr-ose4s390x-dev/$package_name-s390x:$VERSION
9+
docker push icr.io/icr-ose4s390x-dev/$package_name-s390x:$VERSION
1110
if [ $? -ne 0 ]; then
1211
echo "Docker push failed. Exiting script."
1312
exit 1

‎gha-script/upload-scripts/upload_file.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@
33
token_request=$(curl -X POST https://iam.cloud.ibm.com/identity/token \
44
-H "content-type: application/x-www-form-urlencoded" \
55
-H "accept: application/json" \
6-
-d "grant_type=urn%3Aibm%3Aparams%3Aoauth%3Agrant-type%3Aapikey&apikey=$gha_currency_service_id_api_key_dev")
7-
6+
-d "grant_type=urn%3Aibm%3Aparams%3Aoauth%3Agrant-type%3Aapikey&apikey=$GHA_CURRENCY_SERVICE_ID_API_KEY")
87
#token=$(echo "$token_request" | jq -r '.access_token')
98
#curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.us-south.cloud-object-storage.appdomain.cloud/ose-s390x-toolci-bucket-dev/$PACKAGE_NAME/$VERSION/$1"
109

‎gha-script/upload-scripts/upload_wheel.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
token_request=$(curl -X POST https://iam.cloud.ibm.com/identity/token \
44
-H "content-type: application/x-www-form-urlencoded" \
55
-H "accept: application/json" \
6-
-d "grant_type=urn%3Aibm%3Aparams%3Aoauth%3Agrant-type%3Aapikey&apikey=$currency_ecosystem_dev_service_api_key")
6+
-d "grant_type=urn%3Aibm%3Aparams%3Aoauth%3Agrant-type%3Aapikey&apikey=$GHA_CURRENCY_SERVICE_ID_API_KEY")
77

88
#token=$(echo "$token_request" | jq -r '.access_token')
99
#curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/gzip" -T $1 "https://s3.au-syd.cloud-object-storage.appdomain.cloud/currency-automation-toolci-bucket/$PACKAGE_NAME/$VERSION/$1"
@@ -13,7 +13,7 @@ if [[ $(echo "$token_request" | jq -r '.errorCode') == "null" ]]; then
1313
token=$(echo "$token_request" | jq -r '.access_token')
1414

1515
# curl command for uploading the file
16-
response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us-east.cloud-object-storage.appdomain.cloud/ose-power-artifacts/$PACKAGE_NAME/$VERSION/$1")
16+
response=$(curl -X PUT -H "Authorization: bearer $token" -H "Content-Type: application/octet-stream" -T $1 "https://s3.us-south.cloud-object-storage.appdomain.cloud/ose-s390x-artifacts/$PACKAGE_NAME/$VERSION/$1")
1717

1818
# Check if the PUT request was successful based on the absence of an <Error> block
1919
if ! echo "$response" | grep -q "<Error>"; then

0 commit comments

Comments
 (0)