Skip to content

Commit

Permalink
Add GC accessory UI test case (#19470)
Browse files Browse the repository at this point in the history 
Fix #19274

Signed-off-by: Yang Jiao <jiaoya@vmware.com>
  • Loading branch information
@YangJiao0817
YangJiao0817 authored Oct 23, 2023
1 parent 99b40bf commit f58dc2d
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 20 deletions.
25 changes: 13 additions & 12 deletions tests/apitests/python/test_garbage_collection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ def testGarbageCollectionAccessory(self):
"""
url = ADMIN_CLIENT["endpoint"]
user_password = "Aa123456"
deleted_prefix = "delete blob from storage: "
# 1. Create user(UA)
_, user_name = self.user.create_user(user_password = user_password, **ADMIN_CLIENT)
user_client = dict(endpoint = url, username = user_name, password = user_password, with_accessory = True)
Expand Down Expand Up @@ -188,7 +189,7 @@ def testGarbageCollectionAccessory(self):

# 10. Get the GC log and check that the image(IA) Signature of signature is deleted
gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT)
self.assertIn(signature_signature_digest, gc_log)
self.assertIn(deleted_prefix + signature_signature_digest, gc_log)

# 11. Delete image(IA) Signature by user(UA)
self.artifact.delete_artifact(project_name, self.image, image_signature_digest, **user_client)
Expand All @@ -199,7 +200,7 @@ def testGarbageCollectionAccessory(self):

# 13. Get the GC log and check that the image(IA) Signature is deleted
gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT)
self.assertIn(image_signature_digest, gc_log)
self.assertIn(deleted_prefix + image_signature_digest, gc_log)

# 14. Delete image(IA) SBOM by user(UA)
self.artifact.delete_artifact(project_name, self.image, sbom_digest, **user_client)
Expand All @@ -210,8 +211,8 @@ def testGarbageCollectionAccessory(self):

# 16. Get the GC log and check that the image(IA) SBOM and Signature of SBOM is deleted
gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT)
self.assertIn(sbom_digest, gc_log)
self.assertIn(sbom_signature_digest, gc_log)
self.assertIn(deleted_prefix + sbom_digest, gc_log)
self.assertIn(deleted_prefix + sbom_signature_digest, gc_log)

# 17. Push image(IA) SBOM to project(PA) by user(UA)
self.sbom_path = files_directory + "sbom_test.json"
Expand Down Expand Up @@ -246,10 +247,10 @@ def testGarbageCollectionAccessory(self):

# 22. Get the GC log and check that it is not deleted
gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT)
self.assertNotIn(sbom_digest, gc_log)
self.assertNotIn(sbom_signature_digest, gc_log)
self.assertNotIn(image_signature_digest, gc_log)
self.assertNotIn(signature_signature_digest, gc_log)
self.assertNotIn(deleted_prefix + sbom_digest, gc_log)
self.assertNotIn(deleted_prefix + sbom_signature_digest, gc_log)
self.assertNotIn(deleted_prefix + image_signature_digest, gc_log)
self.assertNotIn(deleted_prefix + signature_signature_digest, gc_log)

# 23. Delete tag of image(IA) by user(UA)
self.artifact.delete_tag(project_name, self.image, self.tag, self.tag, **user_client)
Expand All @@ -261,10 +262,10 @@ def testGarbageCollectionAccessory(self):
# 25. Get the GC log and check that the image(IA) and all aeecssory is deleted
gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT)
self.assertIn(self.image, gc_log)
self.assertIn(sbom_digest, gc_log)
self.assertIn(sbom_signature_digest, gc_log)
self.assertIn(image_signature_digest, gc_log)
self.assertIn(signature_signature_digest, gc_log)
self.assertIn(deleted_prefix + sbom_digest, gc_log)
self.assertIn(deleted_prefix + sbom_signature_digest, gc_log)
self.assertIn(deleted_prefix + image_signature_digest, gc_log)
self.assertIn(deleted_prefix + signature_signature_digest, gc_log)


if __name__ == '__main__':
Expand Down
4 changes: 4 additions & 0 deletions tests/resources/Cosign_Util.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,7 @@ Cosign Verify
[Arguments] ${artifact} ${signed}
Run Keyword If ${signed}==${true} Wait Unitl Command Success cosign verify --key cosign.pub ${artifact}
... ELSE Command Should be Failed cosign verify --key cosign.pub ${artifact}

Cosign Push Sbom
[Arguments] ${artifact} ${sbom_path}=${files_directory}/sbom_test.json ${type}=spdx
Wait Unitl Command Success cosign attach sbom --allow-insecure-registry --registry-referrers-mode oci-1-1 --type ${type} --sbom ${sbom_path} ${artifact}
16 changes: 16 additions & 0 deletions tests/resources/Harbor-Pages/GC.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,19 @@ Set GC Schedule
Run Keyword If '${type}'=='custom' Run Keywords Retry Element Click ${vulnerability_dropdown_list_item_custom} AND Retry Text Input ${targetCron_id} ${value}
... ELSE Retry Element Click ${vulnerability_dropdown_list_item_none}
Retry Double Keywords When Error Retry Element Click ${GC_schedule_save_btn} Retry Wait Until Page Not Contains Element ${gc_schedule_save_btn}

Go To GC Log
[Arguments] ${gc_job_id}
Retry Link Click //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//a
Switch Window locator=NEW

Check GC Log
[Arguments] ${gc_job_id} ${log_containing} ${log_excluding}
Go To GC Log ${gc_job_id}
FOR ${log} IN @{log_containing}
Wait Until Page Contains ${log}
END
FOR ${log} IN @{log_excluding}
Wait Until Page Does Not Contain ${log}
END
Switch Window locator=MAIN
14 changes: 14 additions & 0 deletions tests/resources/Harbor-Pages/Project-Artifact-Elements.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,17 @@ ${harbor_helm_name} harbor-helm-1.7.3
${harbor_helm_filename} harbor-helm-1.7.3.tar.gz
${harbor_helm_version} 1.7.3
${harbor_helm_package} harbor-1.7.3.tgz

${artifact_list_accessory_btn} (//clr-dg-row//button)[1]
${artifact_cosign_accessory} //clr-dg-row//clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div[text()=' signature.cosign ']]
${artifact_sbom_accessory} //clr-dg-row//clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div[text()=' subject.accessory ']]
${artifact_cosign_accessory_action_btn} (//clr-dg-row//clr-dg-row[.//div[text()=' signature.cosign ']]//button)[1]
${artifact_sbom_accessory_action_btn} (//clr-dg-row//clr-dg-row[.//div[text()=' subject.accessory ']]//button)[1]
${artifact_cosign_cosign_accessory_action_btn} ${artifact_cosign_accessory}//clr-dg-row//button
${artifact_sbom_cosign_accessory_action_btn} ${artifact_sbom_accessory}//clr-dg-row//button
${artifact_list_cosign_accessory_btn} (${artifact_cosign_accessory}//button)[2]
${artifact_list_sbom_accessory_btn} (${artifact_sbom_accessory}//button)[2]
${copy_digest_btn} //button[text()=' Copy Digest ']
${delete_accessory_btn} //button[text()=' Delete ']
${copy_btn} //button[text()=' COPY ']
${artifact_digest} //textarea
8 changes: 8 additions & 0 deletions tests/resources/Harbor-Pages/Project-Artifact.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,11 @@ Get The Specific Artifact
${cmd}= Set Variable curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -s --insecure -H "Content-Type: application/json" -X GET "${HARBOR_URL}/api/v2.0/projects/${project_name}/repositories/${repo_name}/artifacts/${reference}?page=1&page_size=10&with_tag=true&with_label=true&with_scan_overview=true&with_accessory=true&with_signature=true&with_immutable_status=true"
${rc} ${output}= Run And Return Rc And Output ${cmd}
[Return] ${output}

Refresh Artifacts
Retry Element Click ${artifact_list_refresh_btn}

Delete Accessory By Aeecssory XPath
[Arguments] ${aeecssory_action_xpath}
Retry Double Keywords When Error Retry Button Click ${aeecssory_action_xpath} Retry Button Click ${delete_accessory_btn}
Retry Button Click //button[text()=' DELETE ']
35 changes: 35 additions & 0 deletions tests/resources/TestCaseBody.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,3 +667,38 @@ Reset Schedules For Job Service Dashboard Schedules
# Reset the schedule of log rotation
Switch To Log Rotation
Set Log Rotation Schedule 2 Days None

Prepare Accessory
[Arguments] ${project_name} ${image} ${tag} ${user} ${pwd}
Docker Login ${ip} ${user} ${pwd}
Cosign Generate Key Pair
Cosign Sign ${ip}/${project_name}/${image}:${tag}
Cosign Push Sbom ${ip}/${project_name}/${image}:${tag}
Go Into Repo ${project_name} ${image}
Retry Button Click ${artifact_list_accessory_btn}
# Get SBOM digest
Retry Double Keywords When Error Retry Button Click ${artifact_sbom_accessory_action_btn} Retry Button Click ${copy_digest_btn}
Wait Until Element Is Visible And Enabled ${artifact_digest}
${sbom_digest}= Get Text ${artifact_digest}
Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn}
# Get Signature digest
Retry Double Keywords When Error Retry Button Click ${artifact_cosign_accessory_action_btn} Retry Button Click ${copy_digest_btn}
Wait Until Element Is Visible And Enabled ${artifact_digest}
${signature_digest}= Get Text ${artifact_digest}
Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn}
Cosign Sign ${ip}/${project_name}/${image}@${sbom_digest}
Cosign Sign ${ip}/${project_name}/${image}@${signature_digest}
Refresh Artifacts
Retry Button Click ${artifact_list_accessory_btn}
# Get Signature of SBOM digest
Retry Double Keywords When Error Retry Button Click ${artifact_list_sbom_accessory_btn} Retry Button Click ${artifact_sbom_cosign_accessory_action_btn}
Retry Double Keywords When Error Retry Button Click ${copy_digest_btn} Wait Until Element Is Visible And Enabled ${artifact_digest}
${signature_of_sbom_digest}= Get Text ${artifact_digest}
Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn}
# Get Signature of Signature digest
Retry Double Keywords When Error Retry Button Click ${artifact_list_cosign_accessory_btn} Retry Button Click ${artifact_cosign_cosign_accessory_action_btn}
Retry Double Keywords When Error Retry Button Click ${copy_digest_btn} Wait Until Element Is Visible And Enabled ${artifact_digest}
${signature_of_signature_digest}= Get Text ${artifact_digest}
Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn}
Docker Logout ${ip}
[Return] ${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest}
82 changes: 74 additions & 8 deletions tests/robot-cases/Group1-Nightly/Common_GC.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ ${HARBOR_ADMIN} admin
*** Test Cases ***
Test Case - Garbage Collection
Init Chrome Driver
${d}= Get Current Date result_format=%m%s
${d}= Get Current Date result_format=%m%s
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
GC Now
Create An New Project And Go Into Project project${d}
Expand All @@ -39,22 +39,22 @@ Test Case - Garbage Collection

Test Case - GC Untagged Images
Init Chrome Driver
${d}= Get Current Date result_format=%m%s
${d}= Get Current Date result_format=%m%s
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
GC Now
Create An New Project And Go Into Project project${d}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest
# make hello-world untagged
Go Into Repo project${d} hello-world
Go Into Artifact latest
Should Contain Tag latest
Delete A Tag latest
Should Not Contain Tag latest
Go Into Artifact latest
Should Contain Tag latest
Delete A Tag latest
Should Not Contain Tag latest
# run gc without param delete untagged artifacts checked, should not delete hello-world:latest
GC Now
${latest_job_id}= Get Text ${latest_job_id_xpath}
Retry GC Should Be Successful ${latest_job_id} ${null}
Go Into Repo project${d} hello-world
Go Into Repo project${d} hello-world
Should Contain Artifact
# run gc with param delete untagged artifacts checked, should delete hello-world
Switch To Garbage Collection
Expand All @@ -72,7 +72,7 @@ Test Case - Project Quotas Control Under GC
${storage_quota}= Set Variable 200
${storage_quota_unit}= Set Variable MiB
${image_a}= Set Variable logstash
${image_a_size}= Set Variable 321.03MiB
${image_a_size}= Set Variable 321.03MiB
${image_a_ver}= Set Variable 6.8.3
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
GC Now
Expand All @@ -89,3 +89,69 @@ Test Case - Project Quotas Control Under GC
END
Should Be Equal As Strings '${out2[0]}' 'PASS'
Close Browser

Test Case - Garbage Collection Accessory
Init Chrome Driver
${d}= Get Current Date result_format=%m%s
${image}= Set Variable hello-world
${tag}= Set Variable latest
${deleted_prefix}= Set Variable delete blob from storage:
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
GC Now
Create An New Project And Go Into Project project${d}
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image}

${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest}= Prepare Accessory project${d} ${image} ${tag} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
# Delete the Signature of Signature
Delete Accessory By Aeecssory XPath ${artifact_cosign_cosign_accessory_action_btn}
${gc_job_id}= GC Now
Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS']
${log_containing}= Create List ${deleted_prefix} ${signature_of_signature_digest}
${log_excluding}= Create List ${deleted_prefix} ${sbom_digest}
... ${deleted_prefix} ${signature_of_sbom_digest}
... ${deleted_prefix} ${signature_digest}
Check GC Log ${gc_job_id} ${log_containing} ${log_excluding}
Go Into Repo project${d} ${image}
Retry Button Click ${artifact_list_accessory_btn}
# Delete the Signature
Delete Accessory By Aeecssory XPath ${artifact_cosign_accessory_action_btn}
${gc_job_id}= GC Now
Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS']
${log_containing}= Create List ${deleted_prefix} ${signature_digest}
${log_excluding}= Create List ${deleted_prefix} ${sbom_digest}
... ${deleted_prefix} ${signature_of_sbom_digest}
Check GC Log ${gc_job_id} ${log_containing} ${log_excluding}
Go Into Repo project${d} ${image}
Retry Button Click ${artifact_list_accessory_btn}
# Delete the SBOM
Delete Accessory By Aeecssory XPath ${artifact_sbom_accessory_action_btn}
${gc_job_id}= GC Now
Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS']
${log_containing}= Create List ${deleted_prefix} ${sbom_digest}
... ${deleted_prefix} ${signature_of_sbom_digest}
${log_excluding}= Create List
Check GC Log ${gc_job_id} ${log_containing} ${log_excluding}

${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest}= Prepare Accessory project${d} ${image} ${tag} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
# Delete image tags
Go Into Repo project${d} ${image}
Go Into Artifact ${tag}
Should Contain Tag ${tag}
Delete A Tag ${tag}
${gc_job_id}= GC Now
Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS']
${log_containing}= Create List
${log_excluding}= Create List ${deleted_prefix} ${sbom_digest}
... ${deleted_prefix} ${signature_digest}
... ${deleted_prefix} ${signature_of_sbom_digest}
... ${deleted_prefix} ${signature_of_signature_digest}
Check GC Log ${gc_job_id} ${log_containing} ${log_excluding}
${gc_job_id}= GC Now untag=${true}
Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS']
${log_containing}= Create List ${deleted_prefix} ${sbom_digest}
... ${deleted_prefix} ${signature_digest}
... ${deleted_prefix} ${signature_of_sbom_digest}
... ${deleted_prefix} ${signature_of_signature_digest}
${log_excluding}= Create List
Check GC Log ${gc_job_id} ${log_containing} ${log_excluding}
Close Browser

0 comments on commit f58dc2d

Please sign in to comment.