diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 664c938c..f2479f43 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -115,7 +115,7 @@ jobs: DOCKERDEVARTIFACTORY: ${{ secrets.DOCKERDEVARTIFACTORY }} - name: Test - if: ${{ false }} # disable for now + #if: ${{ false }} # disable for now run: | $TestProjects = Get-ChildItem -Path *test*.csproj -Recurse -exclude TestUtilities.csproj,UnitTestUtilities.csproj Write-Host "**************************The test projects considered for execution: $TestProjects ******************************" diff --git a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/ConanComparisonBOM.json b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/ConanComparisonBOM.json index 8864835c..597917a2 100644 --- a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/ConanComparisonBOM.json +++ b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/ConanComparisonBOM.json @@ -51,20 +51,16 @@ }, { "Name": "internal:siemens:clearing:is-internal", - "Value": "false" + "Value": "true" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "energy-dev-conan-egll" }, { "Name": "internal:siemens:clearing:project-type", "Value": "CONAN" }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "Approved" - }, { "Name": "internal:siemens:clearing:sw360:release-url", "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/e13e0e564b004ef4adabbd01bf0b93ce" @@ -111,7 +107,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrogRepo" }, { diff --git a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/NPMComparisonBOM.json b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/NPMComparisonBOM.json index 7adaed50..b37c8026 100644 --- a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/NPMComparisonBOM.json +++ b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/NPMComparisonBOM.json @@ -1,122 +1,167 @@ { - "BomFormat": "CycloneDX", - "SpecVersion": 4, - "SpecVersionString": "1.4", - "SerialNumber": null, - "Version": null, - "Metadata": { - "Tools": [ - { - "Vendor": "Siemens AG", - "Name": "Clearing Automation Tool", - "Version": "1.0.20", - "Hashes": null - } - ], - "Authors": null, - "Component": null, - "Manufacture": null, - "Supplier": null - }, - "Components": [ - { - "Type": 0, - "MimeType": null, - "BomRef": "pkg:npm/requirejs@2.3.5", - "Supplier": null, - "Author": null, - "Publisher": null, - "Group": null, - "Name": "requirejs", - "Version": "2.3.5", - "Description": "", - "Scope": null, - "Hashes": null, - "Licenses": null, - "Copyright": null, - "Cpe": null, - "Purl": "pkg:npm/requirejs@2.3.5", - "Swid": null, - "Modified": null, - "Pedigree": null, - "Components": null, - "Properties": [ - { - "Name": "internal", - "Value": "false" - }, - { - "Name": "ArtifactoryRepoName", - "Value": "siparty-release-npm-egll" - }, - { - "Name": "ProjectType", - "Value": "NPM" - }, - { - "Name": "ApprovedStatus", - "Value": "APPROVED" - }, - { - "Name": "ReleaseLink", - "Value": "http://localhost:8090/resource/api/releases/887ae944b0864857807e1c8fe58ad767" - }, - { - "Name": "FossologyURL", - "Value": "http://localhost:8081/repo/?mod=view-license&upload=" - } - ], - "Evidence": null + "BomFormat": "CycloneDX", + "SpecVersion": 4, + "SpecVersionString": "1.4", + "SerialNumber": null, + "Version": null, + "Metadata": { + "Tools": [ + { + "Vendor": "Siemens AG", + "Name": "Clearing Automation Tool", + "Version": "1.0.20", + "Hashes": null + } + ], + "Authors": null, + "Component": null, + "Manufacture": null, + "Supplier": null }, - { - "Type": 0, - "MimeType": null, - "BomRef": "pkg:npm/requirejs@2.3.6", - "Supplier": null, - "Author": null, - "Publisher": null, - "Group": null, - "Name": "requirejs", - "Version": "2.3.6", - "Description": "", - "Scope": null, - "Hashes": null, - "Licenses": null, - "Copyright": null, - "Cpe": null, - "Purl": "pkg:npm/requirejs@2.3.6", - "Swid": null, - "Modified": null, - "Pedigree": null, - "Components": null, - "Properties": [ - { - "Name": "internal", - "Value": "false" - }, - { - "Name": "ArtifactoryRepoName", - "Value": "siparty-release-npm-egll" - }, - { - "Name": "ProjectType", - "Value": "NPM" - }, + "Components": [ { - "Name": "ApprovedStatus", - "Value": "APPROVED" + "Type": 0, + "MimeType": null, + "BomRef": "pkg:npm/requirejs@2.3.5", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "requirejs", + "Version": "2.3.5", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:npm/requirejs@2.3.5", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:development", + "Value": "true" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "siparty-release-npm-egll" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "NPM" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/887ae944b0864857807e1c8fe58ad767" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": "http://localhost:8081/repo/?mod=view-license&upload=" + } + ], + "Evidence": null }, { - "Name": "ReleaseLink", - "Value": "http://localhost:8090/resource/api/releases/ce7d8171244047de94df1fd4dd2083b7" + "Type": 0, + "MimeType": null, + "BomRef": "pkg:npm/requirejs@2.3.6", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "requirejs", + "Version": "2.3.6", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:npm/requirejs@2.3.6", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "siparty-release-npm-egll" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "NPM" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/887ae944b0864857807e1c8fe58ad767" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": "http://localhost:8081/repo/?mod=view-license&upload=" + } + ], + "Evidence": null }, - { - "Name": "FossologyURL", - "Value": "http://localhost:8081/repo/?mod=view-license&upload=" - } - ], - "Evidence": null - } - ], - "Compositions": null + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:npm/eGraphicsSuite@1.0.0", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "eGraphicsSuite", + "Version": "1.0.0", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:npm/eGraphicsSuite@1.0.0", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "Discovered" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "true" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "energy-dev-npm-egll" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "NPM" + } + ], + "Evidence": null + } + ], + "Compositions": null } \ No newline at end of file diff --git a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json index bad2c2d4..504829b8 100644 --- a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json +++ b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json @@ -9,7 +9,7 @@ { "Vendor": "Siemens AG", "Name": "Clearing Automation Tool", - "Version": "3.1.0", + "Version": "4.0.0", "Hashes": null } ], @@ -22,20 +22,20 @@ { "Type": 0, "MimeType": null, - "BomRef": "pkg:pypi/html5lib@1.1", + "BomRef": "pkg:pypi/requests@2.28.1", "Supplier": null, "Author": null, "Publisher": null, "Group": null, - "Name": "html5lib", - "Version": "1.1", + "Name": "requests", + "Version": "2.28.1", "Description": "", "Scope": null, "Hashes": null, "Licenses": null, "Copyright": null, "Cpe": null, - "Purl": "pkg:pypi/html5lib@1.1", + "Purl": "pkg:pypi/requests@2.28.1", "Swid": null, "Modified": null, "Pedigree": null, @@ -43,92 +43,23 @@ "Properties": [ { "Name": "internal:siemens:clearing:development", - "Value": "false" + "Value": "true" }, { "Name": "internal:siemens:clearing:identifier-type", - "Value": "ManuallyAdded" + "Value": "Discovered" }, { "Name": "internal:siemens:clearing:is-internal", "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "org1-pythonhosted-pypi-remote-cache" }, { "Name": "internal:siemens:clearing:project-type", "Value": "PYTHON" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "Approved" - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/f697515c180646c3b8b9b70ce6f2a2d8" - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": null - } - ], - "Evidence": null - }, - { - "Type": 0, - "MimeType": null, - "BomRef": "pkg:pypi/attrs@22.1.0", - "Supplier": null, - "Author": null, - "Publisher": null, - "Group": null, - "Name": "attrs", - "Version": "22.1.0", - "Description": "", - "Scope": null, - "Hashes": null, - "Licenses": null, - "Copyright": null, - "Cpe": null, - "Purl": "pkg:pypi/attrs@22.1.0", - "Swid": null, - "Modified": null, - "Pedigree": null, - "Components": null, - "Properties": [ - { - "Name": "internal:siemens:clearing:development", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:identifier-type", - "Value": "ManuallyAdded" - }, - { - "Name": "internal:siemens:clearing:is-internal", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:repo-url", - "Value": "Not Found in JFrogRepo" - }, - { - "Name": "internal:siemens:clearing:project-type", - "Value": "PYTHON" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "Approved" - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/37fcf2d0725a46b8b777c849111e9be4" - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": null } ], "Evidence": null @@ -136,20 +67,20 @@ { "Type": 0, "MimeType": null, - "BomRef": "pkg:pypi/beautifulsoup4@4.11.1", + "BomRef": "pkg:pypi/urllib3@1.26.17", "Supplier": null, "Author": null, "Publisher": null, "Group": null, - "Name": "beautifulsoup4", - "Version": "4.11.1", + "Name": "urllib3", + "Version": "1.26.17", "Description": "", "Scope": null, "Hashes": null, "Licenses": null, "Copyright": null, "Cpe": null, - "Purl": "pkg:pypi/beautifulsoup4@4.11.1", + "Purl": "pkg:pypi/urllib3@1.26.17", "Swid": null, "Modified": null, "Pedigree": null, @@ -161,91 +92,32 @@ }, { "Name": "internal:siemens:clearing:identifier-type", - "Value": "ManuallyAdded" + "Value": "Discovered" }, { "Name": "internal:siemens:clearing:is-internal", - "Value": "false" + "Value": "true" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "org1-pythonhosted-pypi-remote-cache" }, { "Name": "internal:siemens:clearing:project-type", "Value": "PYTHON" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "Approved" - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/4b92d5ac32f84d43a82f2c2c57771e6d" - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": null } ], "Evidence": null + } + ], + "Dependencies": [ + { + "Ref": "pkg:pypi/requests@2.28.1", + "Dependencies": null }, { - "Type": 0, - "MimeType": null, - "BomRef": "pkg:pypi/cachy@0.3.0", - "Supplier": null, - "Author": null, - "Publisher": null, - "Group": null, - "Name": "cachy", - "Version": "0.3.0", - "Description": "", - "Scope": null, - "Hashes": null, - "Licenses": null, - "Copyright": null, - "Cpe": null, - "Purl": "pkg:pypi/cachy@0.3.0", - "Swid": null, - "Modified": null, - "Pedigree": null, - "Components": null, - "Properties": [ - { - "Name": "internal:siemens:clearing:development", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:identifier-type", - "Value": "ManuallyAdded" - }, - { - "Name": "internal:siemens:clearing:is-internal", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:repo-url", - "Value": "org1-pythonhosted-pypi-remote-cache" - }, - { - "Name": "internal:siemens:clearing:project-type", - "Value": "PYTHON" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "Approved" - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/cece81ace4b14374af2432067bca5fb2" - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": null - } - ], - "Evidence": null + "Ref": "pkg:pypi/urllib3@1.26.17", + "Dependencies": null } ], "Compositions": null diff --git a/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Conan/conan.lock b/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Conan/conan.lock index 2a45747b..6367f390 100644 --- a/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Conan/conan.lock +++ b/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Conan/conan.lock @@ -5,146 +5,16 @@ "options": "libcurl:static=None\nopenssl:static=False", "requires": [ "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16" - ], - "build_requires": [ - "17" ], "path": "conan\\conanfile\\linux-x86_64", "context": "host" }, "1": { - "ref": "rapidjson/1.1.0-csc-01@siemens-energy/stable#6d624490b731387491675eebeff7ab66", + "ref": "rapidjson/1.1.0@siemens-energy/stable#6d624490b731387491675eebeff7ab66", "options": "", "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", "prev": "85ca839500f017c06cd5c39b4ad50c5e", "context": "host" - }, - "2": { - "ref": "libest/3.2.0-shared.5@siemens-energy/stable#0", - "options": "", - "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", - "prev": "0", - "context": "host" - }, - "3": { - "ref": "openssl/3.0.9-shared.3@siemens-energy/stable#0", - "options": "static=False", - "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", - "prev": "0", - "context": "host" - }, - "4": { - "ref": "sqlite/3.37.0-shared.1@siemens-energy/stable#0", - "options": "", - "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", - "prev": "0", - "context": "host" - }, - "5": { - "ref": "oss_mbedtls/2.28.2-shared@siemens-energy/stable#0", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "0", - "context": "host" - }, - "6": { - "ref": "mongoose/v7.11-csc-01@siemens-energy/stable#7084912b9de8ac5f93c2e8aa16c259a8", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "c4b1dedbd2bd5223337ce892732c693e", - "context": "host" - }, - "7": { - "ref": "basics/1.0.8@siemens-energy/stable#0", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "0", - "context": "host" - }, - "8": { - "ref": "osal/1.0.30@siemens-energy/stable#21e224648b08c0c356e2e60d0f143ae7", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "f5db57253fc37159485da5d9d37d4844", - "context": "host" - }, - "9": { - "ref": "SecurityBasics/2.10.2@siemens-energy/stable#0", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "0", - "context": "host" - }, - "10": { - "ref": "securityaccessmanager/2.2.6@siemens-energy/stable#7522ada1783555e22afd338d9bd03d60", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "493721829e0ba6c4cccb9304acc9ad71", - "context": "host" - }, - "11": { - "ref": "SecurityEventLogger/2.0.24@siemens-energy/stable#ddd584e3d5551e3ba568f07b95a9e6af", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "f32d263a0ba627319365f9ee41781589", - "context": "host" - }, - "12": { - "ref": "securitypkimanager/2.6.3@siemens-energy/stable#a8d80c7af932e2062513e94fd2001077", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "33612b4203a7c283c1dd56aa795863f6", - "context": "host" - }, - "13": { - "ref": "SecurityStorageManager/2.11.2@siemens-energy/stable#6c98465724cb00ab842f16a6a17c64b6", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "8f87b12afc830478491b566d2b7bf7e1", - "context": "host" - }, - "14": { - "ref": "securitycommunicationmanager/2.6.5@siemens-energy/stable#fbacb77f419f7c1dc2af769841266fba", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "dcd1c2048a844e785cb79e918452d56e", - "context": "host" - }, - "15": { - "ref": "openldap/2.6.4-shared-ossl3.1@siemens-energy/stable#0", - "options": "", - "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", - "prev": "0", - "context": "host" - }, - "16": { - "ref": "libcurl/7.87.0-shared-ossl3.1@siemens-energy/stable#0", - "options": "static=None", - "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", - "prev": "0", - "context": "host" - }, - "17": { - "ref": "googletest/1.8.0@siemens-energy/stable#0", - "options": "", - "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - "prev": "0", - "context": "host" } }, "revisions_enabled": true diff --git a/TestFiles/MavenTestFile/ArtifactoryUploaderTestData/MavenComparisonBOM.json b/TestFiles/MavenTestFile/ArtifactoryUploaderTestData/MavenComparisonBOM.json index 217afd8b..554afc02 100644 --- a/TestFiles/MavenTestFile/ArtifactoryUploaderTestData/MavenComparisonBOM.json +++ b/TestFiles/MavenTestFile/ArtifactoryUploaderTestData/MavenComparisonBOM.json @@ -41,12 +41,16 @@ "Pedigree": null, "Components": null, "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "true" + }, { "Name": "internal:siemens:clearing:is-internal", "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "org1-bintray-maven-remote-cache" }, { @@ -92,10 +96,10 @@ "Properties": [ { "Name": "internal:siemens:clearing:is-internal", - "Value": "false" + "Value": "true" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "org1-bintray-maven-remote-cache" }, { @@ -144,7 +148,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrogRepo" }, { @@ -165,6 +169,55 @@ } ], "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:maven/multi-tenant-token-handler@3.0.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": "com/siemens/simaris", + "Name": "multi-tenant-token-handler", + "Version": "3.0.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:maven/multi-tenant-token-handler@3.0.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "true" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "energy-dev-maven-egll" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "maven" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Not Available" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/4c1cec751c2e40c0a2a357a7e305af28" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null } ], "Compositions": null diff --git a/doc/UsageDoc/CA_UsageDocument.md b/doc/UsageDoc/CA_UsageDocument.md index 7b9cdcfa..678523df 100644 --- a/doc/UsageDoc/CA_UsageDocument.md +++ b/doc/UsageDoc/CA_UsageDocument.md @@ -26,6 +26,8 @@ * [Continuous Clearing Tool Execution Test Mode](#continuous-clearing-tool-execution-test-mode) +* [Artifactory Uploader Release Execution](#artifactory-uploader-release-execution) + * [How to handle multiple project types in same project](#how-to-handle-multiple-project-types-in-same-project) * [Troubleshoot](#troubleshoot) @@ -79,7 +81,7 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and >**_2.Artifactory Token :_** - >>a)For enabling the upload of cleared packages into jfrog artifactory, user's need to have their own jfrog artifactory credentials.This includes a username and an Apikey. + >>a)For enabling the upload of cleared, internal and development packages into jfrog artifactory, user's need to have their own jfrog artifactory credentials.This includes a username and an Apikey. **Pipeline Configuration :** @@ -129,7 +131,9 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and `Note : Since the PackageIdentifier generates an SBOM file both Dev dependency and internal components will be existing in the BOM file.Make sure to set `RemoveDevDependency` Flag as true while running this exe` >**3. Artifactory Uploader** - - Processes the CycloneDXBOM file(i.e., the output of the SW360PackageCreator) and uploads the already cleared components(clearing state-Report approved) to the siparty release repo in Jfrog Artifactory.The components in the states other than "Report approved" will be handled by the clearing experts via the Continuous Clearing Dashboard. + - This processes the CycloneDXBOM file generated by the SW360PackageCreator. It targets components with an already cleared status (i.e., "Report approved") and facilitates the copy of these components from the remote repository to the "siparty release" repository in JFrog Artifactory. Additionally, it handles the copy of development components from the remote repository to the "siparty devdep" repository. Furthermore, internal packages are moved from the "energy-dev-" repository to the "energy-release-" repository. Components in states not meeting the above conditions are designated for handling by clearing experts through the Continuous Clearing Dashboard. + + `Note: The default setting for the Release flag is False. This flag is present to execute a dry run of the component copy/move operation. This dry run is instrumental in verifying the accuracy of the components' paths and permissions before the actual operation takes place.` ### **Prerequisite for Continuous Clearing Tool execution** @@ -210,11 +214,6 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "SW360URL": "", "Fossologyurl": "", "JFrogApi": "", - "JfrogNugetDestRepoName": "", - "JfrogNpmDestRepoName": "", - "JfrogMavenDestRepoName": "", - "JfrogPythonDestRepoName": "", - "JfrogConanDestRepoName": "", "PackageFilePath": "/mnt/Input", "BomFolderPath": "/mnt/Output", "BomFilePath":"/mnt/Output/_Bom.cdx.json", @@ -226,6 +225,7 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "ArtifactoryUploadUser": "",//This should be Jfrog user name "RemoveDevDependency": true, "EnableFossTrigger": true, + "Release": false, "InternalRepoList": [ "", //This should be the internal repo names in JFrog for NPM "",//This should be the internal repo names in JFrog for Nuget @@ -238,7 +238,11 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "JfrogNpmRepoList": [ "",//This is a mirror repo for npm registry in JFrog "", //This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", "ExcludedComponents": [] }, "Nuget": { @@ -247,7 +251,11 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "JfrogNugetRepoList": [ "",//This is a mirror repo for nuget.org in JFrog "",//This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", "ExcludedComponents": [] }, "Maven": { @@ -256,7 +264,11 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "JfrogMavenRepoList": [ "",//This is a mirror repo for repo.maven in JFrog "",//This should be the release repo.maven in JFrog + "" //This should be the development dependency repo in JFrog ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", "ExcludedComponents": [] }, "Debian": { @@ -270,7 +282,11 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "JfrogPythonRepoList": [ "", "",//This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", "ExcludedComponents": [] }, "Conan": { @@ -279,7 +295,11 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "JfrogConanRepoList": [ "", "", + "" //This should be the development dependency repo in JFrog ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", "ExcludedComponents": [] } @@ -306,13 +326,23 @@ Description for the settings in `appSettings.json` file | 14| --logfolderpath | Path to create log | No| If user wants to give configurable log path this parameter is used | | 15 | --fossologyurl | Fossology URL | Yes | https:// | Yes | | Optional (By default it will take from the Package Creator exe location | | | 16 | --artifactoryuploaduser | Jfrog User Email | Yes | -| 17 | --jfrognpmdestreponame | The destination folder name for the NPM package to be copied to | Yes | -| 18 | --jfrognugetdestreponame | The destination folder name for the Nuget package to be copied to | Yes | -| 19 | --jfrogmavendestreponame | The destination folder name for the Maven package to be copied to | Yes | | -| 20 | --jfrogpythondestreponame | The destination folder name for the Python package to be copied to | Yes | | -| 21 | --jfrogconandestreponame | The destination folder name for the Conan package to be copied to | Yes | | -| 22 | --timeout | SW360 response timeout value | No | | - +|17 | --release | Artifactory Uploader release mode |Optional (By default it will be set to False) | | +| 18 | --npm:jfrogthirdpartydestreponame | The destination folder name for the cleared NPM package to be copied to | Yes | +| 19 | --npm:jfroginternaldestreponame | The destination folder name for the internal NPM package to be moved to | Yes | +| 20 | --npm:jfrogdevdestreponame | The destination folder name for the development NPM package to be copied to | Yes | +| 21 | --nuget:jfrogthirdpartydestreponame | The destination folder name for the cleared Nuget package to be copied to | Yes | +| 22 | --nuget:jfroginternaldestreponame | The destination folder name for the internal Nuget package to be moved to | Yes | +| 23 | --nuget:jfrogdevdestreponame | The destination folder name for the development Nuget package to be copied to | Yes | +| 24 | --maven:jfrogthirdpartydestreponame | The destination folder name for the cleared Maven package to be copied to | Yes | +| 25 | --maven:jfroginternaldestreponame | The destination folder name for the internal Maven package to be moved to | Yes | +| 26 | --maven:jfrogdevdestreponame | The destination folder name for the development Maven package to be copied to | Yes | +| 27 | --python:jfrogthirdpartydestreponame | The destination folder name for the cleared Python package to be copied to | Yes | +| 28 | --python:jfroginternaldestreponame | The destination folder name for the internal Python package to be moved to | Yes | +| 29 | --python:jfrogdevdestreponame | The destination folder name for the development Python package to be copied to | Yes | +| 30 | --conan:jfrogthirdpartydestreponame | The destination folder name for the cleared Conan package to be copied to | Yes | +| 31 | --conan:jfroginternaldestreponame | The destination folder name for the internal Conan package to be moved to | Yes | +| 32 | --conan:jfrogdevdestreponame | The destination folder name for the development Conan package to be copied to | Yes | +| 33 | --timeout | SW360 response timeout value | No | | #### **Method 2** @@ -408,12 +438,25 @@ Continuous Clearing Tool can be executed as container or as binaries, - In order to execute the tool in test mode we need to pass an extra parameter to the existing argument list. - **Example** : `docker run --rm -it -v /D/Projects/Output:/mnt/Output -v /D/Projects/DockerLog:/var/log -v /D/Projects/CAConfig:/etc/CATool ghcr.io/siemens/continuous-clearing dotnet ArtifactoryUploader.dll --settingsfilepath /etc/CATool/appSettings.json --mode test` + **Example** : `docker run --rm -it -v /D/Projects/Output:/mnt/Output -v /D/Projects/DockerLog:/var/log -v /D/Projects/CAConfig:/etc/CATool ghcr.io/siemens/continuous-clearing dotnet SW360PackageCreator.dll --settingsfilepath /etc/CATool/appSettings.json --mode test` or - **Example** : `ArtifactoryUploader.exe --settingsfilepath //appSettings.json --mode test` + **Example** : `SW360PackageCreator.exe --settingsfilepath //appSettings.json --mode test` + +# Artifactory Uploader Release Execution + + By default, the release mode is set to `False`. This configuration is designed for the routine execution of the Artifactory uploader on a daily basis during the project's development phase. The primary objective is to continuously verify the accuracy of component paths and permissions before actual operations. + When the release mode is set to `True`, it indicates a shift towards deployment in a production environment. In this mode, the Artifactory uploader is prepared for live operations, signaling the transition from the verification stage to the actual copy/move of components. + + - In order to execute the tool in release mode we need to pass an extra parameter to the existing +argument list. + + **Example** : `docker run --rm -it -v /D/Projects/Output:/mnt/Output -v /D/Projects/DockerLog:/var/log -v /D/Projects/CAConfig:/etc/CATool ghcr.io/siemens/continuous-clearing dotnet ArtifactoryUploader.dll --settingsfilepath /etc/CATool/appSettings.json --release true` + + or + **Example** : `ArtifactoryUploader.exe --settingsfilepath //appSettings.json --release true` # How to handle multiple project types in same project diff --git a/doc/usagedocimg/artifactoryuploader.PNG b/doc/usagedocimg/artifactoryuploader.PNG index 53f5323d..a020c948 100644 Binary files a/doc/usagedocimg/artifactoryuploader.PNG and b/doc/usagedocimg/artifactoryuploader.PNG differ diff --git a/src/AritfactoryUploader.UTest/ArtifactoryUTTestFiles/CyclonedxBom.json b/src/AritfactoryUploader.UTest/ArtifactoryUTTestFiles/CyclonedxBom.json index bfbf6783..963e0925 100644 --- a/src/AritfactoryUploader.UTest/ArtifactoryUTTestFiles/CyclonedxBom.json +++ b/src/AritfactoryUploader.UTest/ArtifactoryUTTestFiles/CyclonedxBom.json @@ -53,7 +53,7 @@ }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "org1-npmjs-npm-remote" }, { @@ -102,21 +102,16 @@ "Properties": [ { "Name": "internal:siemens:clearing:is-internal", - "Value": "false" + "Value": "true" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { "Name": "internal:siemens:clearing:project-type", "Value": "Npm" }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "APPROVED" - - }, { "Name": "internal:siemens:clearing:sw360:release-url", "Value": "false" @@ -154,21 +149,16 @@ "Properties": [ { "Name": "internal:siemens:clearing:is-internal", - "Value": "false" + "Value": "true" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { "Name": "internal:siemens:clearing:project-type", "Value": "Npm" }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "APPROVED" - - }, { "Name": "internal:siemens:clearing:sw360:release-url", "Value": "false" @@ -203,35 +193,34 @@ "Modified": null, "Pedigree": null, "Components": null, - "Properties": [ - { - "Name": "internal:siemens:clearing:is-internal", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:repo-url", - "Value": "Not Found in JFrog" - }, - { - "Name": "internal:siemens:clearing:project-type", - "Value": "Npm" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "APPROVED" - - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "false" - - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": "false" - - } - ], + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "true" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "Not Found in JFrog" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "Npm" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "false" + + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": "false" + + } + ], "Evidence": null }, { @@ -255,35 +244,34 @@ "Modified": null, "Pedigree": null, "Components": null, - "Properties": [ - { - "Name": "internal:siemens:clearing:is-internal", - "Value": "false" - }, - { - "Name": "internal:siemens:clearing:repo-url", - "Value": "Not Found in JFrog" - }, - { - "Name": "internal:siemens:clearing:project-type", - "Value": "Npm" - }, - { - "Name": "internal:siemens:clearing:clearing-state", - "Value": "APPROVED" - - }, - { - "Name": "internal:siemens:clearing:sw360:release-url", - "Value": "false" - - }, - { - "Name": "internal:siemens:clearing:fossology:url", - "Value": "" - - } - ], + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "true" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-name", + "Value": "Not Found in JFrog" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "Npm" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "false" + + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": "" + + } + ], "Evidence": null }, { @@ -313,7 +301,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -365,7 +353,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -417,7 +405,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -469,7 +457,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -521,7 +509,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -573,7 +561,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "Not Found in JFrog" }, { @@ -633,7 +621,7 @@ "Value": "false" }, { - "Name": "internal:siemens:clearing:repo-url", + "Name": "internal:siemens:clearing:repo-name", "Value": "energy-dev-conan-egll" }, { diff --git a/src/AritfactoryUploader.UTest/ArtifactoryUploaderTest.cs b/src/AritfactoryUploader.UTest/ArtifactoryUploaderTest.cs new file mode 100644 index 00000000..34904f86 --- /dev/null +++ b/src/AritfactoryUploader.UTest/ArtifactoryUploaderTest.cs @@ -0,0 +1,99 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using LCT.APICommunications.Interfaces; +using LCT.APICommunications.Model; +using Moq; +using Newtonsoft.Json; +using System.Net; +using LCT.ArtifactoryUploader; +using System.Net.Http; +using System; +using NUnit.Framework; +using System.Threading.Tasks; +using LCT.APICommunications; +using LCT.Common; +using LCT.Facade.Interfaces; +using LCT.Facade; +using LCT.Services.Interface; +using LCT.Services; +using UnitTestUtilities; + +namespace AritfactoryUploader.UTest +{ + public class ArtifactoryUploader + { + [SetUp] + public void Setup() + { + // Method intentionally left empty. + } + + + [Test] + public async Task UploadPackageToRepo_InputEmptyCreds_ReturnsPackgeNotFound() + { + //Arrange + CommonAppSettings appSettings = new CommonAppSettings() + { + JFrogApi = UTParams.JFrogURL + }; + ArtfactoryUploader.jFrogService = GetJfrogService(appSettings); + var componentsToArtifactory = new ComponentsToArtifactory + { + Name = "html5lib", + PackageName = "html5lib", + Version = "1.1", + ComponentType = "PYTHON", + JfrogApi = "https://abc.jfrog.io/artifactory", + SrcRepoName = "org1-pythonhosted-pypi-remote-cache", + SrcRepoPathWithFullName = "org1-pythonhosted-pypi-remote-cache/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", + PypiCompName = "html5lib-1.1-py2.py3-none-any.whl", + DestRepoName = "pypi-test", + ApiKey = "", + Email = "", + CopyPackageApiUrl = "https://abc.jfrog.io/artifactory/api/copy/org1-pythonhosted-pypi-remote-cache/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl?to=/pypi-test/html5lib-1.1-py2.py3-none-any.whl&dry=1", + Path = "", + DryRun = true, + Purl = "pkg:pypi/html5lib@1.1", + JfrogPackageName = "html5lib-1.1-py2.py3-none-any.whl" + }; + + //Act + var responseMessage = await ArtfactoryUploader.UploadPackageToRepo(componentsToArtifactory, 100); + Assert.AreEqual(HttpStatusCode.NotFound, responseMessage.StatusCode); + Assert.AreEqual("Package Not Found", responseMessage.ReasonPhrase); + + } + + [Test] + public void SetConfigurationValues_InputEmptyCreds_ReturnsVoid() + { + //Arrange + bool returnValue = true; + + //Act + ArtfactoryUploader.SetConfigurationValues(); + + //Assert + Assert.That(returnValue, Is.True); + } + + private static IJFrogService GetJfrogService(CommonAppSettings appSettings) + { + ArtifactoryCredentials artifactoryUpload = new ArtifactoryCredentials() + { + ApiKey = appSettings.ArtifactoryUploadApiKey + }; + IJfrogAqlApiCommunication jfrogAqlApiCommunication = + new JfrogAqlApiCommunication(appSettings.JFrogApi, artifactoryUpload, appSettings.TimeOut); + IJfrogAqlApiCommunicationFacade jFrogApiCommunicationFacade = + new JfrogAqlApiCommunicationFacade(jfrogAqlApiCommunication); + IJFrogService jFrogService = new JFrogService(jFrogApiCommunicationFacade); + return jFrogService; + } + } +} \ No newline at end of file diff --git a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs index 138db43f..3e2a1b2f 100644 --- a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs +++ b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs @@ -15,6 +15,7 @@ using System.IO; using UnitTestUtilities; using System.Threading.Tasks; +using System.Linq; namespace AritfactoryUploader.UTest { @@ -67,7 +68,8 @@ public async Task GetComponentsToBeUploadedToArtifactory_GivenFewApprovedCompone { ArtifactoryUploadApiKey = "wfwfwfwfwegwgweg", ArtifactoryUploadUser = "user@account.com", - JfrogNpmDestRepoName = "npm-test", + Npm = new LCT.Common.Model.Config { JfrogThirdPartyDestRepoName = "npm-test" }, + Nuget = new LCT.Common.Model.Config { JfrogThirdPartyDestRepoName = "nuget-test" }, JfrogNpmSrcRepo = "remote-cache", JFrogApi = UTParams.JFrogURL, LogFolderPath = outFolder @@ -99,8 +101,8 @@ public async Task GetComponentsToBeUploadedToArtifactory_GivenAllApprovedCompone { ArtifactoryUploadApiKey = "wfwfwfwfwegwgweg", ArtifactoryUploadUser = "user@account.com", - JfrogNpmDestRepoName = "npm-test", - JfrogNpmSrcRepo = "remote-cache", + Npm = new LCT.Common.Model.Config { JfrogThirdPartyDestRepoName = "npm-test" }, + Nuget = new LCT.Common.Model.Config { JfrogThirdPartyDestRepoName = "nuget-test" }, JFrogApi = UTParams.JFrogURL, LogFolderPath = outFolder }; @@ -126,8 +128,10 @@ public async Task GetComponentsToBeUploadedToArtifactory_GivenNotApprovedCompone { ArtifactoryUploadApiKey = "wfwfwfwfwegwgweg", ArtifactoryUploadUser = "user@account.com", - JfrogNugetDestRepoName = "nuget-test", - JfrogNugetSrcRepo = "remote-cache", + Npm = new LCT.Common.Model.Config + { + JfrogThirdPartyDestRepoName = "nuget-test", + }, JFrogApi = UTParams.JFrogURL }; @@ -138,6 +142,57 @@ public async Task GetComponentsToBeUploadedToArtifactory_GivenNotApprovedCompone Assert.That(0, Is.EqualTo(uploadList.Count), "Checks for components to upload to be zero"); } + [Test] + public void UpdateBomArtifactoryRepoUrl_GivenBomAndComponentsUploadedToArtifactory_UpdatesBom() + { + //Arrange + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string comparisonBOMPath = outFolder + @"\ArtifactoryUTTestFiles\CyclonedxBom.json"; + Bom bom = PackageUploadHelper.GetComponentListFromComparisonBOM(comparisonBOMPath); + List components = new List() + { + new ComponentsToArtifactory() + { + Purl = "pkg:npm/%40angular/animations@11.0.4", + DestRepoName = "siparty-release-npm-egll", + DryRun = false, + } + }; + + //Act + PackageUploadHelper.UpdateBomArtifactoryRepoUrl(ref bom, components); + + //Assert + var repoUrl = bom.Components.First(x => x.Properties[1].Name == "internal:siemens:clearing:repo-name").Properties[1].Value; + Assert.AreEqual("siparty-release-npm-egll", repoUrl); + } + + [Test] + public void UpdateBomArtifactoryRepoUrl_GivenBomAndComponentsUploadedToArtifactoryDryRun_NoUpdateBom() + { + //Arrange + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string comparisonBOMPath = outFolder + @"\ArtifactoryUTTestFiles\CyclonedxBom.json"; + Bom bom = PackageUploadHelper.GetComponentListFromComparisonBOM(comparisonBOMPath); + List components = new List() + { + new ComponentsToArtifactory() + { + Purl = "pkg:npm/%40angular/animations@11.0.4", + DestRepoName = "siparty-release-npm-egll", + } + }; + + //Act + PackageUploadHelper.UpdateBomArtifactoryRepoUrl(ref bom, components); + + //Assert + var repoUrl = bom.Components.First(x => x.Properties[1].Name == "internal:siemens:clearing:repo-name").Properties[1].Value; + Assert.AreNotEqual("siparty-release-npm-egll", repoUrl); + } + private static List GetComponentList() { List componentLists = new List(); diff --git a/src/AritfactoryUploader.UTest/PackageUploaderTest.cs b/src/AritfactoryUploader.UTest/PackageUploaderTest.cs index ca2b9e3f..0b95de74 100644 --- a/src/AritfactoryUploader.UTest/PackageUploaderTest.cs +++ b/src/AritfactoryUploader.UTest/PackageUploaderTest.cs @@ -13,6 +13,13 @@ using System.Threading; using System.Threading.Tasks; using UnitTestUtilities; +using LCT.Services.Interface; +using LCT.APICommunications.Model; +using LCT.APICommunications.Interfaces; +using LCT.APICommunications; +using LCT.Facade.Interfaces; +using LCT.Facade; +using LCT.Services; namespace AritfactoryUploader.UTest { @@ -30,11 +37,22 @@ public async Task UploadPackageToArtifactory_GivenAppsettings() { BomFilePath = comparisonBOMPath, JFrogApi = UTParams.JFrogURL, - JfrogNpmDestRepoName = "npm-test", + Npm = new LCT.Common.Model.Config + { + JfrogThirdPartyDestRepoName = "npm-test", + }, + Conan = new LCT.Common.Model.Config + { + JfrogThirdPartyDestRepoName = "conan-test", + }, JfrogNpmSrcRepo = "test", - + TimeOut = 100, + Release = false }; + IJFrogService jFrogService = GetJfrogService(CommonAppSettings); + PackageUploadHelper.jFrogService = jFrogService; + Program.UploaderStopWatch = new Stopwatch(); Program.UploaderStopWatch.Start(); Thread.Sleep(10); @@ -43,7 +61,27 @@ public async Task UploadPackageToArtifactory_GivenAppsettings() await PackageUploader.UploadPackageToArtifactory(CommonAppSettings); // Assert - Assert.That(12, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesToBeUploaded), "Checks for no of components"); + Assert.That(8, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesToBeUploaded), "Checks for no of cleared third party components"); + Assert.That(2, Is.EqualTo(PackageUploader.uploaderKpiData.DevPackagesToBeUploaded), "Checks for no of development components"); + Assert.That(2, Is.EqualTo(PackageUploader.uploaderKpiData.InternalPackagesToBeUploaded), "Checks for no of internal components"); + Assert.That(12, Is.EqualTo(PackageUploader.uploaderKpiData.ComponentInComparisonBOM), "Checks for no of components in BOM"); + Assert.That(10, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesNotExistingInRemoteCache), "Checks for no of components not present in remote cache"); + Assert.That(2, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesNotUploadedDueToError), "Checks for no of components not uploaded due to error"); + } + + + private static IJFrogService GetJfrogService(CommonAppSettings appSettings) + { + ArtifactoryCredentials artifactoryUpload = new ArtifactoryCredentials() + { + ApiKey = appSettings.ArtifactoryUploadApiKey + }; + IJfrogAqlApiCommunication jfrogAqlApiCommunication = + new JfrogAqlApiCommunication(appSettings.JFrogApi, artifactoryUpload, appSettings.TimeOut); + IJfrogAqlApiCommunicationFacade jFrogApiCommunicationFacade = + new JfrogAqlApiCommunicationFacade(jfrogAqlApiCommunication); + IJFrogService jFrogService = new JFrogService(jFrogApiCommunicationFacade); + return jFrogService; } } } diff --git a/src/ArtifactoryUploader/ArtifactoryUploader.cs b/src/ArtifactoryUploader/ArtifactoryUploader.cs new file mode 100644 index 00000000..5b3d7333 --- /dev/null +++ b/src/ArtifactoryUploader/ArtifactoryUploader.cs @@ -0,0 +1,141 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using LCT.APICommunications; +using LCT.APICommunications.Interfaces; +using LCT.APICommunications.Model; +using LCT.APICommunications.Model.AQL; +using LCT.Services; +using LCT.Services.Interface; +using log4net; +using System; +using System.Configuration; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; + +namespace LCT.ArtifactoryUploader +{ + public static class ArtfactoryUploader + { + //ConfigurationAttribute + private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private static string destRepoName = Environment.GetEnvironmentVariable("JfrogDestRepoName"); + private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi"); + private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo"); + public static IJFrogService jFrogService { get; set; } + + public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout) + { + Logger.Debug("Starting UploadPackageToArtifactory method"); + string operationType = component.PackageType == PackageType.ClearedThirdParty ? "copy" : "move"; + string dryRunSuffix = component.DryRun ? " dry-run" : ""; + HttpResponseMessage responsemessage = new HttpResponseMessage(); + try + { + IJFrogApiCommunication jfrogApicommunication; + + // Package Information + var packageInfo = await GetPackageInfoWithRetry(jFrogService, component); + if (packageInfo == null) + { + return new HttpResponseMessage(HttpStatusCode.NotFound) + { + ReasonPhrase = ApiConstant.PackageNotFound + }; + } + + ArtifactoryCredentials repoCredentials = new ArtifactoryCredentials() + { + ApiKey = component.ApiKey, + Email = component.Email + }; + + // Initialize JFrog API communication based on Component Type + jfrogApicommunication = component.ComponentType?.ToUpperInvariant() switch + { + "MAVEN" => new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout), + "PYTHON" => new PythonJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout), + _ => new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout) + }; + + // Perform Copy or Move operation + responsemessage = component.PackageType switch + { + PackageType.ClearedThirdParty or PackageType.Development => await jfrogApicommunication.CopyFromRemoteRepo(component), + PackageType.Internal => await jfrogApicommunication.MoveFromRepo(component), + _ => new HttpResponseMessage(HttpStatusCode.NotFound) + }; + + // Check status code and handle errors + if (responsemessage.StatusCode != HttpStatusCode.OK) + { + responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload; + return responsemessage; + } + + Logger.Info($"Successful{dryRunSuffix} {operationType} package {component.PackageName}-{component.Version}" + + $" from {component.SrcRepoName} to {component.DestRepoName}"); + + } + catch (HttpRequestException ex) + { + Logger.Error($"Error has occurred in UploadPackageToArtifactory--{ex}"); + responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload; + return responsemessage; + } + finally + { + Logger.Debug("Ending UploadPackageToArtifactory method"); + } + + return responsemessage; + } + + /// + public static void SetConfigurationValues() + { + if (string.IsNullOrEmpty(destRepoName)) + { + destRepoName = ConfigurationManager.AppSettings["JfrogDestRepoName"]; + } + if (string.IsNullOrEmpty(JfrogApi)) + { + JfrogApi = ConfigurationManager.AppSettings["JfrogApi"]; + } + if (string.IsNullOrEmpty(srcRepoName)) + { + srcRepoName = ConfigurationManager.AppSettings["JfrogSrcRepo"]; + } + } + + private static async Task GetPackageInfoWithRetry(IJFrogService jFrogService, ComponentsToArtifactory component) + { + string srcRepoNameLower = component.SrcRepoName.ToLower(); + string packageNameLower = component.JfrogPackageName.ToLower(); + string pathLower = component.Path.ToLower(); + + var packageInfo = await jFrogService.GetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path); + + if (packageInfo == null) + { + // Retry with lowercase parameters + var lowercasePackageInfo = await jFrogService.GetPackageInfo(srcRepoNameLower, packageNameLower, pathLower); + + if (lowercasePackageInfo != null) + { + // Update the package API URL + component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower(); + packageInfo = lowercasePackageInfo; + } + } + + return packageInfo; + } + + } +} diff --git a/src/ArtifactoryUploader/Model/UploaderKpiData.cs b/src/ArtifactoryUploader/Model/UploaderKpiData.cs index fb3e970f..393b69b7 100644 --- a/src/ArtifactoryUploader/Model/UploaderKpiData.cs +++ b/src/ArtifactoryUploader/Model/UploaderKpiData.cs @@ -24,20 +24,38 @@ public class UploaderKpiData [DisplayName(@"Packages in Approved State")] public int PackagesToBeUploaded { get; set; } - [DisplayName(@"Packages Uploaded to Siparty Repo")] + [DisplayName(@"Packages Copied to Siparty Repo")] public int PackagesUploadedToJfrog { get; set; } - [DisplayName(@"Packages Not Uploaded to Siparty Repo")] + [DisplayName(@"Packages Not Copied to Siparty Repo")] public int PackagesNotUploadedToJfrog { get; set; } - [DisplayName(@"Packages Not Existing in Remote Cache")] + [DisplayName(@"Packages Not Existing in Repository")] public int PackagesNotExistingInRemoteCache { get; set; } - [DisplayName(@"Packages Not Uploaded Due To Error")] + [DisplayName(@"Packages Not Actioned Due To Error")] public int PackagesNotUploadedDueToError { get; set; } [DisplayName(@"Time taken by ComponentCreator")] public double TimeTakenByComponentCreator { get; set; } + [DisplayName(@"Development Packages to be Moved to Siparty DevDep Repo")] + public int DevPackagesToBeUploaded { get; set; } + + [DisplayName(@"Development Packages Moved to Siparty DevDep Repo")] + public int DevPackagesUploaded { get; set; } + + [DisplayName(@"Development Packages Not Moved to Siparty DevDep Repo")] + public int DevPackagesNotUploadedToJfrog { get; set; } + + [DisplayName(@"Internal Packages to be Moved")] + public int InternalPackagesToBeUploaded { get; set; } + + [DisplayName(@"Internal Packages Moved to Repo")] + public int InternalPackagesUploaded { get; set; } + + [DisplayName(@"Internal Packages Not Moved to Repo")] + public int InternalPackagesNotUploadedToJfrog { get; set; } + } } diff --git a/src/ArtifactoryUploader/PackageUploadHelper.cs b/src/ArtifactoryUploader/PackageUploadHelper.cs index 88b400f6..e8d3a546 100644 --- a/src/ArtifactoryUploader/PackageUploadHelper.cs +++ b/src/ArtifactoryUploader/PackageUploadHelper.cs @@ -71,15 +71,19 @@ public async static Task> GetComponentsToBeUploade foreach (var item in comparisonBomData) { - if (item.Properties.Exists(p => p.Name == Dataconstant.Cdx_ClearingState && p.Value.ToUpperInvariant() == "APPROVED")) + var packageType = GetPackageType(item); + if (packageType != PackageType.Unknown) { - AqlResult aqlResult = await GetSrcRepoDetailsForPyPiOrConanPackages(item, appSettings); + AqlResult aqlResult = await GetSrcRepoDetailsForPyPiOrConanPackages(item); ComponentsToArtifactory components = new ComponentsToArtifactory() { Name = !string.IsNullOrEmpty(item.Group) ? $"{item.Group}/{item.Name}" : item.Name, PackageName = item.Name, Version = item.Version, + Purl = item.Purl, ComponentType = GetComponentType(item), + PackageType = packageType, + DryRun = !appSettings.Release, SrcRepoName = item.Properties.Find(s => s.Name == Dataconstant.Cdx_ArtifactoryRepoUrl)?.Value, DestRepoName = GetDestinationRepo(item, appSettings), ApiKey = appSettings.ArtifactoryUploadApiKey, @@ -90,18 +94,18 @@ public async static Task> GetComponentsToBeUploade if (aqlResult != null) { components.SrcRepoPathWithFullName = aqlResult.Repo + "/" + aqlResult.Path + "/" + aqlResult.Name; - components.Path = GetConanPath(aqlResult.Path, $"{item.Name}/{item.Version}"); components.PypiCompName = aqlResult.Name; } else { components.SrcRepoPathWithFullName = string.Empty; - components.Path = string.Empty; components.PypiCompName = string.Empty; } - components.PackageInfoApiUrl = GetPackageInfoURL(components); + components.Path = GetPackagePath(components, aqlResult); components.CopyPackageApiUrl = GetCopyURL(components); + components.MovePackageApiUrl = GetMoveURL(components); + components.JfrogPackageName = GetJfrogPackageName(components); componentsToBeUploaded.Add(components); } else @@ -115,6 +119,30 @@ public async static Task> GetComponentsToBeUploade return componentsToBeUploaded; } + private static PackageType GetPackageType(Component item) + { + string GetPropertyValue(string propertyName) => + item.Properties + .Find(p => p.Name == propertyName)? + .Value? + .ToUpperInvariant(); + + if (GetPropertyValue(Dataconstant.Cdx_ClearingState) == "APPROVED") + { + return PackageType.ClearedThirdParty; + } + else if (GetPropertyValue(Dataconstant.Cdx_IsInternal) == "TRUE") + { + return PackageType.Internal; + } + else if (GetPropertyValue(Dataconstant.Cdx_IsDevelopment) == "TRUE") + { + return PackageType.Development; + } + + return PackageType.Unknown; + } + private static string GetCopyURL(ComponentsToArtifactory component) { string url = string.Empty; @@ -142,90 +170,151 @@ private static string GetCopyURL(ComponentsToArtifactory component) { url = $"{component.JfrogApi}{ApiConstant.CopyPackageApi}{component.SrcRepoName}/{component.Path}" + $"?to=/{component.DestRepoName}/{component.Path}"; + // Add a wild card to the path end for jFrog AQL query search + component.Path = $"{component.Path}/*"; } else { // Do nothing } - return url; - } - - private static string GetConanPath(string path, string package) - { - //// Get Path only till PackageName/Version so that everything in folder can be copied - if (path.Contains(package)) - { - int index = path.IndexOf(package); - return path.Substring(0, index + package.Length); - } - else - { - return path; - } + return component.DryRun ? $"{url}&dry=1" : url; } - private static string GetPackageInfoURL(ComponentsToArtifactory component) + private static string GetMoveURL(ComponentsToArtifactory component) { string url = string.Empty; if (component.ComponentType == "NPM") { - url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.Name}/-/{component.PackageName}-{component.Version}{ApiConstant.NpmExtension}"; + url = $"{component.JfrogApi}{ApiConstant.MovePackageApi}{component.SrcRepoName}/{component.Name}/-/{component.PackageName}-{component.Version}" + + $"{ApiConstant.NpmExtension}?to=/{component.DestRepoName}/{component.Name}/-/{component.PackageName}-{component.Version}{ApiConstant.NpmExtension}"; } else if (component.ComponentType == "NUGET") { - url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.PackageName}.{component.Version}{ApiConstant.NugetExtension}"; + url = $"{component.JfrogApi}{ApiConstant.MovePackageApi}{component.SrcRepoName}/{component.PackageName}.{component.Version}" + + $"{ApiConstant.NugetExtension}?to=/{component.DestRepoName}/{component.Name}.{component.Version}{ApiConstant.NugetExtension}"; } else if (component.ComponentType == "MAVEN") { - url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.Name}/{component.Version}"; + url = $"{component.JfrogApi}{ApiConstant.MovePackageApi}{component.SrcRepoName}/{component.Name}/{component.Version}" + + $"?to=/{component.DestRepoName}/{component.Name}/{component.Version}"; } else if (component.ComponentType == "PYTHON") { - url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoPathWithFullName}"; + url = $"{component.JfrogApi}{ApiConstant.MovePackageApi}{component.SrcRepoPathWithFullName}" + + $"?to=/{component.DestRepoName}/{component.PypiCompName}"; } else if (component.ComponentType == "CONAN") { - url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.Path}"; + url = $"{component.JfrogApi}{ApiConstant.MovePackageApi}{component.SrcRepoName}/{component.Path}" + + $"?to=/{component.DestRepoName}/{component.Path}"; + // Add a wild card to the path end for jFrog AQL query search + component.Path = $"{component.Path}/*"; } else { // Do nothing } - return url; + return component.DryRun ? $"{url}&dry=1" : url; } - private static string GetDestinationRepo(Component item, CommonAppSettings appSettings) + private static string GetPackagePath(ComponentsToArtifactory component, AqlResult aqlResult) { - if (item.Purl.Contains("npm", StringComparison.OrdinalIgnoreCase)) - { - return appSettings.JfrogNpmDestRepoName; - } - else if (item.Purl.Contains("nuget", StringComparison.OrdinalIgnoreCase)) + switch (component.ComponentType) { - return appSettings.JfrogNugetDestRepoName; - } - else if (item.Purl.Contains("maven", StringComparison.OrdinalIgnoreCase)) - { - return appSettings.JfrogMavenDestRepoName; - } - else if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase)) - { - return appSettings.JfrogPythonDestRepoName; + case "NPM": + return $"{component.Name}/-"; + + case "CONAN" when aqlResult != null: + string path = aqlResult.Path; + string package = $"{component.Name}/{component.Version}"; + + if (path.Contains(package)) + { + int index = path.IndexOf(package); + return path.Substring(0, index + package.Length); + } + else + { + return path; + } + + case "MAVEN": + return $"{component.Name}/{component.Version}"; + + default: + return string.Empty; } - else if (item.Purl.Contains("conan", StringComparison.OrdinalIgnoreCase)) + } + + private static string GetJfrogPackageName(ComponentsToArtifactory component) + { + string packageName; + + switch (component.ComponentType) { - return appSettings.JfrogConanDestRepoName; + case "NPM": + packageName = $"{component.PackageName}-{component.Version}{ApiConstant.NpmExtension}"; + break; + + case "NUGET": + packageName = $"{component.PackageName}.{component.Version}{ApiConstant.NugetExtension}"; + break; + + case "PYTHON": + packageName = component.PypiCompName; + break; + + default: + packageName = string.Empty; + break; } - else + + return packageName; + } + + private static string GetDestinationRepo(Component item, CommonAppSettings appSettings) + { + var packageType = GetPackageType(item); + var componentType = GetComponentType(item); + + if (!string.IsNullOrEmpty(componentType)) { - // Do nothing + switch (componentType.ToLower()) + { + case "npm": + return GetRepoName(packageType, appSettings.Npm.JfrogInternalDestRepoName, appSettings.Npm.JfrogDevDestRepoName, appSettings.Npm.JfrogThirdPartyDestRepoName); + case "nuget": + return GetRepoName(packageType, appSettings.Nuget.JfrogInternalDestRepoName, appSettings.Nuget.JfrogDevDestRepoName, appSettings.Nuget.JfrogThirdPartyDestRepoName); + case "maven": + return GetRepoName(packageType, appSettings.Maven.JfrogInternalDestRepoName, appSettings.Maven.JfrogDevDestRepoName, appSettings.Maven.JfrogThirdPartyDestRepoName); + case "python": + return GetRepoName(packageType, appSettings.Python.JfrogInternalDestRepoName, appSettings.Python.JfrogDevDestRepoName, appSettings.Python.JfrogThirdPartyDestRepoName); + case "conan": + return GetRepoName(packageType, appSettings.Conan.JfrogInternalDestRepoName, appSettings.Conan.JfrogDevDestRepoName, appSettings.Conan.JfrogThirdPartyDestRepoName); + } } return string.Empty; } + private static string GetRepoName(PackageType packageType, string internalRepo, string developmentRepo, string clearedThirdPartyRepo) + { + switch (packageType) + { + case PackageType.Internal: + return internalRepo; + case PackageType.Development: + return developmentRepo; + case PackageType.ClearedThirdParty: + return clearedThirdPartyRepo; + default: + return string.Empty; + } + } + private static string GetComponentType(Component item) { + if (item.Purl.Contains("npm", StringComparison.OrdinalIgnoreCase)) { return "NPM"; @@ -253,13 +342,12 @@ private static string GetComponentType(Component item) return string.Empty; } - private async static Task GetSrcRepoDetailsForPyPiOrConanPackages(Component item, CommonAppSettings appSettings) + private async static Task GetSrcRepoDetailsForPyPiOrConanPackages(Component item) { - if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase) && aqlResultList.Count == 0) + if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase)) { // get the component list from Jfrog for given repo - aqlResultList = await GetListOfComponentsFromRepo(appSettings.Python?.JfrogPythonRepoList, jFrogService); - + aqlResultList = await GetListOfComponentsFromRepo(new string[] { item.Properties.Find(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoUrl)?.Value }, jFrogService); if (aqlResultList.Count > 0) { return GetArtifactoryRepoName(aqlResultList, item); @@ -267,14 +355,14 @@ private async static Task GetSrcRepoDetailsForPyPiOrConanPackages(Com } else if (item.Purl.Contains("conan", StringComparison.OrdinalIgnoreCase)) { - var aqlConanResultList = await GetListOfComponentsFromRepo(new string[] { item.Properties.Where(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoUrl).FirstOrDefault()?.Value }, jFrogService); + var aqlConanResultList = await GetListOfComponentsFromRepo(new string[] { item.Properties.Find(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoUrl)?.Value }, jFrogService); if (aqlConanResultList.Count > 0) { return GetArtifactoryRepoNameForConan(aqlConanResultList, item); } } - + return null; } @@ -298,26 +386,32 @@ public static async Task UploadingThePackages(List comp private static async Task PackageUploadToArtifactory(UploaderKpiData uploaderKpiData, ComponentsToArtifactory item, int timeout) { - if (!(item.SrcRepoName.Contains("siparty"))) + var packageType = item.PackageType; + + if (!(item.SrcRepoName.Equals(item.DestRepoName, StringComparison.OrdinalIgnoreCase)) && !item.SrcRepoName.Contains("siparty-release")) { if (!(item.SrcRepoName.Contains("Not Found in JFrog"))) { + string operationType = item.PackageType == PackageType.ClearedThirdParty ? "copy" : "move"; + ArtfactoryUploader.jFrogService = jFrogService; HttpResponseMessage responseMessage = await ArtfactoryUploader.UploadPackageToRepo(item, timeout); - if (responseMessage.StatusCode == HttpStatusCode.OK) + + if (responseMessage.StatusCode == HttpStatusCode.OK && !item.DryRun) { - uploaderKpiData.PackagesUploadedToJfrog++; + IncrementCountersBasedOnPackageType(uploaderKpiData, packageType, true); } else if (responseMessage.ReasonPhrase == ApiConstant.PackageNotFound) { Logger.Error($"Package {item.Name}-{item.Version} not found in remote cache, Upload Failed!!"); - uploaderKpiData.PackagesNotUploadedToJfrog++; + IncrementCountersBasedOnPackageType(uploaderKpiData, packageType, false); + item.DestRepoName = null; SetWarningCode = true; } else if (responseMessage.ReasonPhrase == ApiConstant.ErrorInUpload) { - Logger.Error($"Package {item.Name}-{item.Version} Upload Failed!!"); - uploaderKpiData.PackagesNotUploadedToJfrog++; - uploaderKpiData.PackagesNotUploadedDueToError++; + Logger.Error($"Package {item.Name}-{item.Version} {operationType} Failed!!"); + IncrementCountersBasedOnPackageType(uploaderKpiData, packageType, false); + item.DestRepoName = null; } else { @@ -327,13 +421,15 @@ private static async Task PackageUploadToArtifactory(UploaderKpiData uploaderKpi else { uploaderKpiData.PackagesNotExistingInRemoteCache++; + item.DestRepoName = null; Logger.Warn($"Package {item.Name}-{item.Version} is not found in jfrog"); } } else { - uploaderKpiData.PackagesUploadedToJfrog++; - Logger.Info($"Package {item.Name}-{item.Version} is already uploaded to {item.DestRepoName}"); + IncrementCountersBasedOnPackageType(uploaderKpiData, packageType, true); + Logger.Info($"Package {item.Name}-{item.Version} is already uploaded"); + item.DestRepoName = null; } } @@ -354,6 +450,24 @@ public static void WriteCreatorKpiDataToConsole(UploaderKpiData uploaderKpiData) { CommonHelper.Convert(uploaderKpiData,nameof(uploaderKpiData.PackagesNotUploadedToJfrog)), uploaderKpiData.PackagesNotUploadedToJfrog}, + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.DevPackagesToBeUploaded)), + uploaderKpiData.DevPackagesToBeUploaded}, + + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.DevPackagesUploaded)), + uploaderKpiData.DevPackagesUploaded}, + + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.DevPackagesNotUploadedToJfrog)), + uploaderKpiData.DevPackagesNotUploadedToJfrog}, + + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.InternalPackagesToBeUploaded)), + uploaderKpiData.InternalPackagesToBeUploaded}, + + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.InternalPackagesUploaded)), + uploaderKpiData.InternalPackagesUploaded}, + + {CommonHelper.Convert(uploaderKpiData, nameof(uploaderKpiData.InternalPackagesNotUploadedToJfrog)), + uploaderKpiData.InternalPackagesNotUploadedToJfrog}, + { CommonHelper.Convert(uploaderKpiData,nameof(uploaderKpiData.PackagesNotExistingInRemoteCache)), uploaderKpiData.PackagesNotExistingInRemoteCache}, @@ -369,9 +483,42 @@ public static void WriteCreatorKpiDataToConsole(UploaderKpiData uploaderKpiData) CommonHelper.WriteToConsoleTable(printList, printTimingList); } - public static async Task> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService) + private static void IncrementCountersBasedOnPackageType(UploaderKpiData uploaderKpiData, PackageType packageType, bool isSuccess) + { + // Define a dictionary to map package types to counters + Dictionary successActions = new Dictionary + { + { PackageType.Internal, () => uploaderKpiData.InternalPackagesUploaded++ }, + { PackageType.Development, () => uploaderKpiData.DevPackagesUploaded++ }, + { PackageType.ClearedThirdParty, () => uploaderKpiData.PackagesUploadedToJfrog++ }, + }; + + Dictionary failureActions = new Dictionary + { + { PackageType.Internal, () => { uploaderKpiData.InternalPackagesNotUploadedToJfrog++; uploaderKpiData.PackagesNotUploadedDueToError++; } }, + { PackageType.Development, () => { uploaderKpiData.DevPackagesNotUploadedToJfrog++; uploaderKpiData.PackagesNotUploadedDueToError++; } }, + { PackageType.ClearedThirdParty, () => {uploaderKpiData.PackagesNotUploadedToJfrog++; uploaderKpiData.PackagesNotUploadedDueToError++; } }, + }; + + if (isSuccess) + { + if (successActions.TryGetValue(packageType, out var action)) + { + action.Invoke(); + } + } + else + { + if (failureActions.TryGetValue(packageType, out var action)) + { + action.Invoke(); + } + } + } + + private static async Task> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService) { - if (jFrogService != null && repoList != null && repoList.Length > 0) + if (repoList != null && repoList.Length > 0) { foreach (var repo in repoList) { @@ -401,5 +548,19 @@ private static AqlResult GetArtifactoryRepoNameForConan(List aqlResul return repoName; } + + public static void UpdateBomArtifactoryRepoUrl(ref Bom bom, List componentsUploaded) + { + foreach (var component in componentsUploaded) + { + var bomComponent = bom.Components.Find(x => x.Purl.Equals(component.Purl, StringComparison.OrdinalIgnoreCase)); + if (component.DestRepoName != null && !component.DryRun) + { + bomComponent.Properties.First(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoUrl).Value = component.DestRepoName; + } + } + } + } + } diff --git a/src/ArtifactoryUploader/PackageUploader.cs b/src/ArtifactoryUploader/PackageUploader.cs index 170ae9d6..097dd4e3 100644 --- a/src/ArtifactoryUploader/PackageUploader.cs +++ b/src/ArtifactoryUploader/PackageUploader.cs @@ -14,6 +14,9 @@ using System.Reflection; using System.Threading.Tasks; using LCT.Common; +using System.Linq; +using LCT.Common.Constants; +using System.IO; namespace LCT.ArtifactoryUploader { @@ -36,9 +39,18 @@ public static async Task UploadPackageToArtifactory(CommonAppSettings appSetting List m_ComponentsToBeUploaded = await PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(m_ComponentsInBOM.Components, appSettings); //Uploading the component to artifactory - uploaderKpiData.PackagesToBeUploaded = m_ComponentsToBeUploaded.Count; + uploaderKpiData.PackagesToBeUploaded = m_ComponentsToBeUploaded.Count(x => x.PackageType == PackageType.ClearedThirdParty); + uploaderKpiData.DevPackagesToBeUploaded = m_ComponentsToBeUploaded.Count(x => x.PackageType == PackageType.Development); + uploaderKpiData.InternalPackagesToBeUploaded = m_ComponentsToBeUploaded.Count(x => x.PackageType == PackageType.Internal); + await PackageUploadHelper.UploadingThePackages(m_ComponentsToBeUploaded, appSettings.TimeOut); + //Updating the component's new location + var fileOperations = new FileOperations(); + string bomGenerationPath = Path.GetDirectoryName(appSettings.BomFilePath); + PackageUploadHelper.UpdateBomArtifactoryRepoUrl(ref m_ComponentsInBOM, m_ComponentsToBeUploaded); + fileOperations.WriteContentToFile(m_ComponentsInBOM, bomGenerationPath, FileConstant.BomFileName, appSettings.SW360ProjectName); + // write kpi info to console table PackageUploadHelper.WriteCreatorKpiDataToConsole(uploaderKpiData); if (Program.UploaderStopWatch != null) diff --git a/src/ArtifactoryUploader/Program.cs b/src/ArtifactoryUploader/Program.cs index b8d573f3..5a6b89e3 100644 --- a/src/ArtifactoryUploader/Program.cs +++ b/src/ArtifactoryUploader/Program.cs @@ -46,19 +46,20 @@ static async Task Main(string[] args) Logger.Logger.Log(null, Level.Notice, $"\n====================<<<<< Artifactory Uploader >>>>>====================", null); Logger.Logger.Log(null, Level.Notice, $"\nStart of Artifactory Uploader execution: {DateTime.Now}", null); - if (appSettings.IsTestMode) - Logger.Logger.Log(null, Level.Alert, $"Artifactory Uploader is running in TEST mode \n", null); + if (appSettings.Release) + Logger.Logger.Log(null, Level.Alert, $"Artifactory Uploader is running in release mode !!! \n", null); + else + Logger.Logger.Log(null, Level.Alert, $"Artifactory Uploader is running in dry-run mode, no packages will be moved \n", null); + Logger.Logger.Log(null, Level.Notice, $"Input Parameters used in Artifactory Uploader:\n\t" + $"BomFilePath\t\t --> {appSettings.BomFilePath}\n\t" + $"JFrogUrl\t\t --> {appSettings.JFrogApi}\n\t" + $"Artifactory User\t --> {appSettings.ArtifactoryUploadUser}\n\t" + + $"Release\t\t\t --> {appSettings.Release}\n\t" + $"LogFolderPath\t\t --> {Path.GetFullPath(FolderPath)}", null); - if (appSettings.IsTestMode) - Logger.Logger.Log(null, Level.Notice, $"\tMode\t\t\t --> {appSettings.Mode}\n", null); - //Validator method to check token validity ArtifactoryCredentials artifactoryCredentials = new ArtifactoryCredentials() { diff --git a/src/LCT.APICommunications.UTest/ArtifactoryUploaderTest.cs b/src/LCT.APICommunications.UTest/ArtifactoryUploaderTest.cs deleted file mode 100644 index 3945875d..00000000 --- a/src/LCT.APICommunications.UTest/ArtifactoryUploaderTest.cs +++ /dev/null @@ -1,55 +0,0 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT -// -------------------------------------------------------------------------------------------------------------------- - -using LCT.APICommunications.Interfaces; -using LCT.APICommunications.Model; -using Moq; -using Newtonsoft.Json; -using System.Net; - -namespace LCT.APICommunications.UTest -{ - public class ArtifactoryUploader - { - [SetUp] - public void Setup() - { - // Method intentionally left empty. - } - - - [Test] - public void UploadPackageToRepo_InputEmptyCreds_ReturnsInvalidOperationException() - { - //Arrange - ReleasesDetails releasesDetails = new ReleasesDetails(); - var sw360ApiCommunication = new Mock(); - ComponentsToArtifactory componentsToArtifactory = new ComponentsToArtifactory(); - sw360ApiCommunication.Setup(x => x.GetReleaseById(It.IsAny())).ReturnsAsync(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(JsonConvert.SerializeObject(releasesDetails)) - }); - - ///Act & Assert - - Assert.ThrowsAsync(async () => await ArtfactoryUploader.UploadPackageToRepo(componentsToArtifactory,100)); - } - - [Test] - public void SetConfigurationValues_InputEmptyCreds_ReturnsVoid() - { - //Arrange - bool returnValue = true; - - //Act - ArtfactoryUploader.SetConfigurationValues(); - - //Assert - Assert.That(returnValue, Is.True); - } - } -} \ No newline at end of file diff --git a/src/LCT.APICommunications/ApiConstant.cs b/src/LCT.APICommunications/ApiConstant.cs index c957e1b3..7582439d 100644 --- a/src/LCT.APICommunications/ApiConstant.cs +++ b/src/LCT.APICommunications/ApiConstant.cs @@ -36,6 +36,7 @@ public static class ApiConstant public const string PythonExtension = ".whl"; public const string PackageInfoApi = "/api/storage/"; public const string CopyPackageApi = "/api/copy/"; + public const string MovePackageApi = "/api/move/"; public const string Releases = "releases"; public const string Release = "release"; public const string FolderId = "folderId"; diff --git a/src/LCT.APICommunications/ArtifactoryUploader.cs b/src/LCT.APICommunications/ArtifactoryUploader.cs deleted file mode 100644 index b19e2855..00000000 --- a/src/LCT.APICommunications/ArtifactoryUploader.cs +++ /dev/null @@ -1,104 +0,0 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT -// -------------------------------------------------------------------------------------------------------------------- - -using LCT.APICommunications.Interfaces; -using LCT.APICommunications.Model; -using log4net; -using System; -using System.Configuration; -using System.Net; -using System.Net.Http; -using System.Reflection; -using System.Threading.Tasks; - -namespace LCT.APICommunications -{ - public static class ArtfactoryUploader - { - //ConfigurationAttribute - private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private static string destRepoName = Environment.GetEnvironmentVariable("JfrogDestRepoName"); - private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi"); - private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo"); - - public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout) - { - Logger.Debug("Starting UploadPackageToArtifactory method"); - IJFrogApiCommunication jfrogApicommunication; - HttpResponseMessage responsemessage = new HttpResponseMessage(); - HttpResponseMessage responseBodyJfrog = new HttpResponseMessage(); - try - { - ArtifactoryCredentials repoCredentials = new ArtifactoryCredentials() - { - ApiKey = component.ApiKey, - Email = component.Email - }; - if (component.ComponentType?.ToUpperInvariant() == "MAVEN") - { - jfrogApicommunication = new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); - responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); - } - else if (component.ComponentType?.ToUpperInvariant() == "PYTHON") - { - jfrogApicommunication = new PythonJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); - responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); - } - else - { - jfrogApicommunication = new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); - responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); - } - if (responseBodyJfrog.StatusCode == HttpStatusCode.NotFound) - { - component.PackageInfoApiUrl = component.PackageInfoApiUrl.ToLower(); - responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); - component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower(); - } - if (responseBodyJfrog.StatusCode != HttpStatusCode.OK) - { - responsemessage.StatusCode = responseBodyJfrog.StatusCode; - responsemessage.ReasonPhrase = ApiConstant.PackageNotFound; - return responsemessage; - } - - responsemessage = await jfrogApicommunication.CopyFromRemoteRepo(component); - if (responsemessage.StatusCode != HttpStatusCode.OK) - { - responsemessage.StatusCode = responseBodyJfrog.StatusCode; - responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload; - return responsemessage; - } - Logger.Info($"Successfully copied package {component.PackageName}-{component.Version} from {component.SrcRepoName} to {component.DestRepoName}"); - } - catch (HttpRequestException ex) - { - Logger.Error($"Error has occured in UploadPackageToArtifactory--{ex}"); - return responsemessage; - } - Logger.Debug("Ending UploadPackageToArtifactory method"); - return responsemessage; - } - - /// - public static void SetConfigurationValues() - { - if (string.IsNullOrEmpty(destRepoName)) - { - destRepoName = ConfigurationManager.AppSettings["JfrogDestRepoName"]; - } - if (string.IsNullOrEmpty(JfrogApi)) - { - JfrogApi = ConfigurationManager.AppSettings["JfrogApi"]; - } - if (string.IsNullOrEmpty(srcRepoName)) - { - srcRepoName = ConfigurationManager.AppSettings["JfrogSrcRepo"]; - } - } - - } -} diff --git a/src/LCT.APICommunications/Interfaces/IJFrogApiCommunication.cs b/src/LCT.APICommunications/Interfaces/IJFrogApiCommunication.cs index bb53886a..bcd22ad8 100644 --- a/src/LCT.APICommunications/Interfaces/IJFrogApiCommunication.cs +++ b/src/LCT.APICommunications/Interfaces/IJFrogApiCommunication.cs @@ -18,6 +18,8 @@ public interface IJFrogApiCommunication Task CopyFromRemoteRepo(ComponentsToArtifactory component); + Task MoveFromRepo(ComponentsToArtifactory component); + Task GetApiKey(); void UpdatePackagePropertiesInJfrog(string sw360releaseUrl, string destRepoName, UploadArgs uploadArgs); diff --git a/src/LCT.APICommunications/Interfaces/IJfrogAqlApiCommunication.cs b/src/LCT.APICommunications/Interfaces/IJfrogAqlApiCommunication.cs index afa5eb7d..8bd4756b 100644 --- a/src/LCT.APICommunications/Interfaces/IJfrogAqlApiCommunication.cs +++ b/src/LCT.APICommunications/Interfaces/IJfrogAqlApiCommunication.cs @@ -20,5 +20,14 @@ public interface IJfrogAqlApiCommunication /// repoName /// HttpResponseMessage Task GetInternalComponentDataByRepo(string repoName); + + /// + /// Gets the package information in the repo, via the name or path + /// + /// repoName + /// repoName + /// repoName + /// AqlResult + Task GetPackageInfo(string repoName, string packageName = null, string path = null); } } diff --git a/src/LCT.APICommunications/JfrogApicommunication.cs b/src/LCT.APICommunications/JfrogApicommunication.cs index de9175f2..bef74fac 100644 --- a/src/LCT.APICommunications/JfrogApicommunication.cs +++ b/src/LCT.APICommunications/JfrogApicommunication.cs @@ -52,6 +52,8 @@ public async Task DeletePackageFromJFrogRepo(string repoNam public abstract Task CopyFromRemoteRepo(ComponentsToArtifactory component); + public abstract Task MoveFromRepo(ComponentsToArtifactory component); + public abstract void UpdatePackagePropertiesInJfrog(string sw360releaseUrl, string destRepoName, UploadArgs uploadArgs); } diff --git a/src/LCT.APICommunications/JfrogAqlApiCommunication.cs b/src/LCT.APICommunications/JfrogAqlApiCommunication.cs index 9a2d9898..3a6a014e 100644 --- a/src/LCT.APICommunications/JfrogAqlApiCommunication.cs +++ b/src/LCT.APICommunications/JfrogAqlApiCommunication.cs @@ -7,6 +7,7 @@ using LCT.APICommunications.Interfaces; using LCT.APICommunications.Model; using System; +using System.Collections.Generic; using System.Net.Http; using System.Text; using System.Threading.Tasks; @@ -57,6 +58,25 @@ public async Task GetInternalComponentDataByRepo(string rep return await httpClient.PostAsync(uri, httpContent); } + /// + /// Gets the package information in the repo, via the name or path + /// + /// repoName + /// repoName + /// repoName + /// AqlResult + public async Task GetPackageInfo(string repoName, string packageName = null, string path = null) + { + ValidateParameters(packageName, path); + + var aqlQueryToBody = BuildAqlQuery(repoName, packageName, path); + + string uri = $"{DomainName}{ApiConstant.JfrogArtifactoryApiSearchAql}"; + HttpContent httpContent = new StringContent(aqlQueryToBody); + + return await ExecuteSearchAqlAsync(uri, httpContent); + } + private static HttpClient GetHttpClient(ArtifactoryCredentials credentials) { HttpClient httpClient = new HttpClient(); @@ -65,5 +85,45 @@ private static HttpClient GetHttpClient(ArtifactoryCredentials credentials) httpClient.DefaultRequestHeaders.Add(ApiConstant.Email, credentials.Email); return httpClient; } + + private static void ValidateParameters(string packageName, string path) + { + if (string.IsNullOrEmpty(packageName) && string.IsNullOrEmpty(path)) + { + throw new ArgumentException("Either packageName or path, or both must be provided."); + } + } + + private static string BuildAqlQuery(string repoName, string packageName, string path) + { + var queryList = new List() + { + $"\"repo\":{{\"$eq\":\"{repoName}\"}}" + }; + + if (!string.IsNullOrEmpty(path)) + { + queryList.Add($"\"path\":{{\"$match\":\"{path}\"}}"); + } + + if (!string.IsNullOrEmpty(packageName)) + { + queryList.Add($"\"name\":{{\"$match\":\"{packageName}\"}}"); + } + + StringBuilder query = new(); + query.Append($"items.find({{{string.Join(", ", queryList)}}}).include(\"repo\", \"path\", \"name\").limit(1)"); + + return query.ToString(); + } + + private async Task ExecuteSearchAqlAsync(string uri, HttpContent httpContent) + { + HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); + TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec); + httpClient.Timeout = timeOutInSec; + + return await httpClient.PostAsync(uri, httpContent); + } } } diff --git a/src/LCT.APICommunications/MavenJfrogApiCommunication.cs b/src/LCT.APICommunications/MavenJfrogApiCommunication.cs index ba10a8ac..a107fdb3 100644 --- a/src/LCT.APICommunications/MavenJfrogApiCommunication.cs +++ b/src/LCT.APICommunications/MavenJfrogApiCommunication.cs @@ -42,6 +42,13 @@ public override async Task CopyFromRemoteRepo(ComponentsToA return await httpClient.PostAsync(component.CopyPackageApiUrl, httpContent); } + public override async Task MoveFromRepo(ComponentsToArtifactory component) + { + HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); + const HttpContent httpContent = null; + return await httpClient.PostAsync(component.MovePackageApiUrl, httpContent); + } + public override async Task GetPackageInfo(ComponentsToArtifactory component) { HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); diff --git a/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs b/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs index 39a234cd..5a2ed140 100644 --- a/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs +++ b/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs @@ -23,7 +23,12 @@ public class ComponentsToArtifactory public string Email { get; set; } public string PackageInfoApiUrl { get; set; } public string CopyPackageApiUrl { get; set; } + public string MovePackageApiUrl { get; set; } public string PackageExtension { get; set; } public string Path { get; set; } + public PackageType PackageType { get; set; } + public bool DryRun { get; set; } = true; + public string Purl { get; set; } + public string JfrogPackageName { get;set; } } } diff --git a/src/LCT.APICommunications/Model/PackageType.cs b/src/LCT.APICommunications/Model/PackageType.cs new file mode 100644 index 00000000..a64e7871 --- /dev/null +++ b/src/LCT.APICommunications/Model/PackageType.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LCT.APICommunications.Model +{ + public enum PackageType + { + ClearedThirdParty, + Internal, + Development, + Unknown + } +} diff --git a/src/LCT.APICommunications/NpmJfrogAPICommunication.cs b/src/LCT.APICommunications/NpmJfrogAPICommunication.cs index 8c2f6386..55d98e4b 100644 --- a/src/LCT.APICommunications/NpmJfrogAPICommunication.cs +++ b/src/LCT.APICommunications/NpmJfrogAPICommunication.cs @@ -46,6 +46,13 @@ public override async Task CopyFromRemoteRepo(ComponentsToA return await httpClient.PostAsync(component.CopyPackageApiUrl, httpContent); } + public override async Task MoveFromRepo(ComponentsToArtifactory component) + { + HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); + const HttpContent httpContent = null; + return await httpClient.PostAsync(component.MovePackageApiUrl, httpContent); + } + public override async Task GetPackageInfo(ComponentsToArtifactory component) { HttpResponseMessage responseMessage = new HttpResponseMessage(); diff --git a/src/LCT.APICommunications/NugetJfrogAPICommunication.cs b/src/LCT.APICommunications/NugetJfrogAPICommunication.cs index 2d3624b8..0fa30454 100644 --- a/src/LCT.APICommunications/NugetJfrogAPICommunication.cs +++ b/src/LCT.APICommunications/NugetJfrogAPICommunication.cs @@ -46,6 +46,13 @@ public override async Task CopyFromRemoteRepo(ComponentsToA return await httpClient.PostAsync(component.CopyPackageApiUrl, httpContent); } + public override async Task MoveFromRepo(ComponentsToArtifactory component) + { + HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); + const HttpContent httpContent = null; + return await httpClient.PostAsync(component.MovePackageApiUrl, httpContent); + } + public override async Task GetPackageInfo(ComponentsToArtifactory component) { HttpResponseMessage responseMessage = new HttpResponseMessage(); diff --git a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs index 47e4b25b..374c88a0 100644 --- a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs +++ b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs @@ -40,6 +40,14 @@ public override async Task CopyFromRemoteRepo(ComponentsToA const HttpContent httpContent = null; return await httpClient.PostAsync(component.CopyPackageApiUrl, httpContent); } + + public override async Task MoveFromRepo(ComponentsToArtifactory component) + { + HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); + const HttpContent httpContent = null; + return await httpClient.PostAsync(component.MovePackageApiUrl, httpContent); + } + public override async Task GetPackageInfo(ComponentsToArtifactory component) { HttpClient httpClient = GetHttpClient(ArtifactoryCredentials); diff --git a/src/LCT.Common/CommonAppSettings.cs b/src/LCT.Common/CommonAppSettings.cs index 6bd79eed..e5592e07 100644 --- a/src/LCT.Common/CommonAppSettings.cs +++ b/src/LCT.Common/CommonAppSettings.cs @@ -68,15 +68,9 @@ public CommonAppSettings(IFolderAction iFolderAction) public Config Conan { get; set; } public string CaVersion { get; set; } public string CycloneDxSBomTemplatePath { get; set; } - public string[] InternalRepoList { get; set; } + public string[] InternalRepoList { get; set; } = Array.Empty(); public bool EnableFossTrigger { get; set; } = true; - public string JfrogNpmDestRepoName { get; set; } public string JfrogNpmSrcRepo { get; set; } - public string JfrogNugetDestRepoName { get; set; } - public string JfrogMavenDestRepoName { get; set; } - public string JfrogPythonDestRepoName { get; set; } - public string JfrogConanDestRepoName { get; set; } - public string JfrogNugetSrcRepo { get; set; } public string Mode { get; set; } = string.Empty; @@ -307,5 +301,7 @@ public string ArtifactoryUploadUser } + public bool Release { get; set; } = false; + } } diff --git a/src/LCT.Common/Constants/Dataconstant.cs b/src/LCT.Common/Constants/Dataconstant.cs index 0b30c82d..f446de9b 100644 --- a/src/LCT.Common/Constants/Dataconstant.cs +++ b/src/LCT.Common/Constants/Dataconstant.cs @@ -48,7 +48,7 @@ public static class Dataconstant public const string ReleaseAttachmentComment = "Attached by CA Tool"; public const char ForwardSlash = '/'; public const string SourceURLSuffix = "/srcfiles?fileinfo=1"; - public const string Cdx_ArtifactoryRepoUrl = "internal:siemens:clearing:repo-url"; + public const string Cdx_ArtifactoryRepoUrl = "internal:siemens:clearing:repo-name"; public const string Cdx_ProjectType = "internal:siemens:clearing:project-type"; public const string Cdx_ClearingState = "internal:siemens:clearing:clearing-state"; public const string Cdx_IsInternal = "internal:siemens:clearing:is-internal"; diff --git a/src/LCT.Common/Model/Config.cs b/src/LCT.Common/Model/Config.cs index 6bc38c08..3b03b51e 100644 --- a/src/LCT.Common/Model/Config.cs +++ b/src/LCT.Common/Model/Config.cs @@ -23,6 +23,9 @@ public class Config public string[] JfrogMavenRepoList { get; set; } public string[] JfrogPythonRepoList { get; set; } public string[] JfrogConanRepoList { get; set; } + public string JfrogThirdPartyDestRepoName { get; set; } + public string JfrogInternalDestRepoName { get; set; } + public string JfrogDevDestRepoName { get; set; } public string[] DevDependentScopeList { get; set; } } diff --git a/src/LCT.Common/appSettings.json b/src/LCT.Common/appSettings.json index 4500918a..2d1c90bd 100644 --- a/src/LCT.Common/appSettings.json +++ b/src/LCT.Common/appSettings.json @@ -5,85 +5,101 @@ // -------------------------------------------------------------------------------------------------------------------- { - "CaVersion": "5.0.0", - "TimeOut": 200, - "ProjectType": "", - "SW360ProjectName": "", - "SW360ProjectID": "", - "Sw360AuthTokenType": "Bearer", - "Sw360Token": "", - "SW360URL": "", - "Fossologyurl": "", - "JFrogApi": "", - "JfrogNugetDestRepoName": "", - "JfrogNpmDestRepoName": "", - "JfrogMavenDestRepoName": "", - "JfrogPythonDestRepoName": "", - "JfrogConanDestRepoName": "", - "PackageFilePath": "/PathToInputDirectory", //For Docker run set as /mnt/Input - "BomFolderPath": "/PathToOutputDirectory", //For Docker run set as /mnt/Output - "BomFilePath": "/PathToOutputDirectory/_Bom.cdx.json", - //IdentifierBomFilePath : For multiple project type - "IdentifierBomFilePath": "", - //CycloneDxBomFilePath: For Providing Customer maintained SBOM as input.Can be used along with Packagefilepath or individually - "CycloneDxSBomTemplatePath": "", - "ArtifactoryUploadApiKey": "", //This should be Jfrog Key - "ArtifactoryUploadUser": "", //This should be Jfrog user name - "RemoveDevDependency": true, - "EnableFossTrigger": true, - "InternalRepoList": [ - "", //This should be the internal repo names in JFrog for NPM - "" //This should be the internal repo names in JFrog for Nuget - ], - "Npm": { - "Include": [ "p*-lock.json", "*.cdx.json" ], - "Exclude": [ "node_modules" ], - "JfrogNpmRepoList": [ - "", //This is a mirror repo for npm registry in JFrog - "" //This should be the release repo in JFrog + "CaVersion": "6.0.0", + "TimeOut": 200, + "ProjectType": "", + "SW360ProjectName": "", + "SW360ProjectID": "", + "Sw360AuthTokenType": "Bearer", + "Sw360Token": "", + "SW360URL": "", + "Fossologyurl": "", + "JFrogApi": "", + "PackageFilePath": "/PathToInputDirectory", //For Docker run set as /mnt/Input + "BomFolderPath": "/PathToOutputDirectory", //For Docker run set as /mnt/Output + "BomFilePath": "/PathToOutputDirectory/_Bom.cdx.json", + //IdentifierBomFilePath : For multiple project type + "IdentifierBomFilePath": "", + //CycloneDxBomFilePath: For Providing Customer maintained SBOM as input.Can be used along with Packagefilepath or individually + "CycloneDxSBomTemplatePath": "", + "ArtifactoryUploadApiKey": "", //This should be Jfrog Key + "ArtifactoryUploadUser": "", //This should be Jfrog user name + "Release": false, + "RemoveDevDependency": true, + "EnableFossTrigger": true, + "InternalRepoList": [ + "", //This should be the internal repo names in JFrog for NPM + "" //This should be the internal repo names in JFrog for Nuget ], - "ExcludedComponents": [] - }, - "Nuget": { - "Include": [ "pack*.config", "p*.assets.json", "*.cdx.json" ], - "Exclude": [], - "JfrogNugetRepoList": [ - "", //This is a mirror repo for nuget.org in JFrog - "" //This should be the release repo in JFrog - ], - "ExcludedComponents": [] - }, - "Maven": { - "Include": [ "*.cdx.json" ], - "Exclude": [], - "JfrogMavenRepoList": [ - "", //This is a mirror repo for repo.maven in JFrog - "" //This should be the release repo.maven in JFrog - ], - "DevDependentScopeList": [ "test" ], - "ExcludedComponents": [] - }, - "Debian": { - "Include": [ "*.cdx.json" ], - "Exclude": [], - "ExcludedComponents": [] - }, - "Python": { - "Include": [ "poetry.lock", "*.cdx.json" ], - "Exclude": [], - "JfrogPythonRepoList": [ - "", //This is a mirror repo for pypi in JFrog - "" //This should be the release pypi in JFrog - ], - "ExcludedComponents": [] - }, - "Conan": { - "Include": [ "conan.lock" ], - "Exclude": [], - "JfrogConanRepoList": [ - "", //This is a mirror repo for conan in JFrog - "" //This should be the release repo in JFrog - ], - "ExcludedComponents": [] - } + "Npm": { + "Include": [ "p*-lock.json", "*.cdx.json" ], + "Exclude": [ "node_modules" ], + "JfrogNpmRepoList": [ + "", //This is a mirror repo for npm registry in JFrog + "", //This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog + ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", + "ExcludedComponents": [] + }, + "Nuget": { + "Include": [ "pack*.config", "p*.assets.json", "*.cdx.json" ], + "Exclude": [], + "JfrogNugetRepoList": [ + "", //This is a mirror repo for nuget.org in JFrog + "", //This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog + ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", + "ExcludedComponents": [] + }, + "Maven": { + "Include": [ "*.cdx.json" ], + "Exclude": [], + "JfrogMavenRepoList": [ + "", //This is a mirror repo for repo.maven in JFrog + "", //This should be the release repo.maven in JFrog + "" //This should be the development dependency repo in JFrog + ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", + "DevDependentScopeList": [ "test" ], + "ExcludedComponents": [] + }, + "Debian": { + "Include": [ "*.cdx.json" ], + "Exclude": [], + "ExcludedComponents": [] + }, + "Python": { + "Include": [ "poetry.lock", "*.cdx.json" ], + "Exclude": [], + "JfrogPythonRepoList": [ + "", //This is a mirror repo for pypi in JFrog + "", //This should be the release pypi in JFrog + "" //This should be the development dependency repo in JFrog + ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", + "ExcludedComponents": [] + }, + "Conan": { + "Include": [ "conan.lock" ], + "Exclude": [], + "JfrogConanRepoList": [ + "", //This is a mirror repo for conan in JFrog + "", //This should be the release repo in JFrog + "" //This should be the development dependency repo in JFrog + ], + "JfrogThirdPartyDestRepoName": "", + "JfrogInternalDestRepoName": "", + "JfrogDevDestRepoName": "", + "ExcludedComponents": [] + } } diff --git a/src/LCT.Facade.UTest/JfrogAqlApiCommunicationFacadeUTest.cs b/src/LCT.Facade.UTest/JfrogAqlApiCommunicationFacadeUTest.cs index b8eba981..88a91248 100644 --- a/src/LCT.Facade.UTest/JfrogAqlApiCommunicationFacadeUTest.cs +++ b/src/LCT.Facade.UTest/JfrogAqlApiCommunicationFacadeUTest.cs @@ -5,8 +5,10 @@ // -------------------------------------------------------------------------------------------------------------------- using LCT.APICommunications.Interfaces; +using LCT.APICommunications.Model; using Moq; using NUnit.Framework; +using System; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -63,5 +65,27 @@ public async Task GetInternalComponentDataByRepo_Returnswith_HttpCodeNoContent() //Assert Assert.That(actual.StatusCode, Is.EqualTo(HttpStatusCode.NoContent)); } + + [Test] + public async Task GetPackageInfoByRepoAndPackageName_GetsPackageInfo_Successfully() + { + // Arrange + HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.OK); + Mock mockJfrogAqlApiCommunicationFacade = + new Mock(); + mockJfrogAqlApiCommunicationFacade. + Setup(x => x.GetPackageInfo(It.IsAny(), It.IsAny(), It.IsAny())). + ReturnsAsync(httpResponseMessage); + + // Act + JfrogAqlApiCommunicationFacade jfrogAqlApiCommunicationFacade = + new JfrogAqlApiCommunicationFacade(mockJfrogAqlApiCommunicationFacade.Object); + HttpResponseMessage actual = + await jfrogAqlApiCommunicationFacade.GetPackageInfo("org1-nuget-nuget-remote-cache", "System.Numerics.Vectors.4.5.0.nupkg", + string.Empty); + + //Assert + Assert.That(actual.StatusCode, Is.EqualTo(HttpStatusCode.OK)); + } } } diff --git a/src/LCT.Facade/Interfaces/IJfrogAqlApiCommunicationFacade.cs b/src/LCT.Facade/Interfaces/IJfrogAqlApiCommunicationFacade.cs index f88b75c8..f24c8ba2 100644 --- a/src/LCT.Facade/Interfaces/IJfrogAqlApiCommunicationFacade.cs +++ b/src/LCT.Facade/Interfaces/IJfrogAqlApiCommunicationFacade.cs @@ -20,5 +20,14 @@ public interface IJfrogAqlApiCommunicationFacade /// repoName /// HttpResponseMessage Task GetInternalComponentDataByRepo(string repoName); + + /// + /// Gets the package information in the repo, via the name or path + /// + /// repoName + /// repoName + /// repoName + /// AqlResult + Task GetPackageInfo(string repoName, string packageName, string path); } } diff --git a/src/LCT.Facade/JfrogAqlApiCommunicationFacade.cs b/src/LCT.Facade/JfrogAqlApiCommunicationFacade.cs index 10d727af..4c5c6312 100644 --- a/src/LCT.Facade/JfrogAqlApiCommunicationFacade.cs +++ b/src/LCT.Facade/JfrogAqlApiCommunicationFacade.cs @@ -36,5 +36,17 @@ public async Task GetInternalComponentDataByRepo(string rep { return await m_jfrogAqlApiCommunication.GetInternalComponentDataByRepo(repoName); } + + /// + /// Gets the package information in the repo, via the name or path + /// + /// repoName + /// repoName + /// repoName + /// AqlResult + public async Task GetPackageInfo(string repoName, string packageName, string path) + { + return await m_jfrogAqlApiCommunication.GetPackageInfo(repoName, packageName, path); + } } } diff --git a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs index 295755ef..c4390a94 100644 --- a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs @@ -198,7 +198,7 @@ public async Task GetJfrogRepoDetailsOfAComponent_ReturnsWithData_SuccessFully() ConanProcessor conanProcessor = new ConanProcessor(); var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; // Assert Assert.That(actual, Is.Not.Null); @@ -239,7 +239,7 @@ public async Task GetArtifactoryRepoName_Conan_ReturnsNotFound_ReturnsFailure() var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; Assert.That("Not Found in JFrogRepo", Is.EqualTo(reponameActual)); } diff --git a/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs b/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs index e3161baf..4fb96b29 100644 --- a/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs @@ -421,7 +421,7 @@ public async Task GetArtifactoryRepoName_Nuget_ReturnsRepoName_SuccessFully() var actual = await nugetProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; Assert.That(reponameActual, Is.EqualTo(aqlResult.Repo)); } @@ -460,7 +460,7 @@ public async Task GetArtifactoryRepoName_Nuget_ReturnsRepoName_ReturnsFailure() var actual = await nugetProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; Assert.That("Not Found in JFrogRepo", Is.EqualTo(reponameActual)); } @@ -499,7 +499,7 @@ public async Task GetArtifactoryRepoName_Nuget_ReturnsRepoName_ReturnsSuccess() var actual = await nugetProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; Assert.That("internalrepo1", Is.EqualTo(reponameActual)); @@ -539,7 +539,7 @@ public async Task GetArtifactoryRepoName_Nuget_ReturnsNotFound_ReturnsFailure() var actual = await nugetProcessor.GetJfrogRepoDetailsOfAComponent( components, appSettings, mockJfrogService.Object, mockBomHelper.Object); - var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-name").Properties[0].Value; Assert.That("Not Found in JFrogRepo", Is.EqualTo(reponameActual)); } diff --git a/src/LCT.PackageIdentifier.UTest/PythonParserTests.cs b/src/LCT.PackageIdentifier.UTest/PythonParserTests.cs index 0650cfbf..29e23a16 100644 --- a/src/LCT.PackageIdentifier.UTest/PythonParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/PythonParserTests.cs @@ -269,7 +269,7 @@ public async Task GetJfrogRepoDetailsOfAComponentForPython_ReturnsWithData_Succe var components = new List() { component1 }; string[] reooListArr = { "internalrepo1", "internalrepo2" }; CommonAppSettings appSettings = new(); - appSettings.Python = new Config() { JfrogNugetRepoList = reooListArr }; + appSettings.Python = new Config() { JfrogPythonRepoList = reooListArr }; AqlResult aqlResult = new() { Name = "html5lib-1.1.tar.gz", @@ -307,7 +307,7 @@ public async Task GetJfrogRepoDetailsOfAComponentForPython_ReturnsWithData2_Succ var components = new List() { component1 }; string[] reooListArr = { "internalrepo1", "internalrepo2" }; CommonAppSettings appSettings = new(); - appSettings.Python = new Config() { JfrogNugetRepoList = reooListArr }; + appSettings.Python = new Config() { JfrogPythonRepoList = reooListArr }; AqlResult aqlResult = new() { Name = "html5lib-1.1-py2.py3-none-any.whl", diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index 02f6b873..38953c66 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -123,8 +123,9 @@ public async Task IdentificationOfInternalComponents(Co public async Task> GetJfrogRepoDetailsOfAComponent(List componentsForBOM, CommonAppSettings appSettings, IJFrogService jFrogService, IBomHelper bomhelper) { - // get the component list from Jfrog for given repository - List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Conan?.JfrogConanRepoList, jFrogService); + // get the component list from Jfrog for given repo + internal repo + string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Conan?.JfrogConanRepoList).ToArray(); + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService); Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; List modifiedBOM = new List(); diff --git a/src/LCT.PackageIdentifier/LCT.PackageIdentifier.csproj b/src/LCT.PackageIdentifier/LCT.PackageIdentifier.csproj index fe4259d4..4b6cf7c7 100644 --- a/src/LCT.PackageIdentifier/LCT.PackageIdentifier.csproj +++ b/src/LCT.PackageIdentifier/LCT.PackageIdentifier.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 2ba4d133..c6d246a1 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -167,8 +167,9 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Maven?.JfrogMavenRepoList, jFrogService); + // get the component list from Jfrog for given repo + internal repo + string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Maven?.JfrogMavenRepoList).ToArray(); + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService); Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; List modifiedBOM = new List(); diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index f2e3273c..9609a113 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -301,8 +301,9 @@ public async Task IdentificationOfInternalComponents( public async Task> GetJfrogRepoDetailsOfAComponent(List componentsForBOM, CommonAppSettings appSettings, IJFrogService jFrogService, IBomHelper bomhelper) { - // get the component list from Jfrog for given repo - List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Npm?.JfrogNpmRepoList, jFrogService); + // get the component list from Jfrog for given repo + internal repo + string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Npm?.JfrogNpmRepoList).ToArray(); + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService); Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; List modifiedBOM = new List(); diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 2221b406..600aaf65 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -224,8 +224,9 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Nuget?.JfrogNugetRepoList, jFrogService); + // get the component list from Jfrog for given repo + internal repo + string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Nuget?.JfrogNugetRepoList).ToArray(); + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService); Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; List modifiedBOM = new List(); diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index 85500a85..cd826359 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -345,8 +345,9 @@ private static bool IsInternalPythonComponent(List aqlResultList, Com public async Task> GetJfrogRepoDetailsOfAComponent(List componentsForBOM, CommonAppSettings appSettings, IJFrogService jFrogService, IBomHelper bomhelper) { - // get the component list from Jfrog for given repo - List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Python?.JfrogPythonRepoList, jFrogService); + // get the component list from Jfrog for given repo + internal repo + string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Python?.JfrogPythonRepoList).ToArray(); + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService); Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; List modifiedBOM = new List(); diff --git a/src/LCT.SW360PackageCreator/ConanPackageDownloader.cs b/src/LCT.SW360PackageCreator/ConanPackageDownloader.cs deleted file mode 100644 index f6d7e5b3..00000000 --- a/src/LCT.SW360PackageCreator/ConanPackageDownloader.cs +++ /dev/null @@ -1,28 +0,0 @@ -using LCT.Common.Model; -using LCT.SW360PackageCreator.Interfaces; -using LCT.SW360PackageCreator.Model; -using log4net; -using System.Collections.Generic; -using System.Reflection; -using System.Threading.Tasks; - -namespace LCT.SW360PackageCreator -{ - - public class ConanPackageDownloader: IPackageDownloader - { - static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private readonly List m_downloadedSourceInfos = new List(); - private const string Source = "source"; - public async Task DownloadPackage(ComparisonBomData component, string localPathforDownload) - { - string path = Download(component, localPathforDownload); - await Task.Delay(10); - return path; - } - private string Download(ComparisonBomData component, string downloadPath) - { - return ""; - } - } -} diff --git a/src/LCT.SW360PackageCreator/CreatorHelper.cs b/src/LCT.SW360PackageCreator/CreatorHelper.cs index 3e91e5b4..82968e09 100644 --- a/src/LCT.SW360PackageCreator/CreatorHelper.cs +++ b/src/LCT.SW360PackageCreator/CreatorHelper.cs @@ -228,11 +228,8 @@ public async Task> SetContentsForComparisonBOM(List> Patches { get; set; } } + [ExcludeFromCodeCoverage] public class Source { [YamlMember(Alias = "url")] @@ -22,6 +24,7 @@ public class Source } + [ExcludeFromCodeCoverage] public class Patch { public string PatchFile { get; set; } diff --git a/src/LCT.SW360PackageCreator/Program.cs b/src/LCT.SW360PackageCreator/Program.cs index 0a29c66c..81fe28be 100644 --- a/src/LCT.SW360PackageCreator/Program.cs +++ b/src/LCT.SW360PackageCreator/Program.cs @@ -107,8 +107,7 @@ private static async Task InitiatePackageCreatorProcess(CommonAppSettings appSet { { "NPM", new PackageDownloader() }, { "NUGET", new PackageDownloader() }, - { "DEBIAN", new DebianPackageDownloader(debianPatcher) }, - { "CONAN", new ConanPackageDownloader() } + { "DEBIAN", new DebianPackageDownloader(debianPatcher) } }; ICreatorHelper creatorHelper = new CreatorHelper(_packageDownloderList); diff --git a/src/LCT.SW360PackageCreator/URLHelper.cs b/src/LCT.SW360PackageCreator/URLHelper.cs index 4d7d67ec..f2524640 100644 --- a/src/LCT.SW360PackageCreator/URLHelper.cs +++ b/src/LCT.SW360PackageCreator/URLHelper.cs @@ -32,6 +32,7 @@ using Microsoft.Web.Administration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using YamlDotNet.Core; using YamlDotNet.Core.Tokens; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -47,7 +48,7 @@ public class UrlHelper : IUrlHelper, IDisposable private readonly HttpClient httpClient = new HttpClient(); public static string GithubUrl { get; set; } = string.Empty; public static UrlHelper Instance { get; } = new UrlHelper(); - public CommonAppSettings CommonAppSettings =new CommonAppSettings(); + public CommonAppSettings CommonAppSettings { get; } = new CommonAppSettings(); private bool _disposed; @@ -162,7 +163,7 @@ public string GetSourceUrlForNpmPackage(string componentName, string version) /// /// /// string - public async Task GetSourceUrlForConanPackage(string componentName, string version) + public async Task GetSourceUrlForConanPackage(string componentName, string componenVersion) { var downLoadUrl = $"{CommonAppSettings.SourceURLConan}" + componentName + "/all/conandata.yml"; @@ -178,7 +179,7 @@ public async Task GetSourceUrlForConanPackage(string componentName, stri response.EnsureSuccessStatusCode(); var jsonObject = await response.Content.ReadAsStringAsync(); packageSourcesInfo = deserializer.Deserialize(jsonObject); - if (packageSourcesInfo.SourcesData.TryGetValue(version, out var release)) + if (packageSourcesInfo.SourcesData.TryGetValue(componenVersion, out var release)) { if (release.Url.GetType().Name.ToLowerInvariant() == "string") { @@ -187,26 +188,26 @@ public async Task GetSourceUrlForConanPackage(string componentName, stri else { List urlList = (List)release.Url; - componentSrcURL = urlList.FirstOrDefault() != null ? urlList.FirstOrDefault().ToString() : ""; + componentSrcURL = urlList.FirstOrDefault()?.ToString() ?? ""; } } } - catch (Exception) + catch (HttpRequestException ex) { - - var response = new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent(string.Format("Problem Getting Information from Conan Server For = {0}", componentName)), - ReasonPhrase = "Problem Occured while connecting to conan Server" - }; + Logger.Warn($"Identification of SRC url failed for {componentName}, " + + $"Exclude if it is an internal component or manually update the SRC url"); + Logger.Debug($"GetSourceUrlForConanPackage()", ex); + } + catch(YamlException ex) + { + Logger.Warn($"Identification of SRC url failed for {componentName}, " + + $"Exclude if it is an internal component or manually update the SRC url"); + Logger.Debug($"GetSourceUrlForConanPackage()", ex); } - finally - { - _httpClient.Dispose(); + catch (ArgumentNullException ex) + { + Logger.Debug($"GetSourceUrlForConanPackage()", ex); } - - - } return componentSrcURL; } diff --git a/src/LCT.Services.UTest/JFrogServiceUTest.cs b/src/LCT.Services.UTest/JFrogServiceUTest.cs index 864fdcee..44428213 100644 --- a/src/LCT.Services.UTest/JFrogServiceUTest.cs +++ b/src/LCT.Services.UTest/JFrogServiceUTest.cs @@ -153,5 +153,82 @@ public async Task GetInternalComponentDataByRepo_ResultsWith_TaskCanceledExcepti // Assert Assert.That(actual.Count, Is.EqualTo(0)); } + + [Test] + public async Task GetPackageInfo_GetsPackageInfo_Successfully() + { + // Arrange + + AqlResult aqlResult = new AqlResult() + { + Name = "saap-api-node-2.26.3-LicenseClearing.16.sha-058fada.tgz", + Path = "@testfolder/-/folder", + Repo = "energy-dev-npm-egll" + }; + + IList results = new List(); + results.Add(aqlResult); + + AqlResponse aqlResponse = new AqlResponse(); + aqlResponse.Results = results; + + var aqlResponseSerialized = JsonConvert.SerializeObject(aqlResponse); + var content = new StringContent(aqlResponseSerialized, Encoding.UTF8, "application/json"); + HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.OK); + httpResponseMessage.Content = content; + + + Mock mockJfrogApiComFacade = + new Mock(); + mockJfrogApiComFacade + .Setup(x => x.GetPackageInfo(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(httpResponseMessage); + + // Act + IJFrogService jFrogService = new JFrogService(mockJfrogApiComFacade.Object); + AqlResult actual = await jFrogService.GetPackageInfo("energy-dev-npm-egll", "saap-api-node-2.26.3-LicenseClearing.16.sha-058fada.tgz", string.Empty); + + // Assert + Assert.NotNull(actual); + } + + [Test] + public async Task GetPackageInfo_ResultsWith_NoContent() + { + // Arrange + + HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.NoContent); + + Mock mockJfrogApiComFacade = + new Mock(); + mockJfrogApiComFacade + .Setup(x => x.GetPackageInfo(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(httpResponseMessage); + + // Act + IJFrogService jFrogService = new JFrogService(mockJfrogApiComFacade.Object); + AqlResult actual = await jFrogService.GetPackageInfo("energy-dev-npm-egll", "saap-api-node-2.26.3-LicenseClearing.16.sha-058fada.tgz", string.Empty); + + // Assert + Assert.Null(actual); + } + + [Test] + public async Task GetPackageInfo_ResultsWith_HttpRequestException() + { + // Arrange + Mock mockJfrogApiComFacade = + new Mock(); + mockJfrogApiComFacade + .Setup(x => x.GetPackageInfo(It.IsAny(), It.IsAny(), It.IsAny())). + Throws(); + + // Act + IJFrogService jFrogService = new JFrogService(mockJfrogApiComFacade.Object); + AqlResult actual = await jFrogService.GetPackageInfo("energy-dev-npm-egll", "saap-api-node-2.26.3-LicenseClearing.16.sha-058fada.tgz", string.Empty); + + // Assert + Assert.Null(actual); + } } } diff --git a/src/LCT.Services/Interface/IJFrogService.cs b/src/LCT.Services/Interface/IJFrogService.cs index 3db1ed02..4bc647dc 100644 --- a/src/LCT.Services/Interface/IJFrogService.cs +++ b/src/LCT.Services/Interface/IJFrogService.cs @@ -21,5 +21,14 @@ public interface IJFrogService /// repoName /// IList public Task> GetInternalComponentDataByRepo(string repoName); + /// + /// Gets the package information in the repo, via the name or path + /// + /// repoName + /// repoName + /// repoName + /// AqlResult +#nullable enable + public Task GetPackageInfo(string repoName, string packageName, string path); } } diff --git a/src/LCT.Services/JFrogService.cs b/src/LCT.Services/JFrogService.cs index cf8a60a3..4351d5d0 100644 --- a/src/LCT.Services/JFrogService.cs +++ b/src/LCT.Services/JFrogService.cs @@ -14,6 +14,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.Linq; namespace LCT.Services { @@ -38,7 +39,6 @@ public async Task> GetInternalComponentDataByRepo(string repoNa try { httpResponseMessage = await m_JFrogApiCommunicationFacade.GetInternalComponentDataByRepo(repoName); - if (httpResponseMessage == null || !httpResponseMessage.IsSuccessStatusCode) { return new List(); @@ -61,6 +61,36 @@ public async Task> GetInternalComponentDataByRepo(string repoNa Logger.Debug(taskCancelledException); } + return aqlResult; + } + +#nullable enable + public async Task GetPackageInfo(string repoName, string packageName, string path) + { + HttpResponseMessage? httpResponseMessage = null; + AqlResult? aqlResult = null; + try + { + httpResponseMessage = await m_JFrogApiCommunicationFacade.GetPackageInfo(repoName, packageName, path); + httpResponseMessage.EnsureSuccessStatusCode(); + + string stringData = httpResponseMessage.Content?.ReadAsStringAsync()?.Result ?? string.Empty; + var aqlResponse = JsonConvert.DeserializeObject(stringData); + aqlResult = aqlResponse?.Results.FirstOrDefault(); + } + catch (HttpRequestException httpException) + { + Logger.Debug(httpException); + } + catch (InvalidOperationException invalidOperationExcep) + { + Logger.Debug(invalidOperationExcep); + } + catch (TaskCanceledException taskCancelledException) + { + Logger.Debug(taskCancelledException); + } + return aqlResult; } } diff --git a/src/SW360IntegrationTest/Conan/ArtifactoryUploaderConan.cs b/src/SW360IntegrationTest/Conan/ArtifactoryUploaderConan.cs index 2cec1a24..dd723892 100644 --- a/src/SW360IntegrationTest/Conan/ArtifactoryUploaderConan.cs +++ b/src/SW360IntegrationTest/Conan/ArtifactoryUploaderConan.cs @@ -5,7 +5,7 @@ using System.Net.Http; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Conan { [TestFixture, Order(28)] public class ArtifactoryUploaderConan @@ -21,10 +21,14 @@ public void TestArtifactoryUploaderexe() // Test BOM Creator ran with exit code 0 Assert.AreEqual(0, TestHelper.RunArtifactoryUploaderExe(new string[]{ TestConstant.BomFilePath, comparisonBOMPath, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.JfrogConanDestRepoName,testParameters.DestinationRepoName, - TestConstant.JFrogApiURL,testParameters.JfrogApi + TestConstant.JfrogConanThirdPartyDestRepoName,testParameters.ThirdPartyDestinationRepoName, + TestConstant.JfrogConanDevDestRepoName,testParameters.DevDestinationRepoName, + TestConstant.JfrogConanInternalDestRepoName,testParameters.InternalDestinationRepoName, + TestConstant.JFrogApiURL,testParameters.JfrogApi, + TestConstant.Release, false.ToString() }), "Test to run Artifactory Uploader EXE execution"); } diff --git a/src/SW360IntegrationTest/Conan/ComponentCreatorInitialConan.cs b/src/SW360IntegrationTest/Conan/ComponentCreatorInitialConan.cs index 03ef9de3..df25978b 100644 --- a/src/SW360IntegrationTest/Conan/ComponentCreatorInitialConan.cs +++ b/src/SW360IntegrationTest/Conan/ComponentCreatorInitialConan.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Conan { [TestFixture, Order(27)] public class ComponentCreatorInitialConan @@ -19,12 +19,12 @@ public class ComponentCreatorInitialConan public void Setup() { OutFolder = TestHelper.OutFolder; - CCTComparisonBomTestFile = OutFolder + @"..\..\..\src\SW360IntegrationTest\PackageCreatorTestFiles\Npm\CCTComparisonBOMNpmInitial.json"; + CCTComparisonBomTestFile = OutFolder + @"..\..\..\src\SW360IntegrationTest\PackageCreatorTestFiles\Conan\CCTComparisonBOMConanInitial.json"; if (!TestHelper.BOMCreated) { OutFolder = TestHelper.OutFolder; - string packagjsonPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\SystemTest1stIterationData"; + string packagjsonPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\SystemTest1stIterationData\Conan"; string bomPath = OutFolder + @"\..\BOMs"; TestHelper.RunBOMCreatorExe(new string[]{ TestConstant.PackageFilePath, packagjsonPath, @@ -36,8 +36,8 @@ public void Setup() TestConstant.SW360ProjectName, testParameters.SW360ProjectName, TestConstant.JFrogApiURL, testParameters.JfrogApi, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.ProjectType, "NPM", - TestConstant.Mode,"test" + TestConstant.ProjectType, "CONAN", + TestConstant.Mode,"" }); } } @@ -54,7 +54,7 @@ public void TestComponentCreatorExe_Conan() TestConstant.SW360AuthTokenType, testParameters.SW360AuthTokenType, TestConstant.SW360ProjectID, testParameters.SW360ProjectID, TestConstant.SW360ProjectName, testParameters.SW360ProjectName, - TestConstant.Mode,"test" + TestConstant.Mode,"" }), "Test to run Package Creator EXE execution"); } @@ -112,7 +112,7 @@ public async Task TestComponentCreation_Conan() string responseBody = await httpClient.GetStringAsync(url); //GET request var responseData = JsonConvert.DeserializeObject(responseBody); //Assert - Assert.IsTrue(responseData.Embedded.Sw360components.Count == 0); + Assert.IsTrue(responseData.Embedded.Sw360components.Count == 1); } diff --git a/src/SW360IntegrationTest/Conan/PackageIdentifierInitialConan.cs b/src/SW360IntegrationTest/Conan/PackageIdentifierInitialConan.cs index a1c05140..9b95d40d 100644 --- a/src/SW360IntegrationTest/Conan/PackageIdentifierInitialConan.cs +++ b/src/SW360IntegrationTest/Conan/PackageIdentifierInitialConan.cs @@ -3,7 +3,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Conan { [TestFixture, Order(26)] public class PackageIdentifierInitialConan diff --git a/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs b/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs index 8b4a4e13..600f569d 100644 --- a/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs +++ b/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs @@ -11,7 +11,7 @@ using System.Net.Http; using TestUtilities; -namespace SW360IntegrationTest.NPM +namespace SW360IntegrationTest.Maven { [TestFixture, Order(21)] public class ArtifactoryUploaderMaven @@ -27,10 +27,15 @@ public void TestArtifactoryUploaderexe() // Test BOM Creator ran with exit code 0 Assert.AreEqual(0, TestHelper.RunArtifactoryUploaderExe(new string[]{ TestConstant.BomFilePath, comparisonBOMPath, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.JfrogMavenDestRepoName,testParameters.DestinationRepoName, - TestConstant.JFrogApiURL,testParameters.JfrogApi}), + TestConstant.JfrogMavenThirdPartyDestRepoName,testParameters.ThirdPartyDestinationRepoName, + TestConstant.JfrogMavenDevDestRepoName,testParameters.DevDestinationRepoName, + TestConstant.JfrogMavenInternalDestRepoName,testParameters.InternalDestinationRepoName, + TestConstant.JFrogApiURL,testParameters.JfrogApi, + TestConstant.Release, false.ToString() + }), "Test to run Artifactory Uploader EXE execution"); } [Test, Order(2)] diff --git a/src/SW360IntegrationTest/Maven/ComponentCreatorInitialMaven.cs b/src/SW360IntegrationTest/Maven/ComponentCreatorInitialMaven.cs index f27a3952..0dedcc8a 100644 --- a/src/SW360IntegrationTest/Maven/ComponentCreatorInitialMaven.cs +++ b/src/SW360IntegrationTest/Maven/ComponentCreatorInitialMaven.cs @@ -17,7 +17,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Maven { [TestFixture, Order(20)] public class ComponentCreatorInitialMaven diff --git a/src/SW360IntegrationTest/Maven/PackageIdentifierInitialMaven.cs b/src/SW360IntegrationTest/Maven/PackageIdentifierInitialMaven.cs index cd518798..dd23bc01 100644 --- a/src/SW360IntegrationTest/Maven/PackageIdentifierInitialMaven.cs +++ b/src/SW360IntegrationTest/Maven/PackageIdentifierInitialMaven.cs @@ -10,7 +10,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Maven { [TestFixture, Order(19)] public class PackageIdentifierInitialMaven diff --git a/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs b/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs index 9bfafeba..b8930060 100644 --- a/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs +++ b/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs @@ -27,10 +27,14 @@ public void TestArtifactoryUploaderexe() // Test BOM Creator ran with exit code 0 Assert.AreEqual(0, TestHelper.RunArtifactoryUploaderExe(new string[]{ TestConstant.BomFilePath, comparisonBOMPath, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.JfrogNPMDestRepoName,testParameters.DestinationRepoName, - TestConstant.JFrogApiURL,testParameters.JfrogApi + TestConstant.JfrogNpmThirdPartyDestRepoName,testParameters.ThirdPartyDestinationRepoName, + TestConstant.JfrogNpmDevDestRepoName,testParameters.DevDestinationRepoName, + TestConstant.JfrogNpmInternalDestRepoName,testParameters.InternalDestinationRepoName, + TestConstant.JFrogApiURL,testParameters.JfrogApi, + TestConstant.Release, false.ToString() }), "Test to run Artifactory Uploader EXE execution"); } diff --git a/src/SW360IntegrationTest/NPM/ClearingToolLoadTest.cs b/src/SW360IntegrationTest/NPM/ClearingToolLoadTest.cs index c5d07bad..079a14b3 100644 --- a/src/SW360IntegrationTest/NPM/ClearingToolLoadTest.cs +++ b/src/SW360IntegrationTest/NPM/ClearingToolLoadTest.cs @@ -9,7 +9,7 @@ using System.Diagnostics; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.LoadTest { [Ignore("Load test need to run separatly")] [TestFixture, Order(11)] diff --git a/src/SW360IntegrationTest/NPM/ComponentCreatorInitial.cs b/src/SW360IntegrationTest/NPM/ComponentCreatorInitial.cs index f63e7e11..7367f48d 100644 --- a/src/SW360IntegrationTest/NPM/ComponentCreatorInitial.cs +++ b/src/SW360IntegrationTest/NPM/ComponentCreatorInitial.cs @@ -17,7 +17,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(5)] public class ComponentCreatorInitial diff --git a/src/SW360IntegrationTest/NPM/ComponentCreatorTestMode.cs b/src/SW360IntegrationTest/NPM/ComponentCreatorTestMode.cs index 53a4fec0..97e4c435 100644 --- a/src/SW360IntegrationTest/NPM/ComponentCreatorTestMode.cs +++ b/src/SW360IntegrationTest/NPM/ComponentCreatorTestMode.cs @@ -16,7 +16,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(2)] class ComponentCreatorTestMode diff --git a/src/SW360IntegrationTest/NPM/ComponentCreatorWithUpdatedComponents.cs b/src/SW360IntegrationTest/NPM/ComponentCreatorWithUpdatedComponents.cs index 238c1c15..9a9e4f57 100644 --- a/src/SW360IntegrationTest/NPM/ComponentCreatorWithUpdatedComponents.cs +++ b/src/SW360IntegrationTest/NPM/ComponentCreatorWithUpdatedComponents.cs @@ -18,7 +18,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(8)] public class ComponentCreatorWithUpdatedComponents diff --git a/src/SW360IntegrationTest/NPM/PackageIdentifierInitial.cs b/src/SW360IntegrationTest/NPM/PackageIdentifierInitial.cs index 4540ac21..a3b211f8 100644 --- a/src/SW360IntegrationTest/NPM/PackageIdentifierInitial.cs +++ b/src/SW360IntegrationTest/NPM/PackageIdentifierInitial.cs @@ -11,7 +11,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(4)] public class PackageIdentifierInitial diff --git a/src/SW360IntegrationTest/NPM/PackageIdentifierInitialTestMode.cs b/src/SW360IntegrationTest/NPM/PackageIdentifierInitialTestMode.cs index 5e41e660..08938970 100644 --- a/src/SW360IntegrationTest/NPM/PackageIdentifierInitialTestMode.cs +++ b/src/SW360IntegrationTest/NPM/PackageIdentifierInitialTestMode.cs @@ -11,7 +11,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(1)] public class PackageIdentifierInitialTestMode diff --git a/src/SW360IntegrationTest/NPM/PackageIdentifierWithMultiplePackageLockInputs.cs b/src/SW360IntegrationTest/NPM/PackageIdentifierWithMultiplePackageLockInputs.cs index e59d43d0..a67c8ea4 100644 --- a/src/SW360IntegrationTest/NPM/PackageIdentifierWithMultiplePackageLockInputs.cs +++ b/src/SW360IntegrationTest/NPM/PackageIdentifierWithMultiplePackageLockInputs.cs @@ -11,7 +11,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(10)] public class PackageIdentifierWithMultiplePackageLockInputs diff --git a/src/SW360IntegrationTest/NPM/PackageIdentifierWithUpdatedComponents.cs b/src/SW360IntegrationTest/NPM/PackageIdentifierWithUpdatedComponents.cs index e4522a1a..b7dd98a3 100644 --- a/src/SW360IntegrationTest/NPM/PackageIdentifierWithUpdatedComponents.cs +++ b/src/SW360IntegrationTest/NPM/PackageIdentifierWithUpdatedComponents.cs @@ -11,7 +11,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.NPM { [TestFixture, Order(7)] public class PackageIdentifierWithUpdatedComponents diff --git a/src/SW360IntegrationTest/Nuget/ArtifactoryUploaderNuget.cs b/src/SW360IntegrationTest/Nuget/ArtifactoryUploaderNuget.cs index d19706cd..1d521757 100644 --- a/src/SW360IntegrationTest/Nuget/ArtifactoryUploaderNuget.cs +++ b/src/SW360IntegrationTest/Nuget/ArtifactoryUploaderNuget.cs @@ -12,7 +12,7 @@ using System.Net.Http; using TestUtilities; -namespace SW360IntegrationTest.NPM +namespace SW360IntegrationTest.Nuget { [TestFixture, Order(16)] public class ArtifactoryUploaderNuget @@ -30,7 +30,9 @@ public void TestArtifactoryUploaderexe() TestConstant.BomFilePath, comparisonBOMPath, TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.JfrogNugetDestRepoName,testParameters.DestinationRepoName, + TestConstant.JfrogNpmThirdPartyDestRepoName,testParameters.ThirdPartyDestinationRepoName, + TestConstant.JfrogNpmDevDestRepoName,testParameters.DevDestinationRepoName, + TestConstant.JfrogNpmInternalDestRepoName,testParameters.InternalDestinationRepoName, TestConstant.JFrogApiURL,testParameters.JfrogApi }), "Test to run Artifactory Uploader EXE execution"); diff --git a/src/SW360IntegrationTest/Nuget/ComponentCreatorInitialNuget.cs b/src/SW360IntegrationTest/Nuget/ComponentCreatorInitialNuget.cs index b1e44e16..e5aa9ece 100644 --- a/src/SW360IntegrationTest/Nuget/ComponentCreatorInitialNuget.cs +++ b/src/SW360IntegrationTest/Nuget/ComponentCreatorInitialNuget.cs @@ -10,12 +10,13 @@ using Newtonsoft.Json; using NUnit.Framework; using System.IO; +using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Nuget { [TestFixture, Order(14)] public class ComponentCreatorInitialNuget @@ -148,7 +149,7 @@ public async Task ReleaseCreation__AfterSuccessfulExeRun_ReturnsClearingStateAsN string url = TestConstant.Sw360ReleaseApi + TestConstant.componentNameUrl + "Newtonsoft.Json"; string responseBody = await httpClient.GetStringAsync(url);//GET method var responseData = JsonConvert.DeserializeObject(responseBody); - string urlofreleaseid = responseData.Embedded.Sw360Releases[0].Links.Self.Href; + string urlofreleaseid = responseData.Embedded.Sw360Releases.First(x => x.Version == "12.0.3").Links.Self.Href.ToString(); string responseForRelease = await httpClient.GetStringAsync(urlofreleaseid);//GET method for fetching the release details var responseDataForRelease = JsonConvert.DeserializeObject(responseForRelease); diff --git a/src/SW360IntegrationTest/Nuget/NugetTemplate/ComponentCreatorNugetTemplate.cs b/src/SW360IntegrationTest/Nuget/NugetTemplate/ComponentCreatorNugetTemplate.cs index 123a3f48..d10d5653 100644 --- a/src/SW360IntegrationTest/Nuget/NugetTemplate/ComponentCreatorNugetTemplate.cs +++ b/src/SW360IntegrationTest/Nuget/NugetTemplate/ComponentCreatorNugetTemplate.cs @@ -16,7 +16,7 @@ using System.Threading.Tasks; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Nuget { [TestFixture, Order(23)] public class ComponentCreatorNugetTemplate diff --git a/src/SW360IntegrationTest/Nuget/NugetTemplate/PackageIdentifierNugetTemplate.cs b/src/SW360IntegrationTest/Nuget/NugetTemplate/PackageIdentifierNugetTemplate.cs index c2f4cf4a..f127411c 100644 --- a/src/SW360IntegrationTest/Nuget/NugetTemplate/PackageIdentifierNugetTemplate.cs +++ b/src/SW360IntegrationTest/Nuget/NugetTemplate/PackageIdentifierNugetTemplate.cs @@ -10,7 +10,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Nuget { [TestFixture, Order(22)] public class PackageIdentifierNugetTemplate diff --git a/src/SW360IntegrationTest/Nuget/PackageIdentifierInitialNuget.cs b/src/SW360IntegrationTest/Nuget/PackageIdentifierInitialNuget.cs index 13e7d9e7..b8c0815e 100644 --- a/src/SW360IntegrationTest/Nuget/PackageIdentifierInitialNuget.cs +++ b/src/SW360IntegrationTest/Nuget/PackageIdentifierInitialNuget.cs @@ -10,7 +10,7 @@ using System.IO; using TestUtilities; -namespace SW360IntegrationTest +namespace SW360IntegrationTest.Nuget { [TestFixture, Order(13)] public class PackageIdentifierInitialNuget diff --git a/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs b/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs index 5a10e1e7..f4530192 100644 --- a/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs +++ b/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs @@ -28,16 +28,20 @@ public void TestArtifactoryUploaderexe() int result = TestHelper.RunArtifactoryUploaderExe(new string[]{ TestConstant.BomFilePath, comparisonBOMPath, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, - TestConstant.JfrogNPMDestRepoName,testParameters.DestinationRepoName, - TestConstant.JFrogApiURL,testParameters.JfrogApi + TestConstant.JfrogPythonThirdPartyDestRepoName,testParameters.ThirdPartyDestinationRepoName, + TestConstant.JfrogPythonDevDestRepoName,testParameters.DevDestinationRepoName, + TestConstant.JfrogPythonInternalDestRepoName,testParameters.InternalDestinationRepoName, + TestConstant.JFrogApiURL,testParameters.JfrogApi, + TestConstant.Release, false.ToString() }); // Test BOM Creator ran with exit code 0 or 2 (Warning) Assert.IsTrue(result == 0 || result == 2, "Test to run Artifactory Uploader EXE execution"); - } + } [Test, Order(2)] public void ComponentUpload_IsFailure() diff --git a/src/TestUtilities/TestConstant.cs b/src/TestUtilities/TestConstant.cs index 088e6e99..64fcf655 100644 --- a/src/TestUtilities/TestConstant.cs +++ b/src/TestUtilities/TestConstant.cs @@ -17,8 +17,8 @@ public static class TestConstant public static readonly string Sw360ComponentApi = $"{s_testParamObj.SW360URL}/resource/api/components"; public static readonly string Sw360ReleaseApi = $"{s_testParamObj.SW360URL}/resource/api/releases"; - public static readonly string JfrogApi = $"{s_testParamObj.JfrogApi}/api/storage/{s_testParamObj.DestinationRepoName}"; - public static readonly string JfrogApiNuget = $"{s_testParamObjnuget.JfrogApi}/api/storage/{s_testParamObjnuget.DestinationRepoName}"; + public static readonly string JfrogApi = $"{s_testParamObj.JfrogApi}/api/storage/{s_testParamObj.ThirdPartyDestinationRepoName}"; + public static readonly string JfrogApiNuget = $"{s_testParamObjnuget.JfrogApi}/api/storage/{s_testParamObjnuget.ThirdPartyDestinationRepoName}"; @@ -43,12 +43,30 @@ public static class TestConstant public const string Email = "Email"; public const string ArtifactoryUser = "--artifactoryuploaduser"; public const string ArtifactoryKey = "--artifactoryuploadapikey"; - public const string JfrogNPMDestRepoName = "--jfrognpmdestreponame "; - public const string JfrogNugetDestRepoName = "--jfrognugetdestreponame "; - public const string JfrogMavenDestRepoName = "--jfrogmavendestreponame "; - public const string JfrogConanDestRepoName = "--jfrogconandestreponame "; + + public const string JfrogNpmThirdPartyDestRepoName = "--npm:JfrogThirdPartyDestRepoName "; + public const string JfrogNpmInternalDestRepoName = "--npm:JfrogInternalDestRepoName "; + public const string JfrogNpmDevDestRepoName = "--npm:JfrogDevDestRepoName "; + + public const string JfrogMavenThirdPartyDestRepoName = "--maven:JfrogThirdPartyDestRepoName "; + public const string JfrogMavenInternalDestRepoName = "--maven:JfrogInternalDestRepoName "; + public const string JfrogMavenDevDestRepoName = "--maven:JfrogDevDestRepoName "; + + public const string JfrogNugetThirdPartyDestRepoName = "--nuget:JfrogThirdPartyDestRepoName "; + public const string JfrogNugetInternalDestRepoName = "--nuget:JfrogInternalDestRepoName "; + public const string JfrogNugetDevDestRepoName = "--nuget:JfrogDevDestRepoName "; + + public const string JfrogPythonThirdPartyDestRepoName = "--python:JfrogThirdPartyDestRepoName "; + public const string JfrogPythonInternalDestRepoName = "--python:JfrogInternalDestRepoName "; + public const string JfrogPythonDevDestRepoName = "--python:JfrogDevDestRepoName "; + + public const string JfrogConanThirdPartyDestRepoName = "--conan:JfrogThirdPartyDestRepoName "; + public const string JfrogConanInternalDestRepoName = "--conan:JfrogInternalDestRepoName "; + public const string JfrogConanDevDestRepoName = "--conan:JfrogDevDestRepoName "; + public const string NuspecMode = "--NuspecMode"; public const string JFrogApiURL = "--JFrogApi"; - public const string CycloneDxSBomTemplatePath = "--cycloneDxSBomTemplatePath"; + public const string CycloneDxSBomTemplatePath = "--cycloneDxSBomTemplatePath"; + public const string Release = "--release"; } } diff --git a/src/TestUtilities/TestParam.cs b/src/TestUtilities/TestParam.cs index 3b9fa5f4..5a901092 100644 --- a/src/TestUtilities/TestParam.cs +++ b/src/TestUtilities/TestParam.cs @@ -27,7 +27,9 @@ public class TestParam public string ArtifactoryUploadUser { get; set; } public string ArtifactoryUploadApiKey { get; set; } public string JfrogApi { get; set; } - public string DestinationRepoName { get; set; } + public string ThirdPartyDestinationRepoName { get; set; } + public string InternalDestinationRepoName { get; set; } + public string DevDestinationRepoName { get; set; } public TestParam() { @@ -42,7 +44,9 @@ public TestParam() ArtifactoryUploadUser = s_Config["ArtifactoryUploadUser"]; ArtifactoryUploadApiKey = s_Config["ArtifactoryUploadApiKey"]; JfrogApi = s_Config["JfrogApi"]; - DestinationRepoName = "npm-test"; + ThirdPartyDestinationRepoName = "npm-test"; + InternalDestinationRepoName = "npm-test"; + DevDestinationRepoName = "npm-test"; } } } diff --git a/src/TestUtilities/TestParamConan.cs b/src/TestUtilities/TestParamConan.cs index f22d6682..e35b5e60 100644 --- a/src/TestUtilities/TestParamConan.cs +++ b/src/TestUtilities/TestParamConan.cs @@ -24,7 +24,9 @@ public class TestParamConan public string ArtifactoryUploadUser { get; set; } public string ArtifactoryUploadApiKey { get; set; } public string JfrogApi { get; set; } - public string DestinationRepoName { get; set; } + public string ThirdPartyDestinationRepoName { get; set; } + public string InternalDestinationRepoName { get; set; } + public string DevDestinationRepoName { get; set; } public TestParamConan() { @@ -39,7 +41,9 @@ public TestParamConan() ArtifactoryUploadUser = s_Config["ArtifactoryUploadUser"]; ArtifactoryUploadApiKey = s_Config["ArtifactoryUploadApiKey"]; JfrogApi = s_Config["JfrogApi"]; - DestinationRepoName = "conan-test"; + ThirdPartyDestinationRepoName = "conan-test"; + InternalDestinationRepoName = "conan-test"; + DevDestinationRepoName = "conan-test"; } } } diff --git a/src/TestUtilities/TestParamMaven.cs b/src/TestUtilities/TestParamMaven.cs index f6f912b0..da72d3c9 100644 --- a/src/TestUtilities/TestParamMaven.cs +++ b/src/TestUtilities/TestParamMaven.cs @@ -24,6 +24,9 @@ public class TestParamMaven public string ArtifactoryUploadApiKey { get; set; } public string JfrogApi { get; set; } public string DestinationRepoName { get; set; } + public string ThirdPartyDestinationRepoName { get; set; } + public string InternalDestinationRepoName { get; set; } + public string DevDestinationRepoName { get; set; } public TestParamMaven() { @@ -38,7 +41,9 @@ public TestParamMaven() ArtifactoryUploadUser = s_Config["ArtifactoryUploadUser"]; ArtifactoryUploadApiKey = s_Config["ArtifactoryUploadApiKey"]; JfrogApi = s_Config["JfrogApi"]; - DestinationRepoName = "maven-test"; + ThirdPartyDestinationRepoName = "maven-test"; + InternalDestinationRepoName = "maven-test"; + DevDestinationRepoName = "maven-test"; } } } diff --git a/src/TestUtilities/TestParamNuget.cs b/src/TestUtilities/TestParamNuget.cs index 1bf43ed2..9530d2fa 100644 --- a/src/TestUtilities/TestParamNuget.cs +++ b/src/TestUtilities/TestParamNuget.cs @@ -29,7 +29,9 @@ public class TestParamNuget public string ArtifactoryUploadUser { get; set; } public string ArtifactoryUploadApiKey { get; set; } public string JfrogApi { get; set; } - public string DestinationRepoName { get; set; } + public string ThirdPartyDestinationRepoName { get; set; } + public string InternalDestinationRepoName { get; set; } + public string DevDestinationRepoName { get; set; } public TestParamNuget() { @@ -44,7 +46,9 @@ public TestParamNuget() ArtifactoryUploadUser = s_Config["ArtifactoryUploadUser"]; ArtifactoryUploadApiKey = s_Config["ArtifactoryUploadApiKey"]; JfrogApi = s_Config["JfrogApi"]; - DestinationRepoName = "nuget-test"; + ThirdPartyDestinationRepoName = "nuget-test"; + InternalDestinationRepoName = "nuget-test"; + DevDestinationRepoName = "nuget-test"; } } }