From fb86c7176310a52d1a70d50c4caad276a1632c1d Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 24 Nov 2023 17:05:04 +0530 Subject: [PATCH 1/4] Create collect endpoint for ondemand task Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 390 +++++++++++++++++++++++------------------------ purldb/urls.py | 3 + 2 files changed, 193 insertions(+), 200 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index cc47c291..fa468a56 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -283,72 +283,6 @@ def resources(self, request, *args, **kwargs): serializer = ResourceAPISerializer(paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) - @action(detail=False) - def get_package(self, request, *args, **kwargs): - purl = request.query_params.get('purl') - - # validate purl - try: - package_url = PackageURL.from_string(purl) - except ValueError as e: - message = { - 'status': f'purl validation error: {e}' - } - return Response(message, status=status.HTTP_400_BAD_REQUEST) - - lookups = purl_to_lookups(purl) - packages = Package.objects.filter(**lookups) - if packages.count() == 0: - # add to queue - PriorityResourceURI.objects.insert(purl) - return Response({}) - - serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return Response(serializer.data) - - @action(detail=False) - def get_or_fetch_package(self, request, *args, **kwargs): - """ - Return Package data for the purl passed in the `purl` query parameter. - - If the package does not exist, we will fetch the Package data and return - it in the same request. - """ - purl = request.query_params.get('purl') - - # validate purl - try: - package_url = PackageURL.from_string(purl) - except ValueError as e: - message = { - 'status': f'purl validation error: {e}' - } - return Response(message, status=status.HTTP_400_BAD_REQUEST) - - lookups = purl_to_lookups(purl) - packages = Package.objects.filter(**lookups) - if packages.count() == 0: - try: - errors = priority_router.process(purl) - except NoRouteAvailable: - message = { - 'status': f'cannot fetch Package data for {purl}: no available handler' - } - return Response(message, status=status.HTTP_400_BAD_REQUEST) - - lookups = purl_to_lookups(purl) - packages = Package.objects.filter(**lookups) - if packages.count() == 0: - message = {} - if errors: - message = { - 'status': f'error(s) occured when fetching metadata for {purl}: {errors}' - } - return Response(message) - - serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return Response(serializer.data) - @action(detail=True) def get_enhanced_package_data(self, request, *args, **kwargs): """ @@ -358,86 +292,6 @@ def get_enhanced_package_data(self, request, *args, **kwargs): package_data = get_enhanced_package(package) return Response(package_data) - @action(detail=False, methods=['post']) - def index_packages(self, request, *args, **kwargs): - """ - Take a list of `packages` where each item is a dictionary containing either PURL - or versionless PURL along with vers range. - **Note:** When a versionless PURL is supplied without a vers range, then all the versions - of that package will be indexed. - - **Input example:** - - { - "packages": [ - { - "purl": "pkg:npm/foobar@12.3.1", - }, - { - "purl": "pkg:npm/foobar", - "vers": "vers:npm/>=1.0.0|<=4.1.0" - }, - { - "purl": "pkg:npm/foobar2", - } - ... - ] - } - - Then return a mapping containing: - - - queued_packages_count - - The number of package urls placed on the queue. - - queued_packages - - A list of package urls that were placed on the queue. - - unqueued_packages_count - - The number of package urls not placed on the queue. This is - because the package url already exists on the queue and has not - yet been processed. - - unqueued_packages - - A list of package urls that were not placed on the queue. - - unsupported_packages_count - - The number of package urls that are not processable by the queue. - - unsupported_packages - - A list of package urls that are not processable by the queue. The - package indexing queue can only handle npm and maven purls. - - unqueued_packages - - A list of package urls that were not placed on the queue. - - unsupported_vers_count - - The number of vers range that are not supported by the univers or package_manager. - - unsupported_vers - - A list of vers range that are not supported by the univers or package_manager. - """ - packages = request.data.get('packages') or [] - queued_packages = [] - unqueued_packages = [] - supported_ecosystems = ['maven', 'npm'] - - unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems) - - for purl in unique_purls: - is_routable_purl = priority_router.is_routable(purl) - if not is_routable_purl: - unsupported_packages.append(purl) - else: - # add to queue - priority_resource_uri = PriorityResourceURI.objects.insert(purl) - if priority_resource_uri: - queued_packages.append(purl) - else: - unqueued_packages.append(purl) - response_data = { - 'queued_packages_count': len(queued_packages), - 'queued_packages': queued_packages, - 'unqueued_packages_count': len(unqueued_packages), - 'unqueued_packages': unqueued_packages, - 'unsupported_packages_count': len(unsupported_packages), - 'unsupported_packages': unsupported_packages, - 'unsupported_vers_count': len(unsupported_vers), - 'unsupported_vers': unsupported_vers, - } - return Response(response_data) - @action(detail=True) def reindex_package(self, request, *args, **kwargs): """ @@ -450,60 +304,6 @@ def reindex_package(self, request, *args, **kwargs): } return Response(data) - @action(detail=False, methods=['post']) - def reindex_packages(self, request, *args, **kwargs): - """ - Take a list of `package_urls` and for each Package URL, reindex the - corresponding package. - If the field `reindex_set` is True, then the Packages in the same - package set as the packages from `package_urls` will be reindexed. - - Then return a mapping containing: - - - requeued_packages_count - - The number of package urls placed on the queue. - - requeued_packages - - A list of package urls that were placed on the queue. - - nonexistent_packages_count - - The number of package urls that do not correspond to a package in - the database. - - nonexistent_packages - - A list of package urls that do not correspond to a package in the - database. - """ - def _reindex_package(package, reindexed_packages): - if package in reindexed_packages: - return - package.rescan() - reindexed_packages.append(package) - - purls = request.data.getlist('package_urls') - reindex_set = request.data.get('reindex_set') or False - - nonexistent_packages = [] - reindexed_packages = [] - for purl in purls: - lookups = purl_to_lookups(purl) - packages = Package.objects.filter(**lookups) - if packages.count() > 0: - for package in packages: - _reindex_package(package, reindexed_packages) - if reindex_set: - for package_set in package.package_sets.all(): - for p in package_set.packages.all(): - _reindex_package(p, reindexed_packages) - else: - nonexistent_packages.append(purl) - - requeued_packages = [p.package_url for p in reindexed_packages] - response_data = { - 'requeued_packages_count': len(requeued_packages), - 'requeued_packages': requeued_packages, - 'nonexistent_packages_count': len(nonexistent_packages), - 'nonexistent_packages': nonexistent_packages, - } - return Response(response_data) - @action(detail=False, methods=['post']) def filter_by_checksums(self, request, *args, **kwargs): """ @@ -709,6 +509,196 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet): queryset = PackageSet.objects.prefetch_related('packages') serializer_class = PackageSetAPISerializer +class CollectViewSet(viewsets.ViewSet): + """ + Return Package data for the purl passed in the `purl` query parameter. + + If the package does not exist, we will fetch the Package data and return + it in the same request. + + **Note:** Use `Index packages` for bulk indexing of packages; use `Reindex packages` + for bulk reindexing of existing packages. + """ + + def list(self, request, format=None): + purl = request.query_params.get('purl') + + # validate purl + try: + package_url = PackageURL.from_string(purl) + except ValueError as e: + message = { + 'status': f'purl validation error: {e}' + } + return Response(message, status=status.HTTP_400_BAD_REQUEST) + + lookups = purl_to_lookups(purl) + packages = Package.objects.filter(**lookups) + if packages.count() == 0: + try: + errors = priority_router.process(purl) + except NoRouteAvailable: + message = { + 'status': f'cannot fetch Package data for {purl}: no available handler' + } + return Response(message, status=status.HTTP_400_BAD_REQUEST) + + lookups = purl_to_lookups(purl) + packages = Package.objects.filter(**lookups) + if packages.count() == 0: + message = {} + if errors: + message = { + 'status': f'error(s) occured when fetching metadata for {purl}: {errors}' + } + return Response(message) + + serializer = PackageAPISerializer(packages, many=True, context={'request': request}) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def index_packages(self, request, *args, **kwargs): + """ + Take a list of `packages` where each item is a dictionary containing either PURL + or versionless PURL along with vers range. + **Note:** When a versionless PURL is supplied without a vers range, then all the versions + of that package will be indexed. + + **Input example:** + + { + "packages": [ + { + "purl": "pkg:npm/less@1.0.32", + "vers": null + }, + { + "purl": "pkg:npm/less", + "vers": "vers:npm/>=1.1.0|<=1.1.4" + }, + { + "purl": "pkg:npm/foobar", + "vers": null + } + ] + } + + Then return a mapping containing: + + - queued_packages_count + - The number of package urls placed on the queue. + - queued_packages + - A list of package urls that were placed on the queue. + - unqueued_packages_count + - The number of package urls not placed on the queue. This is + because the package url already exists on the queue and has not + yet been processed. + - unqueued_packages + - A list of package urls that were not placed on the queue. + - unsupported_packages_count + - The number of package urls that are not processable by the queue. + - unsupported_packages + - A list of package urls that are not processable by the queue. The + package indexing queue can only handle npm and maven purls. + - unqueued_packages + - A list of package urls that were not placed on the queue. + - unsupported_vers_count + - The number of vers range that are not supported by the univers or package_manager. + - unsupported_vers + - A list of vers range that are not supported by the univers or package_manager. + """ + packages = request.data.get('packages') or [] + queued_packages = [] + unqueued_packages = [] + supported_ecosystems = ['maven', 'npm'] + + unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems) + + for purl in unique_purls: + is_routable_purl = priority_router.is_routable(purl) + if not is_routable_purl: + unsupported_packages.append(purl) + else: + # add to queue + priority_resource_uri = PriorityResourceURI.objects.insert(purl) + if priority_resource_uri: + queued_packages.append(purl) + else: + unqueued_packages.append(purl) + response_data = { + 'queued_packages_count': len(queued_packages), + 'queued_packages': queued_packages, + 'unqueued_packages_count': len(unqueued_packages), + 'unqueued_packages': unqueued_packages, + 'unsupported_packages_count': len(unsupported_packages), + 'unsupported_packages': unsupported_packages, + 'unsupported_vers_count': len(unsupported_vers), + 'unsupported_vers': unsupported_vers, + } + return Response(response_data) + + @action(detail=False, methods=['post']) + def reindex_packages(self, request, *args, **kwargs): + """ + Take a list of `package_urls` and for each Package URL, reindex the + corresponding package. + If the field `reindex_set` is True, then the Packages in the same + package set as the packages from `package_urls` will be reindexed. + + **Input example:** + + { + "package_urls": [ + "pkg:npm/foobar@1.1.0", + ], + "reindex_set": true, + } + + Then return a mapping containing: + + - requeued_packages_count + - The number of package urls placed on the queue. + - requeued_packages + - A list of package urls that were placed on the queue. + - nonexistent_packages_count + - The number of package urls that do not correspond to a package in + the database. + - nonexistent_packages + - A list of package urls that do not correspond to a package in the + database. + """ + def _reindex_package(package, reindexed_packages): + if package in reindexed_packages: + return + package.rescan() + reindexed_packages.append(package) + + purls = request.data.get('package_urls') or [] + reindex_set = request.data.get('reindex_set') or False + + nonexistent_packages = [] + reindexed_packages = [] + for purl in purls: + lookups = purl_to_lookups(purl) + packages = Package.objects.filter(**lookups) + if packages.count() > 0: + for package in packages: + _reindex_package(package, reindexed_packages) + if reindex_set: + for package_set in package.package_sets.all(): + for p in package_set.packages.all(): + _reindex_package(p, reindexed_packages) + else: + nonexistent_packages.append(purl) + + requeued_packages = [p.package_url for p in reindexed_packages] + response_data = { + 'requeued_packages_count': len(requeued_packages), + 'requeued_packages': requeued_packages, + 'nonexistent_packages_count': len(nonexistent_packages), + 'nonexistent_packages': nonexistent_packages, + } + return Response(response_data) def get_resolved_purls(packages, supported_ecosystems): """ diff --git a/purldb/urls.py b/purldb/urls.py index 045a2d34..3b66682a 100644 --- a/purldb/urls.py +++ b/purldb/urls.py @@ -21,6 +21,7 @@ from matchcode.api import ExactFileIndexViewSet from matchcode.api import ExactPackageArchiveIndexViewSet from minecode.api import PriorityResourceURIViewSet +from packagedb.api import CollectViewSet api_router = routers.DefaultRouter() @@ -33,6 +34,8 @@ api_router.register('exact_package_archive_index', ExactPackageArchiveIndexViewSet) api_router.register('cditems', CDitemViewSet, 'cditems') api_router.register('on_demand_queue', PriorityResourceURIViewSet) +api_router.register('collect', CollectViewSet, 'collect') + urlpatterns = [ path('api/', include((api_router.urls, 'api'))), From 1d404daaf9313b8e3e3cee5d6675f375af2f746f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 24 Nov 2023 17:07:43 +0530 Subject: [PATCH 2/4] Test collect endpoint Signed-off-by: Keshav Priyadarshi --- packagedb/tests/test_api.py | 550 ++++++++++++++++++------------------ 1 file changed, 280 insertions(+), 270 deletions(-) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index b2ad0b3b..6f6b1b20 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -460,173 +460,6 @@ def test_api_package_list_endpoint_multiple_char_filters(self): self.assertIn(self.package3.purl, purls) self.assertNotIn(self.package.purl, purls) - def test_package_api_index_packages_endpoint(self): - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(0, priority_resource_uris_count) - packages = [ - {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, - {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, - {'purl':'pkg:bitbucket/example/example@1.0.0'}, - ] - data = { - 'packages': packages - } - response = self.client.post('/api/packages/index_packages/', data=data, format='json') - self.assertEqual(2, response.data['queued_packages_count']) - expected_queued_packages = [ - 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', - 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0', - ] - self.assertEqual( - sorted(expected_queued_packages), - sorted(response.data['queued_packages']) - ) - self.assertEqual(0, response.data['unqueued_packages_count']) - self.assertEqual([], response.data['unqueued_packages']) - self.assertEqual(1, response.data['unsupported_packages_count']) - expected_unsupported_packages = [ - 'pkg:bitbucket/example/example@1.0.0' - ] - self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(2, priority_resource_uris_count) - - # Ensure that we don't add the same packages to the queue if they have - # not yet been processed - purls = [ - {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, - {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, - {'purl':'pkg:bitbucket/example/example@1.0.0'}, - ] - data = { - 'packages': purls - } - response = self.client.post('/api/packages/index_packages/', data=data, format='json') - self.assertEqual(0, response.data['queued_packages_count']) - self.assertEqual([], response.data['queued_packages']) - self.assertEqual(2, response.data['unqueued_packages_count']) - expected_unqueued_packages = [ - 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', - 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0', - ] - self.assertEqual( - sorted(expected_unqueued_packages), - sorted(response.data['unqueued_packages']) - ) - self.assertEqual(1, response.data['unsupported_packages_count']) - expected_unsupported_packages = [ - 'pkg:bitbucket/example/example@1.0.0' - ] - self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) - - @mock.patch("packagedb.api.get_all_versions") - def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versions): - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(0, priority_resource_uris_count) - packages = [ - { - "purl": "pkg:maven/ch.qos.reload4j/reload4j", - "vers": "vers:maven/>=1.2.18.2|<=1.2.23", - }, - ] - data = {"packages": packages} - - mock_get_all_versions.return_value = [ - MavenVersion("1.2.18.0"), - MavenVersion("1.2.18.1"), - MavenVersion("1.2.18.2"), - MavenVersion("1.2.18.3"), - MavenVersion("1.2.18.4"), - MavenVersion("1.2.18.5"), - MavenVersion("1.2.19"), - MavenVersion("1.2.20"), - MavenVersion("1.2.21"), - MavenVersion("1.2.22"), - MavenVersion("1.2.23"), - MavenVersion("1.2.24"), - MavenVersion("1.2.25"), - ] - - response = self.client.post( - "/api/packages/index_packages/", data=data, format="json" - ) - self.assertEqual(9, response.data["queued_packages_count"]) - - expected_queued_packages = [ - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.19", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.20", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.21", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.22", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", - ] - self.assertEqual( - sorted(expected_queued_packages), sorted(response.data["queued_packages"]) - ) - self.assertEqual(0, response.data["unqueued_packages_count"]) - self.assertEqual([], response.data["unqueued_packages"]) - self.assertEqual(0, response.data["unsupported_packages_count"]) - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(9, priority_resource_uris_count) - - @mock.patch("packagedb.api.get_all_versions") - def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_all_versions): - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(0, priority_resource_uris_count) - packages = [ - { - "purl": "pkg:maven/ch.qos.reload4j/reload4j", - }, - ] - data = {"packages": packages} - - mock_get_all_versions.return_value = [ - MavenVersion("1.2.18.0"), - MavenVersion("1.2.18.1"), - MavenVersion("1.2.18.2"), - MavenVersion("1.2.18.3"), - MavenVersion("1.2.18.4"), - MavenVersion("1.2.18.5"), - MavenVersion("1.2.19"), - MavenVersion("1.2.20"), - MavenVersion("1.2.21"), - MavenVersion("1.2.22"), - MavenVersion("1.2.23"), - MavenVersion("1.2.24"), - MavenVersion("1.2.25"), - ] - - response = self.client.post( - "/api/packages/index_packages/", data=data, format="json" - ) - self.assertEqual(13, response.data["queued_packages_count"]) - - expected_queued_packages = [ - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.0", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.1", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.19", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.20", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.21", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.22", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.24", - "pkg:maven/ch.qos.reload4j/reload4j@1.2.25", - ] - self.assertEqual( - sorted(expected_queued_packages), sorted(response.data["queued_packages"]) - ) - self.assertEqual(0, response.data["unqueued_packages_count"]) - self.assertEqual([], response.data["unqueued_packages"]) - self.assertEqual(0, response.data["unsupported_packages_count"]) - priority_resource_uris_count = PriorityResourceURI.objects.all().count() - self.assertEqual(13, priority_resource_uris_count) def test_package_api_filter_by_checksums(self): sha1s = [ @@ -682,34 +515,6 @@ def setUp(self): self.scan_request_date = timezone.now() self.scannableuri.scan_request_date = self.scan_request_date - package_download_url2 = 'http://somethingelse.org' - self.package_data2 = { - 'type': 'npm', - 'namespace': 'example', - 'name': 'bar', - 'version': '56.78', - 'qualifiers': '', - 'subpath': '', - 'download_url': package_download_url2, - 'filename': 'Bar.zip', - 'sha1': 'testsha1-2', - 'md5': 'testmd5-2', - 'size': 100, - } - self.package2 = Package.objects.create(**self.package_data2) - self.package2.refresh_from_db() - self.scannableuri2 = ScannableURI.objects.create( - package=self.package2, - uri=package_download_url2, - ) - self.scannableuri2.scan_status = ScannableURI.SCAN_INDEXED - self.scan_uuid2 = uuid4() - self.scannableuri2.scan_uuid = self.scan_uuid2 - self.scannableuri2.scan_error = 'error' - self.scannableuri2.index_error = 'error' - self.scan_request_date2 = timezone.now() - self.scannableuri2.scan_request_date = self.scan_request_date2 - def test_reindex_package(self): self.assertEqual(False, self.scannableuri.rescan_uri) self.assertEqual(0, self.scannableuri.priority) @@ -727,52 +532,6 @@ def test_reindex_package(self): self.assertEqual(None, self.scannableuri.index_error) self.assertEqual(None, self.scannableuri.scan_request_date) - def test_reindex_packages_basic(self): - self.assertEqual(False, self.scannableuri.rescan_uri) - self.assertEqual(0, self.scannableuri.priority) - self.assertEqual(self.scan_uuid, self.scannableuri.scan_uuid) - self.assertEqual('error', self.scannableuri.scan_error) - self.assertEqual('error', self.scannableuri.index_error) - self.assertEqual(self.scan_request_date, self.scannableuri.scan_request_date) - - self.assertEqual(False, self.scannableuri2.rescan_uri) - self.assertEqual(0, self.scannableuri2.priority) - self.assertEqual(self.scan_uuid2, self.scannableuri2.scan_uuid) - self.assertEqual('error', self.scannableuri2.scan_error) - self.assertEqual('error', self.scannableuri2.index_error) - self.assertEqual(self.scan_request_date2, self.scannableuri2.scan_request_date) - existing_purls = [ - 'pkg:maven/sample/Baz@90.12', - 'pkg:npm/example/bar@56.78', - ] - nonexistent_purls = [ - 'pkg:pypi/does/not-exist@1', - ] - data = { - 'package_urls': existing_purls + nonexistent_purls, - } - response = self.client.post(f'/api/packages/reindex_packages/', data=data) - self.assertEqual(2, response.data['requeued_packages_count']) - self.assertEqual(existing_purls, response.data['requeued_packages']) - self.assertEqual(1, response.data['nonexistent_packages_count']) - self.assertEqual(nonexistent_purls, response.data['nonexistent_packages']) - - self.scannableuri.refresh_from_db() - self.assertEqual(True, self.scannableuri.rescan_uri) - self.assertEqual(100, self.scannableuri.priority) - self.assertEqual(None, self.scannableuri.scan_uuid) - self.assertEqual(None, self.scannableuri.scan_error) - self.assertEqual(None, self.scannableuri.index_error) - self.assertEqual(None, self.scannableuri.scan_request_date) - - self.scannableuri2.refresh_from_db() - self.assertEqual(True, self.scannableuri2.rescan_uri) - self.assertEqual(100, self.scannableuri.priority) - self.assertEqual(None, self.scannableuri2.scan_uuid) - self.assertEqual(None, self.scannableuri2.scan_error) - self.assertEqual(None, self.scannableuri2.index_error) - self.assertEqual(None, self.scannableuri2.scan_request_date) - class PackageApiPurlFilterTestCase(JsonBasedTesting, TestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') @@ -955,30 +714,73 @@ def test_package_api_purl_filter_by_multiple_blank_purl(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(4, response.data.get('count')) - def test_package_api_get_package(self): - from minecode.models import PriorityResourceURI + def test_package_api_get_enhanced_package(self): + response = self.client.get(reverse('api:package-get-enhanced-package-data', args=[self.package3.uuid])) + result = response.data + expected = self.get_test_loc('api/enhanced_package.json') + self.check_expected_results(result, expected, fields_to_remove=['package_sets'], regen=False) - self.assertEqual(0, PriorityResourceURI.objects.all().count()) - response = self.client.get(f'/api/packages/get_package/?purl={self.purl1}') - self.assertEqual(0, PriorityResourceURI.objects.all().count()) - self.assertEqual(1, len(response.data)) - result = response.data[0] - self.assertEqual(result.get('type'), self.package_data1.get('type')) - self.assertEqual(result.get('namespace'), self.package_data1.get('namespace')) - self.assertEqual(result.get('name'), self.package_data1.get('name')) - self.assertEqual(result.get('version'), self.package_data1.get('version')) - self.assertEqual(result.get('download_url'), self.package_data1.get('download_url')) - self.assertEqual(result.get('extra_data'), self.package_data1.get('extra_data')) - - def test_package_api_get_package_does_not_exist_in_db(self): - from minecode.models import PriorityResourceURI - purl = 'pkg:maven/org.apache.twill/twill-core@0.12.0' - self.assertEqual(0, PriorityResourceURI.objects.all().count()) - response = self.client.get(f'/api/packages/get_package/?purl={purl}') - self.assertEqual(1, PriorityResourceURI.objects.all().count()) - self.assertEqual({}, response.data) - - def test_package_api_get_or_fetch_package(self): +class CollectApiTestCase(JsonBasedTesting, TestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') + + def setUp(self): + package_download_url = 'http://anotherexample.com' + self.package_data = { + 'type': 'maven', + 'namespace': 'sample', + 'name': 'Baz', + 'version': '90.12', + 'qualifiers': '', + 'subpath': '', + 'download_url': package_download_url, + 'filename': 'Baz.zip', + 'sha1': 'testsha1-3', + 'md5': 'testmd5-3', + 'size': 100, + } + self.package = Package.objects.create(**self.package_data) + self.package.refresh_from_db() + self.scannableuri = ScannableURI.objects.create( + package=self.package, + uri=package_download_url, + ) + self.scannableuri.scan_status = ScannableURI.SCAN_INDEXED + self.scan_uuid = uuid4() + self.scannableuri.scan_uuid = self.scan_uuid + self.scannableuri.scan_error = 'error' + self.scannableuri.index_error = 'error' + self.scan_request_date = timezone.now() + self.scannableuri.scan_request_date = self.scan_request_date + + package_download_url2 = 'http://somethingelse.org' + self.package_data2 = { + 'type': 'npm', + 'namespace': 'example', + 'name': 'bar', + 'version': '56.78', + 'qualifiers': '', + 'subpath': '', + 'download_url': package_download_url2, + 'filename': 'Bar.zip', + 'sha1': 'testsha1-2', + 'md5': 'testmd5-2', + 'size': 100, + } + self.package2 = Package.objects.create(**self.package_data2) + self.package2.refresh_from_db() + self.scannableuri2 = ScannableURI.objects.create( + package=self.package2, + uri=package_download_url2, + ) + self.scannableuri2.scan_status = ScannableURI.SCAN_INDEXED + self.scan_uuid2 = uuid4() + self.scannableuri2.scan_uuid = self.scan_uuid2 + self.scannableuri2.scan_error = 'error' + self.scannableuri2.index_error = 'error' + self.scan_request_date2 = timezone.now() + self.scannableuri2.scan_request_date = self.scan_request_date2 + + def test_package_live(self): purl_str = 'pkg:maven/org.apache.twill/twill-core@0.12.0' download_url = 'https://repo1.maven.org/maven2/org/apache/twill/twill-core/0.12.0/twill-core-0.12.0.jar' purl_sources_str = f'{purl_str}?classifier=sources' @@ -986,7 +788,7 @@ def test_package_api_get_or_fetch_package(self): self.assertEqual(0, Package.objects.filter(download_url=download_url).count()) self.assertEqual(0, Package.objects.filter(download_url=sources_download_url).count()) - response = self.client.get(f'/api/packages/get_or_fetch_package/?purl={purl_str}') + response = self.client.get(f'/api/collect/?purl={purl_str}') self.assertEqual(1, Package.objects.filter(download_url=download_url).count()) self.assertEqual(1, Package.objects.filter(download_url=sources_download_url).count()) expected = self.get_test_loc('api/twill-core-0.12.0.json') @@ -1003,13 +805,221 @@ def test_package_api_get_or_fetch_package(self): ] self.check_expected_results(result, expected, fields_to_remove=fields_to_remove, regen=False) + + def test_package_api_index_packages_endpoint(self): + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(0, priority_resource_uris_count) + packages = [ + {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, + {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, + {'purl':'pkg:bitbucket/example/example@1.0.0'}, + ] + data = { + 'packages': packages + } + response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") + self.assertEqual(2, response.data['queued_packages_count']) + expected_queued_packages = [ + 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', + 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0', + ] + self.assertEqual( + sorted(expected_queued_packages), + sorted(response.data['queued_packages']) + ) + self.assertEqual(0, response.data['unqueued_packages_count']) + self.assertEqual([], response.data['unqueued_packages']) + self.assertEqual(1, response.data['unsupported_packages_count']) + expected_unsupported_packages = [ + 'pkg:bitbucket/example/example@1.0.0' + ] + self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(2, priority_resource_uris_count) - def test_package_api_get_enhanced_package(self): - response = self.client.get(reverse('api:package-get-enhanced-package-data', args=[self.package3.uuid])) - result = response.data - expected = self.get_test_loc('api/enhanced_package.json') - self.check_expected_results(result, expected, fields_to_remove=['package_sets'], regen=False) + # Ensure that we don't add the same packages to the queue if they have + # not yet been processed + purls = [ + {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, + {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, + {'purl':'pkg:bitbucket/example/example@1.0.0'}, + ] + data = { + 'packages': purls + } + response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") + self.assertEqual(0, response.data['queued_packages_count']) + self.assertEqual([], response.data['queued_packages']) + self.assertEqual(2, response.data['unqueued_packages_count']) + expected_unqueued_packages = [ + 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', + 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0', + ] + self.assertEqual( + sorted(expected_unqueued_packages), + sorted(response.data['unqueued_packages']) + ) + self.assertEqual(1, response.data['unsupported_packages_count']) + expected_unsupported_packages = [ + 'pkg:bitbucket/example/example@1.0.0' + ] + self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) + + @mock.patch("packagedb.api.get_all_versions") + def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versions): + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(0, priority_resource_uris_count) + packages = [ + { + "purl": "pkg:maven/ch.qos.reload4j/reload4j", + "vers": "vers:maven/>=1.2.18.2|<=1.2.23", + }, + ] + data = {"packages": packages} + + mock_get_all_versions.return_value = [ + MavenVersion("1.2.18.0"), + MavenVersion("1.2.18.1"), + MavenVersion("1.2.18.2"), + MavenVersion("1.2.18.3"), + MavenVersion("1.2.18.4"), + MavenVersion("1.2.18.5"), + MavenVersion("1.2.19"), + MavenVersion("1.2.20"), + MavenVersion("1.2.21"), + MavenVersion("1.2.22"), + MavenVersion("1.2.23"), + MavenVersion("1.2.24"), + MavenVersion("1.2.25"), + ] + + response = self.client.post( + "/api/collect/index_packages/", data=data, content_type="application/json" + ) + self.assertEqual(9, response.data["queued_packages_count"]) + + expected_queued_packages = [ + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.19", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.20", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.21", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.22", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", + ] + self.assertEqual( + sorted(expected_queued_packages), sorted(response.data["queued_packages"]) + ) + self.assertEqual(0, response.data["unqueued_packages_count"]) + self.assertEqual([], response.data["unqueued_packages"]) + self.assertEqual(0, response.data["unsupported_packages_count"]) + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(9, priority_resource_uris_count) + + @mock.patch("packagedb.api.get_all_versions") + def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_all_versions): + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(0, priority_resource_uris_count) + packages = [ + { + "purl": "pkg:maven/ch.qos.reload4j/reload4j", + }, + ] + data = {"packages": packages} + + mock_get_all_versions.return_value = [ + MavenVersion("1.2.18.0"), + MavenVersion("1.2.18.1"), + MavenVersion("1.2.18.2"), + MavenVersion("1.2.18.3"), + MavenVersion("1.2.18.4"), + MavenVersion("1.2.18.5"), + MavenVersion("1.2.19"), + MavenVersion("1.2.20"), + MavenVersion("1.2.21"), + MavenVersion("1.2.22"), + MavenVersion("1.2.23"), + MavenVersion("1.2.24"), + MavenVersion("1.2.25"), + ] + + response = self.client.post( + "/api/collect/index_packages/", data=data, content_type="application/json" + ) + self.assertEqual(13, response.data["queued_packages_count"]) + expected_queued_packages = [ + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.0", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.1", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.19", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.20", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.21", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.22", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.24", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.25", + ] + self.assertEqual( + sorted(expected_queued_packages), sorted(response.data["queued_packages"]) + ) + self.assertEqual(0, response.data["unqueued_packages_count"]) + self.assertEqual([], response.data["unqueued_packages"]) + self.assertEqual(0, response.data["unsupported_packages_count"]) + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(13, priority_resource_uris_count) + + def test_reindex_packages_bulk(self): + self.assertEqual(False, self.scannableuri.rescan_uri) + self.assertEqual(0, self.scannableuri.priority) + self.assertEqual(self.scan_uuid, self.scannableuri.scan_uuid) + self.assertEqual('error', self.scannableuri.scan_error) + self.assertEqual('error', self.scannableuri.index_error) + self.assertEqual(self.scan_request_date, self.scannableuri.scan_request_date) + + self.assertEqual(False, self.scannableuri2.rescan_uri) + self.assertEqual(0, self.scannableuri2.priority) + self.assertEqual(self.scan_uuid2, self.scannableuri2.scan_uuid) + self.assertEqual('error', self.scannableuri2.scan_error) + self.assertEqual('error', self.scannableuri2.index_error) + self.assertEqual(self.scan_request_date2, self.scannableuri2.scan_request_date) + existing_purls = [ + 'pkg:maven/sample/Baz@90.12', + 'pkg:npm/example/bar@56.78', + ] + nonexistent_purls = [ + 'pkg:pypi/does/not-exist@1', + ] + data = { + 'package_urls': existing_purls + nonexistent_purls, + } + response = self.client.post(f'/api/collect/reindex_packages/', data=data, content_type="application/json") + self.assertEqual(2, response.data['requeued_packages_count']) + self.assertEqual(existing_purls, response.data['requeued_packages']) + self.assertEqual(1, response.data['nonexistent_packages_count']) + self.assertEqual(nonexistent_purls, response.data['nonexistent_packages']) + + self.scannableuri.refresh_from_db() + self.assertEqual(True, self.scannableuri.rescan_uri) + self.assertEqual(100, self.scannableuri.priority) + self.assertEqual(None, self.scannableuri.scan_uuid) + self.assertEqual(None, self.scannableuri.scan_error) + self.assertEqual(None, self.scannableuri.index_error) + self.assertEqual(None, self.scannableuri.scan_request_date) + + self.scannableuri2.refresh_from_db() + self.assertEqual(True, self.scannableuri2.rescan_uri) + self.assertEqual(100, self.scannableuri.priority) + self.assertEqual(None, self.scannableuri2.scan_uuid) + self.assertEqual(None, self.scannableuri2.scan_error) + self.assertEqual(None, self.scannableuri2.index_error) + self.assertEqual(None, self.scannableuri2.scan_request_date) + class ResourceApiTestCase(TestCase): From 63874c9fde1daa4b291c14d0693cf86f71a908de Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 6 Dec 2023 04:12:04 +0530 Subject: [PATCH 3/4] Merge `reindex_packages` endpoint to `index_packages` Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 154 +++++++++++++++++++++-------------------------- 1 file changed, 68 insertions(+), 86 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index fa468a56..08315777 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -509,6 +509,7 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet): queryset = PackageSet.objects.prefetch_related('packages') serializer_class = PackageSetAPISerializer + class CollectViewSet(viewsets.ViewSet): """ Return Package data for the purl passed in the `purl` query parameter. @@ -559,10 +560,15 @@ def list(self, request, format=None): @action(detail=False, methods=['post']) def index_packages(self, request, *args, **kwargs): """ - Take a list of `packages` where each item is a dictionary containing either PURL - or versionless PURL along with vers range. + Take a list of `packages` (where each item is a dictionary containing either PURL + or versionless PURL along with vers range) and index it. + + If `reindex` flag is True then existing package will be rescanned, if `reindex_set` + is True then all the package in the same set will be rescanned. + If reindex flag is set to true then all the non existing package will be indexed. + **Note:** When a versionless PURL is supplied without a vers range, then all the versions - of that package will be indexed. + of that package will be considered for indexing/reindexing. **Input example:** @@ -581,53 +587,91 @@ def index_packages(self, request, *args, **kwargs): "vers": null } ] + "reindex": true, + "reindex_set": false, } Then return a mapping containing: - queued_packages_count - - The number of package urls placed on the queue. + - The number of package urls placed on the index queue. - queued_packages - - A list of package urls that were placed on the queue. + - A list of package urls that were placed on the index queue. + - requeued_packages_count + - The number of existing package urls placed on the rescan queue. + - requeued_packages + - A list of existing package urls that were placed on the rescan queue. - unqueued_packages_count - - The number of package urls not placed on the queue. This is - because the package url already exists on the queue and has not - yet been processed. + - The number of package urls not placed on the index queue. + This is because the package url already exists on the index queue and has not + yet been processed. - unqueued_packages - - A list of package urls that were not placed on the queue. + - A list of package urls that were not placed on the index queue. - unsupported_packages_count - - The number of package urls that are not processable by the queue. + - The number of package urls that are not processable by the index queue. - unsupported_packages - - A list of package urls that are not processable by the queue. The - package indexing queue can only handle npm and maven purls. - - unqueued_packages - - A list of package urls that were not placed on the queue. + - A list of package urls that are not processable by the index queue. + The package indexing queue can only handle npm and maven purls. - unsupported_vers_count - The number of vers range that are not supported by the univers or package_manager. - unsupported_vers - A list of vers range that are not supported by the univers or package_manager. """ + def _reindex_package(package, reindexed_packages): + if package in reindexed_packages: + return + package.rescan() + reindexed_packages.append(package) + packages = request.data.get('packages') or [] + reindex = request.data.get('reindex') or False + reindex_set = request.data.get('reindex_set') or False + queued_packages = [] unqueued_packages = [] + + nonexistent_packages = [] + reindexed_packages = [] + requeued_packages = [] + supported_ecosystems = ['maven', 'npm'] unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems) - for purl in unique_purls: - is_routable_purl = priority_router.is_routable(purl) - if not is_routable_purl: - unsupported_packages.append(purl) - else: - # add to queue - priority_resource_uri = PriorityResourceURI.objects.insert(purl) - if priority_resource_uri: - queued_packages.append(purl) + if reindex: + for purl in unique_purls: + lookups = purl_to_lookups(purl) + packages = Package.objects.filter(**lookups) + if packages.count() > 0: + for package in packages: + _reindex_package(package, reindexed_packages) + if reindex_set: + for package_set in package.package_sets.all(): + for p in package_set.packages.all(): + _reindex_package(p, reindexed_packages) + else: + nonexistent_packages.append(purl) + requeued_packages.extend([p.package_url for p in reindexed_packages]) + + elif not reindex or nonexistent_packages: + interesting_purls = nonexistent_packages if nonexistent_packages else unique_purls + for purl in interesting_purls: + is_routable_purl = priority_router.is_routable(purl) + if not is_routable_purl: + unsupported_packages.append(purl) else: - unqueued_packages.append(purl) + # add to queue + priority_resource_uri = PriorityResourceURI.objects.insert(purl) + if priority_resource_uri: + queued_packages.append(purl) + else: + unqueued_packages.append(purl) + response_data = { 'queued_packages_count': len(queued_packages), 'queued_packages': queued_packages, + 'requeued_packages_count': len(requeued_packages), + 'requeued_packages': requeued_packages, 'unqueued_packages_count': len(unqueued_packages), 'unqueued_packages': unqueued_packages, 'unsupported_packages_count': len(unsupported_packages), @@ -637,68 +681,6 @@ def index_packages(self, request, *args, **kwargs): } return Response(response_data) - @action(detail=False, methods=['post']) - def reindex_packages(self, request, *args, **kwargs): - """ - Take a list of `package_urls` and for each Package URL, reindex the - corresponding package. - If the field `reindex_set` is True, then the Packages in the same - package set as the packages from `package_urls` will be reindexed. - - **Input example:** - - { - "package_urls": [ - "pkg:npm/foobar@1.1.0", - ], - "reindex_set": true, - } - - Then return a mapping containing: - - - requeued_packages_count - - The number of package urls placed on the queue. - - requeued_packages - - A list of package urls that were placed on the queue. - - nonexistent_packages_count - - The number of package urls that do not correspond to a package in - the database. - - nonexistent_packages - - A list of package urls that do not correspond to a package in the - database. - """ - def _reindex_package(package, reindexed_packages): - if package in reindexed_packages: - return - package.rescan() - reindexed_packages.append(package) - - purls = request.data.get('package_urls') or [] - reindex_set = request.data.get('reindex_set') or False - - nonexistent_packages = [] - reindexed_packages = [] - for purl in purls: - lookups = purl_to_lookups(purl) - packages = Package.objects.filter(**lookups) - if packages.count() > 0: - for package in packages: - _reindex_package(package, reindexed_packages) - if reindex_set: - for package_set in package.package_sets.all(): - for p in package_set.packages.all(): - _reindex_package(p, reindexed_packages) - else: - nonexistent_packages.append(purl) - - requeued_packages = [p.package_url for p in reindexed_packages] - response_data = { - 'requeued_packages_count': len(requeued_packages), - 'requeued_packages': requeued_packages, - 'nonexistent_packages_count': len(nonexistent_packages), - 'nonexistent_packages': nonexistent_packages, - } - return Response(response_data) def get_resolved_purls(packages, supported_ecosystems): """ From 59b74b84777d8abd472d89573e21300f0570ead5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 6 Dec 2023 04:12:50 +0530 Subject: [PATCH 4/4] Test merged `index_packages` endpoint Signed-off-by: Keshav Priyadarshi --- packagedb/tests/test_api.py | 46 ++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 6f6b1b20..32002cc7 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -850,6 +850,8 @@ def test_package_api_index_packages_endpoint(self): response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") self.assertEqual(0, response.data['queued_packages_count']) self.assertEqual([], response.data['queued_packages']) + self.assertEqual(0, response.data['requeued_packages_count']) + self.assertEqual([], response.data['requeued_packages']) self.assertEqual(2, response.data['unqueued_packages_count']) expected_unqueued_packages = [ 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', @@ -912,6 +914,8 @@ def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versio self.assertEqual( sorted(expected_queued_packages), sorted(response.data["queued_packages"]) ) + self.assertEqual(0, response.data['requeued_packages_count']) + self.assertEqual([], response.data['requeued_packages']) self.assertEqual(0, response.data["unqueued_packages_count"]) self.assertEqual([], response.data["unqueued_packages"]) self.assertEqual(0, response.data["unsupported_packages_count"]) @@ -968,9 +972,12 @@ def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_al self.assertEqual( sorted(expected_queued_packages), sorted(response.data["queued_packages"]) ) + self.assertEqual(0, response.data['requeued_packages_count']) + self.assertEqual([], response.data['requeued_packages']) self.assertEqual(0, response.data["unqueued_packages_count"]) self.assertEqual([], response.data["unqueued_packages"]) self.assertEqual(0, response.data["unsupported_packages_count"]) + self.assertEqual([], response.data["unsupported_packages"]) priority_resource_uris_count = PriorityResourceURI.objects.all().count() self.assertEqual(13, priority_resource_uris_count) @@ -988,21 +995,44 @@ def test_reindex_packages_bulk(self): self.assertEqual('error', self.scannableuri2.scan_error) self.assertEqual('error', self.scannableuri2.index_error) self.assertEqual(self.scan_request_date2, self.scannableuri2.scan_request_date) + + packages = [ + # Existing package + { + "purl": 'pkg:maven/sample/Baz@90.12', + }, + { + "purl": 'pkg:npm/example/bar@56.78', + }, + # NOt in DB and unsupported + { + "purl":'pkg:pypi/does/not-exist@1', + }, + ] + data = {"packages": packages, "reindex":True} + existing_purls = [ 'pkg:maven/sample/Baz@90.12', 'pkg:npm/example/bar@56.78', ] - nonexistent_purls = [ + + unsupported_purls = [ 'pkg:pypi/does/not-exist@1', ] - data = { - 'package_urls': existing_purls + nonexistent_purls, - } - response = self.client.post(f'/api/collect/reindex_packages/', data=data, content_type="application/json") + + response = self.client.post(f'/api/collect/index_packages/', data=data, content_type="application/json") + self.assertEqual(2, response.data['requeued_packages_count']) - self.assertEqual(existing_purls, response.data['requeued_packages']) - self.assertEqual(1, response.data['nonexistent_packages_count']) - self.assertEqual(nonexistent_purls, response.data['nonexistent_packages']) + self.assertListEqual(sorted(existing_purls), sorted(response.data['requeued_packages'])) + + self.assertEqual(1, response.data['unsupported_packages_count']) + self.assertListEqual(unsupported_purls, response.data['unsupported_packages']) + + self.assertEqual(0, response.data['queued_packages_count']) + self.assertEqual([], response.data['queued_packages']) + self.assertEqual(0, response.data["unqueued_packages_count"]) + self.assertEqual([], response.data["unqueued_packages"]) + self.scannableuri.refresh_from_db() self.assertEqual(True, self.scannableuri.rescan_uri)