Add API Documentation

CHANGELOG.rst

@@ -1,10 +1,10 @@
 Changelog
 =========
 
-next-version
+v4.0.0
 ------------
 
-- TBD
+- Add `/api/docs` Swagger API documentation for API endpoints.
 
 v3.0.0
 -------

packagedb/api.py

@@ -17,12 +17,26 @@
 from django_filters.filters import Filter
 from django_filters.filters import OrderingFilter
 from django_filters.rest_framework import FilterSet
+from drf_spectacular.utils import OpenApiParameter
+from drf_spectacular.utils import extend_schema
+from packageurl import PackageURL
+from packageurl.contrib.django.utils import purl_to_lookups
+from rest_framework import mixins
+from rest_framework import status
+from rest_framework import viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.throttling import AnonRateThrottle
+from univers.version_constraint import InvalidConstraintsError
+from univers.version_range import RANGE_CLASS_BY_SCHEMES
+from univers.version_range import VersionRange
+from univers.versions import InvalidVersion
+
 from matchcode.api import MultipleCharFilter
 from matchcode.api import MultipleCharInFilter
-
-from minecode import priority_router
 # UnusedImport here!
 # But importing the mappers and visitors module triggers routes registration
+from minecode import priority_router
 from minecode import visitors  # NOQA
 from minecode.models import PriorityResourceURI
 from minecode.models import ScannableURI
@@ -37,6 +51,8 @@
 from packagedb.package_managers import get_api_package_name
 from packagedb.package_managers import get_version_fetcher
 from packagedb.serializers import DependentPackageSerializer
+from packagedb.serializers import IndexPackagesResponseSerializer
+from packagedb.serializers import IndexPackagesSerializer
 from packagedb.serializers import PackageAPISerializer
 from packagedb.serializers import PackageSetAPISerializer
 from packagedb.serializers import PackageWatchAPISerializer
@@ -47,23 +63,11 @@
 from packagedb.serializers import PurlValidateSerializer
 from packagedb.serializers import ResourceAPISerializer
 from packagedb.throttling import StaffUserRateThrottle
-from packageurl import PackageURL
-from packageurl.contrib.django.utils import purl_to_lookups
-from rest_framework import mixins
-from rest_framework import status
-from rest_framework import viewsets
-from rest_framework.decorators import action
-from rest_framework.response import Response
-from rest_framework.throttling import AnonRateThrottle
-from univers.version_constraint import InvalidConstraintsError
-from univers.version_range import RANGE_CLASS_BY_SCHEMES
-from univers.version_range import VersionRange
-from univers.versions import InvalidVersion
 
 logger = logging.getLogger(__name__)
 
 
-class CreateListRetrieveUpdateViewSet(
+class CreateListRetrieveUpdateViewSetMixin(
     mixins.CreateModelMixin,
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
@@ -550,7 +554,12 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet):
     serializer_class = PackageSetAPISerializer
 
 
-class PackageWatchViewSet(CreateListRetrieveUpdateViewSet):
+class PackageWatchViewSet(CreateListRetrieveUpdateViewSetMixin):
+    """
+    Take a `purl` and periodically watch for the new version of the package.
+    Add the new package version to the scan queue. 
+    Default watch interval is 7 days.
+    """
     queryset = PackageWatch.objects.get_queryset().order_by('-id')
     serializer_class = PackageWatchAPISerializer
     lookup_field = 'package_url'
@@ -570,11 +579,14 @@ class CollectViewSet(viewsets.ViewSet):
 
     If the package does not exist, we will fetch the Package data and return
     it in the same request.
-
-    **Note:** Use `Index packages` for bulk indexing of packages; use `Reindex packages`
-    for bulk reindexing of existing packages.
+
+    **Note:** Use `Index packages` for bulk indexing/reindexing of packages.
     """
-
+    serializer_class=None
+    @extend_schema(
+            parameters=[OpenApiParameter('purl', str, 'query', description='PackageURL')],
+            responses={200:PackageAPISerializer()},
+    )
     def list(self, request, format=None):
         purl = request.query_params.get('purl')
 
@@ -604,14 +616,20 @@ def list(self, request, format=None):
                 message = {}
                 if errors:
                     message = {
-                        'status': f'error(s) occured when fetching metadata for {purl}: {errors}'
+                        'status': f'error(s) occurred when fetching metadata for {purl}: {errors}'
                     }
                 return Response(message)
 
         serializer = PackageAPISerializer(packages, many=True, context={'request': request})
         return Response(serializer.data)
-
-    @action(detail=False, methods=['post'])
+
+    @extend_schema(
+        request=IndexPackagesSerializer,
+        responses={
+            200: IndexPackagesResponseSerializer(),
+        },
+    )
+    @action(detail=False, methods=['post'], serializer_class=IndexPackagesSerializer)
     def index_packages(self, request, *args, **kwargs):
         """
         Take a list of `packages` (where each item is a dictionary containing either PURL
@@ -624,7 +642,7 @@ def index_packages(self, request, *args, **kwargs):
         **Note:** When a versionless PURL is supplied without a vers range, then all the versions
         of that package will be considered for indexing/reindexing.
 
-        **Input example:**
+        **Request example:**
 
                 {
                     "packages": [
@@ -676,10 +694,16 @@ def _reindex_package(package, reindexed_packages):
                 return
             package.rescan()
             reindexed_packages.append(package)
+
+        serializer = self.serializer_class(data=request.data)
 
-        packages = request.data.get('packages') or []
-        reindex = request.data.get('reindex') or False
-        reindex_set = request.data.get('reindex_set') or False
+        if not serializer.is_valid():
+            return Response({'errors': serializer.errors}, status=400)
+
+        validated_data = serializer.validated_data
+        packages = validated_data.get('packages', [])
+        reindex = validated_data.get('reindex', False)
+        reindex_set = validated_data.get('reindex_set', False)
 
         queued_packages = []
         unqueued_packages = []
@@ -733,7 +757,9 @@ def _reindex_package(package, reindexed_packages):
             'unsupported_vers_count': len(unsupported_vers),
             'unsupported_vers': unsupported_vers,
         }
-        return Response(response_data)
+
+        serializer = IndexPackagesResponseSerializer(response_data, context={'request': request})
+        return Response(serializer.data)
 
 
 class PurlValidateViewSet(viewsets.ViewSet):
@@ -745,7 +771,7 @@ class PurlValidateViewSet(viewsets.ViewSet):
     `gem`, `golang`, `hex`, `maven`, `npm`, `nuget` and `pypi` ecosystems.
 
     **Example request:**
-            ```doc
+            ```
             GET /api/validate/?purl=pkg:npm/foobar@12.3.1&check_existence=false
             ```
 
@@ -761,6 +787,13 @@ class PurlValidateViewSet(viewsets.ViewSet):
     def get_view_name(self):
         return 'Validate PURL'
 
+    @extend_schema(
+        parameters=[
+            OpenApiParameter('purl', str, 'query', description='PackageURL'),
+            OpenApiParameter('check_existence', bool, 'query', description='Check existence', default=False),
+        ],
+        responses={200: PurlValidateResponseSerializer()},
+    )
     def list(self, request):
         serializer = self.serializer_class(data=request.query_params)
 

packagedb/serializers.py

@@ -20,7 +20,9 @@
 from rest_framework.serializers import HyperlinkedIdentityField
 from rest_framework.serializers import HyperlinkedModelSerializer
 from rest_framework.serializers import HyperlinkedRelatedField
+from rest_framework.serializers import IntegerField
 from rest_framework.serializers import JSONField
+from rest_framework.serializers import ListField
 from rest_framework.serializers import ModelSerializer
 from rest_framework.serializers import Serializer
 from rest_framework.serializers import SerializerMethodField
@@ -366,6 +368,43 @@ class Meta:
         fields = ['depth', 'watch_interval', 'is_active']
 
 
+class PackageVersSerializer(Serializer):
+    purl = CharField()
+    vers = CharField(required=False)
+
+
+class IndexPackagesSerializer(Serializer):
+    packages = PackageVersSerializer(many=True)
+    reindex = BooleanField(default=False)
+    reindex_set = BooleanField(default=False)
+
+
+class IndexPackagesResponseSerializer(Serializer):
+    queued_packages_count = IntegerField(help_text="Number of package urls placed on the index queue.")
+    queued_packages = ListField(
+        child=CharField(),
+        help_text="List of package urls that were placed on the index queue."
+    )
+    requeued_packages_count = IntegerField(help_text="Number of existing package urls placed on the rescan queue.")
+    requeued_packages = ListField(
+        child=CharField(),
+        help_text="List of existing package urls that were placed on the rescan queue."
+    )
+    unqueued_packages_count = IntegerField(help_text="Number of package urls not placed on the index queue.")
+    unqueued_packages = ListField(
+        child=CharField(),
+        help_text="List of package urls that were not placed on the index queue."
+    )
+    unsupported_packages_count = IntegerField(help_text="Number of package urls that are not processable by the index queue.")
+    unsupported_packages = ListField(
+        child=CharField(),
+        help_text="List of package urls that are not processable by the index queue."
+    )
+    unsupported_vers_count = IntegerField(help_text="Number of vers range that are not supported by the univers or package_manager.")
+    unsupported_vers = ListField(
+        child=CharField(),
+        help_text="List of vers range that are not supported by the univers or package_manager."
+    )
 class PurlValidateResponseSerializer(Serializer):
     valid = BooleanField()
     exists = BooleanField(required=False)
@@ -374,4 +413,4 @@ class PurlValidateResponseSerializer(Serializer):
 
 class PurlValidateSerializer(Serializer):
     purl = CharField(required=True)
-    check_existence = BooleanField(required=False, default=False)
+    check_existence = BooleanField(required=False, default=False)

packagedb/tests/test_api.py

@@ -1096,7 +1096,7 @@ def test_api_resource_checksum_filter(self):
             self.resource1.name,
             self.resource2.name,
         ])
-        self.assertEquals(expected_names, names)
+        self.assertEqual(expected_names, names)
 
         filters = f'?sha1={self.resource1.sha1}&sha1={self.resource2.sha1}'
         response = self.client.get(f'/api/resources/{filters}')
@@ -1106,7 +1106,7 @@ def test_api_resource_checksum_filter(self):
             self.resource1.name,
             self.resource2.name,
         ])
-        self.assertEquals(expected_names, names)
+        self.assertEqual(expected_names, names)
 
 class PurlValidateApiTestCase(TestCase):
 
@@ -1140,15 +1140,15 @@ def test_api_purl_validation(self):
         }
         response2 = self.client.get(f"/api/validate/", data=data2)
 
-        self.assertEquals(True, response1.data["valid"])
-        self.assertEquals(True, response1.data["exists"])
-        self.assertEquals(
+        self.assertEqual(True, response1.data["valid"])
+        self.assertEqual(True, response1.data["exists"])
+        self.assertEqual(
             "The provided Package URL is valid, and the package exists in the upstream repo.",
             response1.data["message"],
         )
 
-        self.assertEquals(False, response2.data["valid"])
-        self.assertEquals(
+        self.assertEqual(False, response2.data["valid"])
+        self.assertEqual(
             "The provided PackageURL is not valid.", response2.data["message"]
         )
 
@@ -1160,11 +1160,11 @@ def test_api_purl_validation_unsupported_package_type(self):
         response1 = self.client.get(f"/api/validate/", data=data1)
 
 
-        self.assertEquals(True, response1.data["valid"])
-        self.assertEquals(
+        self.assertEqual(True, response1.data["valid"])
+        self.assertEqual(
             "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", response1.data["message"]
         )
-        self.assertEquals(None, response1.data["exists"])
+        self.assertEqual(None, response1.data["exists"])
 
     def test_api_purl_validation_empty_request(self):
         data1 = {}

purldb_project/__init__.py

@@ -10,6 +10,8 @@
 import os
 import sys
 
+__version__ = "3.0.0"
+
 
 def command_line():
     '''Command line entry point.'''

purldb_project/settings.py

@@ -11,7 +11,9 @@
 from pathlib import Path
 
 import environ
+from purldb_project import __version__
 
+PURLDB_VERSION = __version__
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent
@@ -73,6 +75,7 @@
     # Third-party apps
     'django_filters',
     'rest_framework',
+    'drf_spectacular',
     'rest_framework.authtoken',
     'django_rq',
 )
@@ -258,6 +261,7 @@
     'DEFAULT_THROTTLE_RATES': REST_FRAMEWORK_DEFAULT_THROTTLE_RATES,
     'EXCEPTION_HANDLER': 'packagedb.throttling.throttled_exception_handler',
     'DEFAULT_PAGINATION_CLASS': 'packagedb.api_custom.PageSizePagination',
+    'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
     # Limit the load on the Database returning a small number of records by default. https://github.com/nexB/vulnerablecode/issues/819
     "PAGE_SIZE": 20,
 }
@@ -293,11 +297,17 @@
         "127.0.0.1",
     ]
 
-# Active seeders: each active seeder class need to be added explictly here
+# Active seeders: each active seeder class need to be added explicitly here
 ACTIVE_SEEDERS = [
     'minecode.visitors.maven.MavenSeed',
 ]
 
+SPECTACULAR_SETTINGS = {
+    'TITLE': 'PurlDB API',
+    'DESCRIPTION': 'Tools to create and expose a database of purls (Package URLs)',
+    'VERSION': PURLDB_VERSION,
+    'SERVE_INCLUDE_SCHEMA': False,
+}
 RQ_QUEUES = {
     'default': {
         "HOST": env.str("PURLDB_REDIS_HOST", default="localhost"),

purldb_project/urls.py

@@ -26,6 +26,8 @@
 from minecode.api import PriorityResourceURIViewSet
 from packagedb.api import PurlValidateViewSet
 from packagedb.api import CollectViewSet
+from drf_spectacular.views import SpectacularAPIView
+from drf_spectacular.views import SpectacularSwaggerView
 
 
 api_router = routers.DefaultRouter()
@@ -49,5 +51,7 @@
         TemplateView.as_view(template_name='robots.txt', content_type='text/plain'),
     ),
     path('api/', include((api_router.urls, 'api'))),
-    path('', RedirectView.as_view(url='api/')),
+    path("", RedirectView.as_view(url="api/")),
+    path('api/schema/', SpectacularAPIView.as_view(), name='schema'),
+    path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'),
 ]

setup.cfg

@@ -44,6 +44,7 @@ install_requires =
     django-rq == 2.10.1
     djangorestframework == 3.14.0
     django-filter == 23.5
+    drf-spectacular == 0.26.5
     fetchcode == 0.3.0
     gunicorn == 21.2.0
     ftputil == 5.0.4