Merge branch 'main' into license-detection-models

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout code
@@ -44,6 +44,9 @@ jobs:
 
       - name: Install universal ctags
         run: sudo apt-get install -y universal-ctags
+
+      - name: Install xgettext
+        run: sudo apt-get install -y gettext
 
       - name: Install dependencies
         run: make dev envfile

.github/workflows/pypi-release.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Install pypa/build
         run: python -m pip install build --user

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 *.py[cod]
 
 *.db
+*.sqlite3
 .installed.cfg
 parts
 develop-eggs
@@ -46,7 +47,6 @@ local
 /.python-version
 /.pytest_cache/
 /scancodeio.egg-info/
-policies.yml
 *.rdb
 *.aof
 .vscode

CHANGELOG.rst

@@ -1,7 +1,253 @@
 Changelog
 =========
 
-v34.1.0 (unreleased)
+v34.7.1 (2024-07-15)
+--------------------
+
+- Add pipeline step selection for a run execution.
+  This allows to run a pipeline in an advanced mode allowing to skip some steps,
+  or restart from a step, like the last failed step.
+  The steps can be edited from the Run "status" modal using the "Select steps" button.
+  This is an advanced feature and should we used with caution.
+  https://github.com/nexB/scancode.io/issues/1303
+
+- Display the resolved_to_package as link in the dependencies tab.
+  https://github.com/nexB/scancode.io/pull/1314
+
+- Add support for multiple instances of a PackageURL in the CycloneDX outputs.
+  The `package_uid` is now included in each BOM Component as a property.
+  https://github.com/nexB/scancode.io/issues/1316
+
+- Add administration interface. Can be enabled with the SCANCODEIO_ENABLE_ADMIN_SITE
+  setting.
+  Add ``--admin`` and ``--super`` options to the ``create-user`` management command.
+  https://github.com/nexB/scancode.io/pull/1323
+
+- Add ``results_url`` and ``summary_url`` on the API ProjectSerializer.
+  https://github.com/nexB/scancode.io/issues/1325
+
+v34.7.0 (2024-07-02)
+--------------------
+
+- Add all "classify" plugin fields from scancode-toolkit on the CodebaseResource model.
+  https://github.com/nexB/scancode.io/issues/1275
+
+- Refine the extraction errors reporting to include the resource path for rendering
+  link to the related resources in the UI.
+  https://github.com/nexB/scancode.io/issues/1273
+
+- Add a ``flush-projects`` management command, to Delete all project data and their
+  related work directories created more than a specified number of days ago.
+  https://github.com/nexB/scancode.io/issues/1289
+
+- Update the ``inspect_packages`` pipeline to have an optional ``StaticResolver``
+  group to create resolved packages and dependency relationships from lockfiles
+  and manifests having pre-resolved dependencies. Also update this pipeline to
+  perform package assembly from multiple manifests and files to create
+  discovered packages. Also update the ``resolve_dependencies`` pipeline to have
+  the same ``StaticResolver`` group and mode the dynamic resolution part to a new
+  optional ``DynamicResolver`` group.
+  See https://github.com/nexB/scancode.io/pull/1244
+
+- Add a new attribute ``is_direct`` to the DiscoveredDependency model and two new
+  attributes ``is_private`` and ``is_virtual`` to the DiscoveredPackage model.
+  Also update the UIs to show these attributes and show the ``package_data`` field
+  contents for CodebaseResources in the ``extra_data`` tab.
+  See https://github.com/nexB/scancode.io/pull/1244
+
+- Update scancode-toolkit to version ``32.2.1``. For the complete list of updates
+  and improvements see https://github.com/nexB/scancode-toolkit/releases/tag/v32.2.0
+  and https://github.com/nexB/scancode-toolkit/releases/tag/v32.2.1
+
+- Add support for providing pipeline "selected_groups" in the ``run`` entry point.
+  https://github.com/nexB/scancode.io/issues/1306
+
+v34.6.3 (2024-06-21)
+--------------------
+
+- Use the ``--option=value`` syntax for args entries in place of ``--option value``
+  for fetching Docker images using skopeo through ``run_command_safely`` calls.
+  https://github.com/nexB/scancode.io/issues/1257
+
+- Fix an issue in the d2d JavaScript mapper.
+  https://github.com/nexB/scancode.io/pull/1274
+
+- Add support for a ``ignored_vulnerabilities`` field on the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1271
+
+v34.6.2 (2024-06-18)
+--------------------
+
+- Store SBOMs headers in the `Project.extra_data` field during the load_sboms
+  pipeline.
+  https://github.com/nexB/scancode.io/issues/1253
+
+- Add support for fetching Git repository as Project input.
+  https://github.com/nexB/scancode.io/issues/921
+
+- Enhance the logging and reporting of input fetch exceptions.
+  https://github.com/nexB/scancode.io/issues/1257
+
+v34.6.1 (2024-06-07)
+--------------------
+
+- Remove print statements from migration files.
+- Display full traceback on error in the ``execute`` management command.
+- Log the Project message creation.
+- Refactor the ``get_env_from_config_file`` to support empty config file.
+
+v34.6.0 (2024-06-07)
+--------------------
+
+- Add a new ``scan_for_virus`` add-on pipeline based on ClamAV scan.
+  Found viruses are stored as "error" Project messages and on their related codebase
+  resource instance using the ``extra_data`` field.
+  https://github.com/nexB/scancode.io/issues/1182
+
+- Add ability to filter by tag on the resource list view.
+  https://github.com/nexB/scancode.io/issues/1217
+
+- Use "unknown" as the Package URL default type when no values are provided for that
+  field. This allows to create a discovered package instance instead of raising a
+  Project error message.
+  https://github.com/nexB/scancode.io/issues/1249
+
+- Rename DiscoveredDependency ``resolved_to`` to ``resolved_to_package``, and
+  ``resolved_dependencies`` to ``resolved_from_dependencies`` for clarity and
+  consistency.
+  Add ``children_packages`` and ``parent_packages`` ManyToMany field on the
+  DiscoveredPackage model.
+  Add full dependency tree in the CycloneDX output.
+  https://github.com/nexB/scancode.io/issues/1066
+
+- Add a new ``run`` entry point for executing pipeline as a single command.
+  https://github.com/nexB/scancode.io/pull/1256
+
+- Generate a DiscoveredPackage.package_uid in create_from_data when not provided.
+  https://github.com/nexB/scancode.io/issues/1256
+
+v34.5.0 (2024-05-22)
+--------------------
+
+- Display the current path location in the "Codebase" panel as a navigation breadcrumbs.
+  https://github.com/nexB/scancode.io/issues/1158
+
+- Fix a rendering issue in the dependency details view when for_package or
+  datafile_resource fields do not have a value.
+  https://github.com/nexB/scancode.io/issues/1177
+
+- Add a new `CollectPygmentsSymbolsAndStrings` pipeline (addon) for collecting source
+  symbol, string and comments using Pygments.
+  https://github.com/nexB/scancode.io/pull/1179
+
+- Workaround an issue with the cyclonedx-python-lib that does not allow to load
+  SBOMs that contains properties with no values.
+  Also, a few fixes pre-validation are applied before deserializing thr SBOM for
+  maximum compatibility.
+  https://github.com/nexB/scancode.io/issues/1185
+  https://github.com/nexB/scancode.io/issues/1230
+
+- Add a new `CollectTreeSitterSymbolsAndStrings` pipeline (addon) for collecting source
+  symbol and string using tree-sitter.
+  https://github.com/nexB/scancode.io/pull/1181
+
+- Fix `inspect_packages` pipeline to properly link discovered packages and dependencies to
+  codebase resources of package manifests where they were found. Also correctly assign
+  the datasource_ids attribute for packages and dependencies.
+  https://github.com/nexB/scancode.io/pull/1180
+
+- Add "Product name" and "Product version" as new project settings.
+  https://github.com/nexB/scancode.io/issues/1197
+
+- Add "Product name" and "Product version" as new project settings.
+  https://github.com/nexB/scancode.io/issues/1197
+
+- Raise the minimum RAM required per CPU code in the docs.
+  A good rule of thumb is to allow **2 GB of memory per CPU**.
+  For example, if Docker is configured for 8 CPUs, a minimum of 16 GB of memory is
+  required.
+  https://github.com/nexB/scancode.io/issues/1191
+
+- Add value validation for the search complex query syntax.
+  https://github.com/nexB/scancode.io/issues/1183
+
+- Bump matchcode-toolkit version to v5.0.0.
+
+- Fix the content of the ``package_url`` field in CycloneDX outputs.
+  https://github.com/nexB/scancode.io/issues/1224
+
+- Enhance support for encoded ``package_url`` during the conversion to model fields.
+  https://github.com/nexB/scancode.io/issues/1171
+
+- Remove the ``scancode_license_score`` option from the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1231
+
+- Remove the ``extract_recursively`` option from the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1236
+
+- Add support for a ``ignored_dependency_scopes`` field on the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1197
+
+- Add support for storing the scancode-config.yml file in codebase.
+  The scancode-config.yml file can be provided as a project input, or can be located
+  in the codebase/ immediate subdirectories. This allows to provide the configuration
+  file as part of an input archive or a git clone for example.
+  https://github.com/nexB/scancode.io/issues/1236
+
+- Provide a downloadable YAML scancode-config.yml template in the documentation.
+  https://github.com/nexB/scancode.io/issues/1197
+
+- Add support for CycloneDX SBOM component properties as generated by external tools.
+  For example, the ``ResolvedUrl`` generated by cdxgen is now imported as the package
+  ``download_url``.
+
+v34.4.0 (2024-04-22)
+--------------------
+
+- Upgrade Gunicorn to v22.0.0 security release.
+
+- Display the list of fields available for the advanced search syntax in the modal UI.
+  https://github.com/nexB/scancode.io/issues/1164
+
+- Add support for CycloneDX 1.6 outputs and inputs.
+  Also, the CycloneDX outputs can be downloaded as 1.6, 1.5, and 1.4 spec versions.
+  https://github.com/nexB/scancode.io/pull/1165
+
+- Update matchcode-toolkit to v4.1.0
+
+- Add a new function
+  `scanpipe.pipes.matchcode.fingerprint_codebase_resources()`, which computes
+  approximate file matching fingerprints for text files using the new
+  `get_file_fingerprint_hashes` function from matchcode-toolkit.
+
+- Rename the `purldb-scan-queue-worker` management command to `purldb-scan-worker`.
+
+- Add `docker-compose.purldb-scan-worker.yml` to run ScanCode.io as a PurlDB
+  scan worker service.
+
+v34.3.0 (2024-04-10)
+--------------------
+
+- Associate resolved packages with their source codebase resource.
+  https://github.com/nexB/scancode.io/issues/1140
+
+- Add a new `CollectSourceStrings` pipeline (addon) for collecting source string using
+  xgettext.
+  https://github.com/nexB/scancode.io/pull/1160
+
+v34.2.0 (2024-03-28)
+--------------------
+
+- Add support for Python 3.12 and upgrade to Python 3.12 in the Dockerfile.
+  https://github.com/nexB/scancode.io/pull/1138
+
+- Add support for CycloneDX XML inputs.
+  https://github.com/nexB/scancode.io/issues/1136
+
+- Upgrade the SPDX schema to v2.3.1
+  https://github.com/nexB/scancode.io/issues/1130
+
+v34.1.0 (2024-03-27)
 --------------------
 
 - Add support for importing CycloneDX SBOM 1.2, 1.3, 1.4 and 1.5 spec formats.
@@ -39,6 +285,22 @@ v34.1.0 (unreleased)
   A data migration is included to facilitate the migration of existing data.
   https://github.com/nexB/scancode.io/issues/1099
 
+- Add PurlDB tab, displayed when the PURLDB_URL settings is configured.
+  When loading the package details view, a request is made on the PurlDB to fetch and
+  and display any available data.
+  https://github.com/nexB/scancode.io/issues/1125
+
+- Create a new management command `purldb-scan-queue-worker`, that runs
+  scancode.io as a Package scan queue worker for PurlDB.
+  `purldb-scan-queue-worker` gets the next available Package to be scanned and
+  the list of pipeline names to be run on the Package from PurlDB, creates a
+  Project, fetches the Package, runs the specified pipelines, and returns the
+  results to PurlDB.
+  https://github.com/nexB/scancode.io/pull/1078
+  https://github.com/nexB/purldb/issues/236
+
+- Update matchcode-toolkit to v4.0.0
+
 v34.0.0 (2024-03-04)
 --------------------
 

Dockerfile

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-FROM --platform=linux/amd64 python:3.11-slim
+FROM --platform=linux/amd64 python:3.12-slim
 
 LABEL org.opencontainers.image.source="https://github.com/nexB/scancode.io"
 LABEL org.opencontainers.image.description="ScanCode.io"
@@ -40,7 +40,7 @@ ENV PYTHONPATH $PYTHONPATH:$APP_DIR
 
 # OS requirements as per
 # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html
-# Also install universal-ctags for symbol collection.
+# Also install universal-ctags and xgettext for symbol and string collection.
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
        bzip2 \
@@ -60,6 +60,7 @@ RUN apt-get update \
        git \
        wait-for-it \
        universal-ctags \
+       gettext \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

Makefile

@@ -104,6 +104,11 @@ migrate:
 	@echo "-> Apply database migrations"
 	${MANAGE} migrate
 
+upgrade:
+	@echo "-> Upgrade local git checkout"
+	@git pull
+	@$(MAKE) migrate
+
 postgresdb:
 	@echo "-> Configure PostgreSQL database"
 	@echo "-> Create database user ${SCANCODEIO_DB_NAME}"
@@ -158,4 +163,4 @@ offline-package: docker-images
 	@mkdir -p dist/
 	@tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/
 
-.PHONY: virtualenv conf dev envfile install check bandit valid isort check-deploy clean migrate postgresdb sqlitedb backupdb run test docs bump docker-images offline-package
+.PHONY: virtualenv conf dev envfile install check bandit valid isort check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run test docs bump docker-images offline-package

docker-compose-offline.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   db:
     image: postgres:13

docker-compose.dev.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 # Mount the local scanpipe/ directory in the containers
 
 # This can be used to refresh fixtures from the docker container:

docker-compose.purldb-scan-worker.yml

@@ -0,0 +1,17 @@
+include:
+  - docker-compose.yml
+
+services:
+  purldb_scan_worker:
+    build: .
+    command: wait-for-it --strict --timeout=120 web:8000 -- sh -c "
+        ./manage.py purldb-scan-worker --async --sleep 3"
+    env_file:
+      - docker.env
+    volumes:
+      - .env:/opt/scancodeio/.env
+      - /etc/scancodeio/:/etc/scancodeio/
+      - workspace:/var/scancodeio/workspace/
+    depends_on:
+      - db
+      - web