Introduce output data format versioning #2653

docs/source/contribute/cut_new_release.rst

@@ -1,8 +1,17 @@
 How to cut a new release:
 =========================
 
+Update version
+--------------
+
 - Run bumpversion with major, minor or patch to bump the version in
-  ``setup.cfg`` and ``src/scancode_config.py``
+  ``setup.cfg`` and ``src/scancode_config.py``. Note that this is CalVer.
+
+- If scancode output data format is changed, increment manually the major, minor or patch
+  to bump the version in ``src/scancode_config.py``. Note that this is SemVer.
+
+Tag and publish
+---------------
 
 - Update the CHANGELOG.rst
 
@@ -22,6 +31,9 @@ How to cut a new release:
 - Draft a new release in GitHub, using the previous release blurb as a base. Highlight new and
   noteworthy changes from the CHANGELOG.rst.
 
+Create Release on GitHub
+------------------------
+
 - Run ``etc/release/scancode_release.sh`` locally.
 
 - Upload the release archives created in the ``dist/`` directory to the GitHub release page.
@@ -33,6 +45,9 @@ How to cut a new release:
 
 - publish the release on GitHub
 
+Upload wheels to pypi and Test
+------------------------------
+
 - then build and publish the released wheel on Pypi. For this you need your own Pypi credentials
   (and get authorized to publish Pypi release: ask @pombredanne) and you need to have the ``twine``
   package installed and configured.

requirements.txt

@@ -9,7 +9,7 @@ cffi==1.14.5
 chardet==4.0.0
 click==8.0.1
 colorama==0.4.4
-commoncode==21.8.27
+commoncode==21.8.31
 construct==2.10.67
 cryptography==3.4.7
 debian-inspector==21.5.25

setup.cfg

@@ -59,7 +59,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 21.8.27
+    commoncode >= 21.8.31
     debian-inspector >= 21.5.25
     dparse >= 0.5.1
     fasteners

src/scancode/cli.py

@@ -94,6 +94,7 @@ def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
     click.echo('ScanCode version ' + scancode_config.__version__)
+    click.echo('Output Format version ' + scancode_config.__output_format_version__)
     ctx.exit()
 
 
@@ -838,6 +839,7 @@ def echo_func(*_args, **_kwargs):
         cle.start_timestamp = start_timestamp
         cle.tool_name = 'scancode-toolkit'
         cle.tool_version = scancode_config.__version__
+        cle.output_format_version = scancode_config.__output_format_version__
         cle.notice = notice
         cle.options = pretty_params or {}
 

src/scancode/cli_test_utils.py

@@ -168,7 +168,7 @@ def check_json_scan(
     result_file,
     regen=False,
     remove_file_date=False,
-    ignore_headers=False
+    check_headers=False,
 ):
     """
     Check the scan `result_file` JSON results against the `expected_file`
@@ -179,7 +179,7 @@ def check_json_scan(
     expectations. But use with caution.
 
     If `remove_file_date` is True, the file.date attribute is removed.
-    If `ignore_headers` is True, the scan headers attribute is removed.
+    If `check_headers` is True, the scan headers attribute is not removed.
     """
     results = load_json_result(result_file, remove_file_date)
     if regen:
@@ -188,7 +188,7 @@ def check_json_scan(
 
     expected = load_json_result(expected_file, remove_file_date)
 
-    if ignore_headers:
+    if not check_headers:
         results.pop('headers', None)
         expected.pop('headers', None)
 
@@ -289,6 +289,7 @@ def check_jsonlines_scan(
     result_file,
     regen=False,
     remove_file_date=False,
+    check_headers=False,
 ):
     """
     Check the scan result_file JSON Lines results against the expected_file
@@ -311,6 +312,10 @@ def check_jsonlines_scan(
         expected = json.load(res)
 
     streamline_jsonlines_scan(expected, remove_file_date)
+
+    if not check_headers:
+        results[0].pop('headers', None)
+        expected[0].pop('headers', None)
 
     expected = json.dumps(expected, indent=2, separators=(',', ': '))
     results = json.dumps(results, indent=2, separators=(',', ': '))

src/scancode_config.py

@@ -75,6 +75,12 @@ def _create_dir(location):
 # in case package is not installed or we do not have setutools/pkg_resources
 # on hand fall back to this version
 __version__ = '21.8.4'
+
+# See https://github.com/nexB/scancode-toolkit/issues/2653 for more information
+# on the data format version
+__output_format_version__ = '1.0.0'
+
+
 try:
     from pkg_resources import get_distribution, DistributionNotFound
     try:

tests/cluecode/test_plugin_filter_clues.py

@@ -41,7 +41,7 @@ def test_scan_plugin_filter_clues_for_rule():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected.json')
-    check_json_scan(expected, result_file, remove_file_date=True, ignore_headers=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=False)
 
 
 def test_scan_plugin_filter_clues_does_not_filter_incorrectly():
@@ -53,7 +53,7 @@ def test_scan_plugin_filter_clues_does_not_filter_incorrectly():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected2.json')
-    check_json_scan(expected, result_file, remove_file_date=True, ignore_headers=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=False)
 
 
 # Regression on types tracked in https://github.com/nexB/typecode/issues/21
@@ -66,4 +66,4 @@ def test_scan_plugin_filter_clues_for_license():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected3.json')
-    check_json_scan(expected, result_file, remove_file_date=True, ignore_headers=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=False)

tests/scancode/data/help/help.txt

@@ -70,10 +70,10 @@ Options:
                                     is not treated as findings).
 
   output control:
-    --full-root   Report full, absolute paths.
-    --strip-root  Strip the root directory segment of all paths. The default is to
-                  always include the last directory segment of the scanned path
-                  such that all paths have a common root directory.
+    --full-root      Report full, absolute paths.
+    --strip-root     Strip the root directory segment of all paths. The default is
+                     to always include the last directory segment of the scanned
+                     path such that all paths have a common root directory.
 
   pre-scan:
     --ignore <pattern>         Ignore files matching <pattern>.

tests/summarycode/test_plugin_consolidate.py

@@ -46,19 +46,19 @@ def test_consolidate_component_package_from_json_can_run_twice(self):
 
         result_file = self.get_temp_file('json')
         run_scan_click(['--from-json', scan_file, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
         # rerun with result_file from last run
         result_file2 = self.get_temp_file('json')
         run_scan_click(['--from-json', result_file, '--consolidate', '--json', result_file2])
-        check_json_scan(expected_file, result_file2, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file2, regen=False, remove_file_date=True)
 
     def test_consolidate_component_package_from_live_scan(self):
         scan_loc = self.get_test_loc('plugin_consolidate/component-package')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/component-package-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
     def test_consolidate_package_always_include_own_manifest_file(self):
         scan_loc = self.get_test_loc('plugin_consolidate/package-manifest')
@@ -83,7 +83,7 @@ def test_consolidate_multiple_same_holder_and_license(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/multiple-same-holder-and-license-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
     def test_consolidate_origin_summary_license_holder_rollup(self):
         scan_loc = self.get_test_loc('plugin_consolidate/license-holder-rollup')
@@ -108,25 +108,25 @@ def test_consolidate_component_package_build_from_live_scan(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/component-package-build-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
     def test_consolidate_report_minority_origin_directory(self):
         scan_loc = self.get_test_loc('plugin_consolidate/report-subdirectory-with-minority-origin')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/report-subdirectory-with-minority-origin-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
     def test_consolidate_zlib(self):
         scan_loc = self.get_test_loc('plugin_consolidate/zlib.json')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/zlib-expected.json')
         run_scan_click(['--from-json', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
 
     def test_consolidate_e2fsprogs(self):
         scan_loc = self.get_test_loc('plugin_consolidate/e2fsprogs.json')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/e2fsprogs-expected.json')
         run_scan_click(['--from-json', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)

tests/summarycode/test_score.py

@@ -52,7 +52,8 @@ def closure_test_function(*args, **kwargs):
             test_env.get_test_loc(expected_file),
             result_file,
             remove_file_date=True,
-            regen=regen)
+            regen=regen,
+        )
 
     test_name = 'test_license_clarity_score_%(test_name)s' % locals()
     test_name = python_safe_name(test_name)