Use packages attribute throughout #1237

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

src/packagedcode/maven.py

@@ -87,7 +87,7 @@ def get_package_root(cls, manifest_resource, codebase):
         if manifest_resource.name.endswith(('pom.xml', '.pom',)):
             # the root is either the parent or further up for poms stored under
             # a META-INF dir
-            package_data = manifest_resource.package_manifests
+            package_data = manifest_resource.packages
             if not package_data:
                 return manifest_resource
             package_data = package_data[0]

src/packagedcode/models.py

@@ -447,10 +447,10 @@ class Package(BasePackage):
         label='notice text',
         help='A notice text for this package.')
 
-    manifest_path = String(
-        label='manifest path',
-        help='A relative path to the manifest file if any, such as a '
-             'Maven .pom or a npm package.json.')
+    root_path = String(
+        label='package root path',
+        help='The path to the root of the package documented in this manifest '
+             'if any, such as a Maven .pom or a npm package.json parent directory.')
 
     dependencies = List(
         item_type=DependentPackage,

src/packagedcode/plugin_package.py

@@ -28,13 +28,11 @@
 from __future__ import unicode_literals
 
 from collections import OrderedDict
-import posixpath
 
 import attr
 import click
 click.disable_unicode_literals_warning = True
 
-from commoncode.fileutils import parent_directory
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
 from scancode import CommandLineOption
@@ -74,7 +72,6 @@ class PackageScanner(ScanPlugin):
     """
 
     resource_attributes = OrderedDict()
-    resource_attributes['package_manifests'] = attr.ib(default=attr.Factory(list), repr=False)
     resource_attributes['packages'] = attr.ib(default=attr.Factory(list), repr=False)
 
     sort_order = 6
@@ -84,7 +81,7 @@ class PackageScanner(ScanPlugin):
     options = [
         CommandLineOption(('-p', '--package',),
             is_flag=True, default=False,
-            help='Scan <input> for package manifests and packages.',
+            help='Scan <input> for package manifests and build scripts.',
             help_group=SCAN_GROUP,
             sort_order=20),
 
@@ -116,73 +113,40 @@ def process_codebase(self, codebase, **kwargs):
             return
 
         for resource in codebase.walk(topdown=False):
-            set_packages_at_root(resource, codebase)
+            set_packages_root(resource, codebase)
 
 
-def set_packages_at_root(resource, codebase):
+def set_packages_root(resource, codebase):
     """
-    Set the packages attribute at the root Resource for a given package manifest
-    that may exist in resource.
+    Set the root_path attribute as the path to the root Resource for a given
+    package package or build script that may exist in a `resource`.
     """
-    # only files can have a manifest
-    if resource.is_root or not resource.is_file:
+    # only files can have a package
+    if not resource.is_file:
         return
-    packages_info = resource.package_manifests
-    if not packages_info:
+
+    packages = resource.packages
+    if not packages:
         return
-    manifest_path = resource.path
+    # NOTE: we are dealing with a single file therefore there should be only be
+    # a single package detected. But some package manifests can document more
+    # than one package at a time such as multiple arches/platforms for a gempsec
+    # or multiple sub package (with "%package") in an RPM .spec file.
 
-    # NOTE: we are dealing with a single file therefore there should be
-    # only be a single package detected. But some package manifests can
-    # document more than one package at a time such as multiple
-    # arches/platforms for a gempsec or multiple sub package (with
-    # "%package") in an RPM .spec file.
     modified = False
-    for package_info in packages_info:
-        modified = True
-        package_info['manifest_path'] = manifest_path
-
-        package_class = get_package_class(package_info)
+    for package in packages:
+        package_class = get_package_class(package)
         package_root = package_class.get_package_root(resource, codebase)
-
         if not package_root:
-            # this can happen if we scan a single resource that is a package manifest
+            # this can happen if we scan a single resource that is a package package
             continue
-
-        if package_root == resource:
-            continue
-
-        # here new_package_root != resource:
-
         # What if the target resource (e.g. a parent) is the root and we are in stripped root mode?
         if package_root.is_root and codebase.strip_root:
             continue
+        package['root_path'] = package_root.path
+        modified = True
 
-        # Determine if this package applies to more than just the manifest
-        # file (typically it means the whole parent directory is the
-        # package) and if yes:
-        # 1. fetch this resource
-        # 2. move the package data to this new resource
-        # 3. set the manifest_path if needed.
-        # 4. save.
-
-        # TODO: this is a hack for the ABOUT file Package parser, ABOUT files are kept alongside
-        # the resource its for
-        if package_root.is_file and resource.path.endswith('.ABOUT'):
-            new_manifest_path = posixpath.join(parent_directory(package_root.path), resource.name)
-        else:
-            # here we have a relocated Resource and we compute the manifest path
-            # relative to the new package root
-            # TODO: We should have codebase relative paths for manifests
-            new_package_root_path = package_root.path and package_root.path.strip('/')
-            if new_package_root_path:
-                _, _, new_manifest_path = resource.path.partition(new_package_root_path)
-                # note we could have also deserialized and serialized again instead
-        package_info['manifest_path'] = new_manifest_path.lstrip('/')
-
-        package_root.packages.append(package_info)
-        codebase.save_resource(package_root)
     if modified:
-        # we did set the manifest_path
+        # we did set the root_path
         codebase.save_resource(resource)
     return resource

src/scancode/api.py

@@ -282,12 +282,13 @@ def get_package_info(location, **kwargs):
     try:
         manifest = recognize_package(location)
         if manifest:
-            return dict(package_manifests=[manifest.to_dict()])
+            # TODO: what if we have more than one package in a manifest?
+            return dict(packages=[manifest.to_dict()])
     except Exception as e:
         if TRACE:
             logger.error('get_package_info: {}: Exception: {}'.format(location, e))
         pass
-    return dict(package_manifests=[])
+    return dict(packages=[])
 
 
 def get_file_info(location, **kwargs):

tests/formattedcode/data/csv/livescan/expected.csv

@@ -1,4 +1,4 @@
-Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,files_count,dirs_count,size_count,scan_errors,license_expression,license__key,license__score,license__name,license__short_name,license__category,license__is_exception,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_expression,matched_rule__licenses,matched_rule__is_license_text,matched_rule__is_license_notice,matched_rule__is_license_reference,matched_rule__is_license_tag,matched_rule__matcher,matched_rule__rule_length,matched_rule__matched_length,matched_rule__match_coverage,matched_rule__rule_relevance,copyright,copyright_holder,email,url,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__manifest_path,package__contains_source_code
+Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,files_count,dirs_count,size_count,scan_errors,license_expression,license__key,license__score,license__name,license__short_name,license__category,license__is_exception,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_expression,matched_rule__licenses,matched_rule__is_license_text,matched_rule__is_license_notice,matched_rule__is_license_reference,matched_rule__is_license_tag,matched_rule__matcher,matched_rule__rule_length,matched_rule__matched_length,matched_rule__match_coverage,matched_rule__rule_relevance,copyright,copyright_holder,email,url,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__root_path,package__contains_source_code
 json2csv.rb,file,json2csv.rb,json2csv,.rb,912,2018-11-15,1236469a06a2bacbdd8e172ad718482af5b0a936,1307c281e0b153202e291b217eab85d5,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 json2csv.rb,,,,,,,,,,,,,,,,,,,,,,apache-2.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,100.00,Apache License 2.0,Apache 2.0,Permissive,False,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,13,apache-2.0_7.RULE,apache-2.0,apache-2.0,,True,,,2-aho,85,85,100.00,100.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

tests/formattedcode/data/csv/packages/expected-no-root.csv

@@ -1,3 +1,3 @@
-Resource,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__manifest_path,package__contains_source_code
+Resource,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__root_path,package__contains_source_code
 package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,
 package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,artistic-2.0,Artistic-2.0,,,

tests/formattedcode/data/csv/packages/expected.csv

@@ -1,4 +1,4 @@
-Resource,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__manifest_path,package__contains_source_code
-/scan/,directory,,,,,,,,,,,,,,,,,,,,,,,,,,
-/scan/,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,artistic-2.0,Artistic-2.0,,scan/package.json,
-/scan/package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,
+Resource,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__root_path,package__contains_source_code
+scan/,directory,,,,,,,,,,,,,,,,,,,,,,,,,,
+scan/package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,
+scan/package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,artistic-2.0,Artistic-2.0,,scan,

tests/formattedcode/data/json/simple-expected.json

@@ -42,7 +42,6 @@
       "copyrights": [],
       "holders": [],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 1,
       "dirs_count": 0,
@@ -84,7 +83,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,

tests/formattedcode/data/json/simple-expected.jsonpp

@@ -42,7 +42,6 @@
       "copyrights": [],
       "holders": [],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 1,
       "dirs_count": 0,
@@ -84,7 +83,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,

tests/formattedcode/data/json/tree/expected.json

@@ -55,7 +55,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -97,7 +96,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -139,7 +137,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -169,7 +166,6 @@
       "copyrights": [],
       "holders": [],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 4,
       "dirs_count": 0,
@@ -211,7 +207,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -253,7 +248,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -295,7 +289,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,
@@ -337,7 +330,6 @@
         }
       ],
       "authors": [],
-      "package_manifests": [],
       "packages": [],
       "files_count": 0,
       "dirs_count": 0,

tests/licensedcode/data/plugin_license/package/package.expected.json

@@ -57,7 +57,7 @@
       "license_expressions": [
         "mit"
       ],
-      "package_manifests": [
+      "packages": [
         {
           "type": "npm",
           "namespace": null,
@@ -102,7 +102,7 @@
             }
           ],
           "notice_text": null,
-          "manifest_path": null,
+          "root_path": null,
           "dependencies": [
             {
               "purl": "pkg:npm/dicer",
@@ -129,7 +129,6 @@
           "api_data_url": "https://registry.npmjs.org/busboy/0.2.14"
         }
       ],
-      "packages": [],
       "scan_errors": []
     }
   ]

tests/packagedcode/data/about/apipkg.ABOUT-expected

@@ -32,7 +32,7 @@
   "license_expression": "mit",
   "declared_license": "mit",
   "notice_text": null,
-  "manifest_path": null,
+  "root_path": null,
   "dependencies": [],
   "contains_source_code": null,
   "source_packages": [],

tests/packagedcode/data/about/appdirs.ABOUT-expected

@@ -32,7 +32,7 @@
   "license_expression": "mit",
   "declared_license": "mit",
   "notice_text": null,
-  "manifest_path": null,
+  "root_path": null,
   "dependencies": [],
   "contains_source_code": null,
   "source_packages": [],

tests/packagedcode/data/bower/author-objects/expected.json

@@ -47,7 +47,7 @@
     "BSD-3-Clause"
   ],
   "notice_text": null,
-  "manifest_path": null,
+  "root_path": null,
   "dependencies": [
     {
       "purl": "pkg:bower/get-size",

tests/packagedcode/data/bower/basic/expected.json

@@ -38,7 +38,7 @@
     "MIT"
   ],
   "notice_text": null,
-  "manifest_path": null,
+  "root_path": null,
   "dependencies": [
     {
       "purl": "pkg:bower/get-size",

tests/packagedcode/data/bower/list-of-licenses/expected.json

@@ -40,7 +40,7 @@
     "BSD-3-Clause"
   ],
   "notice_text": null,
-  "manifest_path": null,
+  "root_path": null,
   "dependencies": [
     {
       "purl": "pkg:bower/get-size",