fix(license): stop spliting a long license text

pkg/dependency/parser/python/packaging/parse.go

@@ -80,7 +80,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
  }
 
  if license == "" && h.Get("License-File") != "" {
- license = "file://" + h.Get("License-File")
+ license = licensing.LicenseFilePrefix + h.Get("License-File")
  }
 
  return []ftypes.Package{

pkg/fanal/analyzer/language/python/packaging/packaging.go

@@ -122,11 +122,11 @@ func (a packagingAnalyzer) fillAdditionalData(fsys fs.FS, app *types.Application
  // Parser adds `file://` prefix to filepath from `License-File` field
  // We need to read this file to find licenses
  // Otherwise, this is the name of the license
- if !strings.HasPrefix(lic, "file://") {
+ if !strings.HasPrefix(lic, licensing.LicenseFilePrefix) {
  licenses = append(licenses, lic)
  continue
  }
- licenseFilePath := path.Base(strings.TrimPrefix(lic, "file://"))
+ licenseFilePath := path.Base(strings.TrimPrefix(lic, licensing.LicenseFilePrefix))
 
  findings, err := classifyLicense(app.FilePath, licenseFilePath, a.licenseClassifierConfidenceLevel, fsys)
  if err != nil {

pkg/licensing/normalize.go

@@ -159,6 +159,12 @@ var mapping = map[string]string{
  "PUBLIC DOMAIN": Unlicense,
 }
 
+const (
+ LicenseTextPrefix = "text://"
+ LicenseFilePrefix = "file://"
+ CustomLicensePrefix = "CUSTOM License"
+)
+
 // pythonLicenseExceptions contains licenses that we cannot separate correctly using our logic.
 // first word after separator (or/and) => license name
 var pythonLicenseExceptions = map[string]string{
@@ -179,6 +185,39 @@ var pythonLicenseExceptions = map[string]string{
 
 var licenseSplitRegexp = regexp.MustCompile("(,?[_ ]+(?:or|and)[_ ]+)|(,[ ]*)")
 
+// Typical keywords for license texts
+var licenseTextKeywords = []string{
+ "http://",
+ "https://",
+ "(c)",
+ "as-is",
+ ";",
+ "hereby",
+ "permission to use",
+ "permission is",
+ "use in source",
+ "use, copy, modify",
+ "using",
+}
+
+func isLicenseText(str string) bool {
+ for _, keyword := range licenseTextKeywords {
+ if strings.Contains(str, keyword) {
+ return true
+ }
+ }
+ return false
+}
+
+func TrimLicenseText(text string) string {
+ s := strings.Split(text, " ")
+ n := len(s)
+ if n > 3 {
+ n = 3
+ }
+ return strings.Join(s[:n], " ") + "..."
+}
+
 func Normalize(name string) string {
  name = strings.TrimSpace(name)
  if l, ok := mapping[strings.ToUpper(name)]; ok {
@@ -191,6 +230,12 @@ func SplitLicenses(str string) []string {
  if str == "" {
  return nil
  }
+ if isLicenseText(strings.ToLower(str)) {
+ return []string{
+ LicenseTextPrefix + str,
+ }
+ }
+
  var licenses []string
  for _, maybeLic := range licenseSplitRegexp.Split(str, -1) {
  lower := strings.ToLower(maybeLic)

pkg/licensing/normalize_test.go

@@ -97,6 +97,13 @@ func TestSplitLicenses(t *testing.T) {
  "Historical Permission Notice and Disclaimer (HPND)",
  },
  },
+ {
+ name: "License text",
+ license: "* Permission to use this software in any way is granted without",
+ licenses: []string{
+ "text://* Permission to use this software in any way is granted without",
+ },
+ },
  }
 
  for _, tt := range tests {

pkg/rpc/convert.go

@@ -431,13 +431,14 @@ func ConvertFromRPCDetectedLicenses(rpcLicenses []*common.DetectedLicense) []typ
  for _, l := range rpcLicenses {
  severity := dbTypes.Severity(l.Severity)
  licenses = append(licenses, types.DetectedLicense{
- Severity: severity.String(),
- Category: ConvertFromRPCLicenseCategory(l.Category),
- PkgName: l.PkgName,
- FilePath: l.FilePath,
- Name: l.Name,
- Confidence: float64(l.Confidence),
- Link: l.Link,
+ Severity: severity.String(),
+ Category: ConvertFromRPCLicenseCategory(l.Category),
+ PkgName: l.PkgName,
+ FilePath: l.FilePath,
+ Name: l.Name,
+ LicenseText: l.LicenseText,
+ Confidence: float64(l.Confidence),
+ Link: l.Link,
  })
  }
  return licenses
@@ -978,13 +979,14 @@ func ConvertToRPCLicenses(licenses []types.DetectedLicense) []*common.DetectedLi
  log.Warn("Severity conversion error", log.Err(err))
  }
  rpcLicenses = append(rpcLicenses, &common.DetectedLicense{
- Severity: common.Severity(severity),
- Category: ConvertToRPCLicenseCategory(l.Category),
- PkgName: l.PkgName,
- FilePath: l.FilePath,
- Name: l.Name,
- Confidence: float32(l.Confidence),
- Link: l.Link,
+ Severity: common.Severity(severity),
+ Category: ConvertToRPCLicenseCategory(l.Category),
+ PkgName: l.PkgName,
+ FilePath: l.FilePath,
+ Name: l.Name,
+ LicenseText: l.LicenseText,
+ Confidence: float32(l.Confidence),
+ Link: l.Link,
  })
  }
 

pkg/rpc/convert_test.go

@@ -755,24 +755,26 @@ func TestConvertFromRPCLicenses(t *testing.T) {
  name: "happy",
  rpcLicenses: []*common.DetectedLicense{
  {
- Severity: common.Severity_HIGH,
- Category: common.LicenseCategory_RESTRICTED,
- PkgName: "alpine-baselayout",
- FilePath: "some-path",
- Name: "GPL-2.0",
- Confidence: 1,
- Link: "https://some-link",
+ Severity: common.Severity_HIGH,
+ Category: common.LicenseCategory_RESTRICTED,
+ PkgName: "alpine-baselayout",
+ FilePath: "some-path",
+ Name: "GPL-2.0",
+ LicenseText: "text://* Permission to use this software in any way is granted without",
+ Confidence: 1,
+ Link: "https://some-link",
  },
  },
  want: []types.DetectedLicense{
  {
- Severity: "HIGH",
- Category: "restricted",
- PkgName: "alpine-baselayout",
- FilePath: "some-path",
- Name: "GPL-2.0",
- Confidence: 1,
- Link: "https://some-link",
+ Severity: "HIGH",
+ Category: "restricted",
+ PkgName: "alpine-baselayout",
+ FilePath: "some-path",
+ Name: "GPL-2.0",
+ LicenseText: "text://* Permission to use this software in any way is granted without",
+ Confidence: 1,
+ Link: "https://some-link",
  },
  },
  },
@@ -801,24 +803,26 @@ func TestConvertToRPCLicenses(t *testing.T) {
  name: "happy",
  licenses: []types.DetectedLicense{
  {
- Severity: "HIGH",
- Category: "restricted",
- PkgName: "alpine-baselayout",
- FilePath: "some-path",
- Name: "GPL-2.0",
- Confidence: 1,
- Link: "https://some-link",
+ Severity: "HIGH",
+ Category: "restricted",
+ PkgName: "alpine-baselayout",
+ FilePath: "some-path",
+ Name: "GPL-2.0",
+ LicenseText: "text://* Permission to use this software in any way is granted without",
+ Confidence: 1,
+ Link: "https://some-link",
  },
  },
  want: []*common.DetectedLicense{
  {
- Severity: common.Severity_HIGH,
- Category: common.LicenseCategory_RESTRICTED,
- PkgName: "alpine-baselayout",
- FilePath: "some-path",
- Name: "GPL-2.0",
- Confidence: 1,
- Link: "https://some-link",
+ Severity: common.Severity_HIGH,
+ Category: common.LicenseCategory_RESTRICTED,
+ PkgName: "alpine-baselayout",
+ FilePath: "some-path",
+ Name: "GPL-2.0",
+ LicenseText: "text://* Permission to use this software in any way is granted without",
+ Confidence: 1,
+ Link: "https://some-link",
  },
  },
  },

pkg/scanner/local/scan.go

@@ -261,6 +261,19 @@ func (s Scanner) scanLicenses(target types.ScanTarget, options types.ScanOptions
  var osPkgLicenses []types.DetectedLicense
  for _, pkg := range target.Packages {
  for _, license := range pkg.Licenses {
+ if strings.HasPrefix(license, licensing.LicenseTextPrefix) {
+ licenseText := strings.TrimPrefix(license, licensing.LicenseTextPrefix)
+ osPkgLicenses = append(osPkgLicenses, types.DetectedLicense{
+ Severity: dbTypes.SeverityUnknown.String(),
+ Category: ftypes.CategoryUnknown,
+ PkgName: pkg.Name,
+ Name: licensing.CustomLicensePrefix + ": " + licensing.TrimLicenseText(licenseText),
+ LicenseText: licenseText,
+ Confidence: 1.0,
+ })
+ continue
+ }
+
  category, severity := scanner.Scan(license)
  osPkgLicenses = append(osPkgLicenses, types.DetectedLicense{
  Severity: severity,
@@ -282,6 +295,20 @@ func (s Scanner) scanLicenses(target types.ScanTarget, options types.ScanOptions
  var langLicenses []types.DetectedLicense
  for _, lib := range app.Packages {
  for _, license := range lib.Licenses {
+ if strings.HasPrefix(license, licensing.LicenseTextPrefix) {
+ licenseText := strings.TrimPrefix(license, licensing.LicenseTextPrefix)
+ langLicenses = append(langLicenses, types.DetectedLicense{
+ Severity: dbTypes.SeverityUnknown.String(),
+ Category: ftypes.CategoryUnknown,
+ PkgName: lib.Name,
+ Name: licensing.CustomLicensePrefix + ": " + licensing.TrimLicenseText(licenseText),
+ LicenseText: licenseText,
+ FilePath: lo.Ternary(lib.FilePath != "", lib.FilePath, app.FilePath),
+ Confidence: 1.0,
+ })
+ continue
+ }
+
  category, severity := scanner.Scan(license)
  langLicenses = append(langLicenses, types.DetectedLicense{
  Severity: severity,

pkg/types/license.go

@@ -22,6 +22,9 @@ type DetectedLicense struct {
  // Name holds a detected license name
  Name string
 
+ // LicenseText holds a long license text if Trivy detects a license name as a license text
+ LicenseText string
+
  // Confidence is level of the match. The confidence level is between 0.0 and 1.0, with 1.0 indicating an
  // exact match and 0.0 indicating a complete mismatch
  Confidence float64