Enforce licenses in third_party code added to brave-core

.github/CODEOWNERS

@@ -50,6 +50,7 @@ components/brave_wayback_machine @simonhong
 common/licenses/ @fmarier
 components/third_party/ @fmarier
 script/brave_license_helper.py @fmarier
+script/generate_licenses.py @fmarier
 
 # Crypto Wallets
 browser/brave_wallet/ @bbondy @ryanml

DEPS

@@ -2,7 +2,7 @@ use_relative_paths = True
 
 deps = {
   "vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@8e45290ced608da6c9b992a92b10355d9e140aae",
-  "vendor/extension-whitelist": "https://github.com/brave/extension-whitelist.git@7843f62e26a23c51336330e220e9d7992680aae9",
+  "vendor/extension-whitelist": "https://github.com/brave/extension-whitelist.git@b4d059c73042cacf3a5e9156d4b1698e7bc18678",
   "vendor/hashset-cpp": "https://github.com/brave/hashset-cpp.git@6eab0271d014ff09bd9f38abe1e0c117e13e9aa9",
   "vendor/requests": "https://github.com/kennethreitz/requests@e4d59bedfd3c7f4f254f4f5d036587bcd8152458",
   "vendor/boto": "https://github.com/boto/boto@f7574aa6cc2c819430c1f05e9a1a1a666ef8169b",
@@ -58,5 +58,10 @@ hooks = [
     'pattern': '.',
     'action': ['python', 'src/brave/script/build-simple-js-bundle.py', '--repo_dir_path', 'src/brave/components/brave_sync/extension/brave-sync-android'],
     'condition': 'checkout_android',
-  }
+  },
+  {
+    'name': 'generate_licenses',
+    'pattern': '.',
+    'action': ['python', 'src/brave/script/generate_licenses.py'],
+  },
 ]

components/brave_new_tab_ui/data/README.chromium

@@ -1,3 +1,3 @@
-Name: Background images
+Name: Brave background images
 URL: https://github.com/brave/brave-core/tree/master/components/img/newtab
 License: various

patches/tools-licenses.py.patch

@@ -1,12 +1,12 @@
 diff --git a/tools/licenses.py b/tools/licenses.py
-index 6405b31497a425c68b0486a2653071f23e515671..e957c14c924a7a405bf9e20f3c8e2ffc56ea1306 100755
+index 6405b31497a425c68b0486a2653071f23e515671..5402e94334b0ecb6ebcbefb9e4cd93b070eb5cbd 100755
 --- a/tools/licenses.py
 +++ b/tools/licenses.py
 @@ -35,6 +35,7 @@ else:
  _REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
  sys.path.insert(0, os.path.join(_REPOSITORY_ROOT, 'build/android/gyp'))
  from util import build_utils
-+from brave_license_helper import AddBraveCredits, BRAVE_THIRD_PARTY_DIRS
++from brave_license_helper import AddBraveCredits, BRAVE_THIRD_PARTY_DIRS, CheckBraveMissingLicense
 
 
  # Paths from the root of the tree to directories to skip.
@@ -28,3 +28,13 @@ index 6405b31497a425c68b0486a2653071f23e515671..e957c14c924a7a405bf9e20f3c8e2ffc
              # Add all subdirectories that are not marked for skipping.
              for dir in dirs:
                  dirpath = os.path.join(path, dir)
+@@ -663,7 +666,8 @@ def GenerateCredits(
+     for path in third_party_dirs:
+         try:
+             metadata = ParseDir(path, _REPOSITORY_ROOT)
+-        except LicenseError:
++        except LicenseError as e:
++            CheckBraveMissingLicense(target_os, path, e)
+             # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
+             continue
+         if metadata['License File'] == NOT_SHIPPED:

script/brave_license_helper.py

@@ -9,6 +9,14 @@
     'vendor',
 ]
 
+ANDROID_ONLY_PATHS = [
+    os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync-android'),
+]
+
+DESKTOP_ONLY_PATHS = [
+    os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync'),
+]
+
 
 def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
     # Exclude these specific paths from needing a README.chromium file.
@@ -30,6 +38,18 @@ def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
     # Add the licensing info that would normally be in a README.chromium file.
     # This is for when we pull in external repos directly.
     special_cases.update({
+        os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync'): {
+            "Name": "Brave Sync",
+            "URL": "https://github.com/brave/sync",
+            "License": "MPL-2.0",
+            "License File": "/brave/components/brave_sync/extension/brave-sync/LICENSE.txt",
+        },
+        os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync-android'): {
+            "Name": "Brave Sync",
+            "URL": "https://github.com/brave/sync",
+            "License": "MPL-2.0",
+            "License File": "/brave/components/brave_sync/extension/brave-sync-android/LICENSE.txt",
+        },
         os.path.join('brave', 'vendor', 'adblock_rust_ffi'): {
             "Name": "adblock-rust-ffi",
             "URL": "https://github.com/brave/adblock-rust-ffi",
@@ -109,6 +129,11 @@ def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
             "URL": "https://github.com/brave/Sparkle",
             "License": "MIT",
         },
+        os.path.join('brave', 'vendor', 'speedreader_rust_ffi'): {
+            "Name": "speedreader-rust-ffi",
+            "URL": "https://github.com/brave-experiments/speedreader-rust-ffi",
+            "License": "MPL-2.0",
+        },
     })
 
     # Don't recurse into these directories looking for third-party code.
@@ -126,8 +151,20 @@ def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
     additional_list += [
         os.path.join('brave', 'components', 'brave_prochlo'),
         os.path.join('brave', 'components', 'brave_new_tab_ui', 'data'),
-        os.path.join('brave', 'components', 'brave_sync'),
+        os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync'),
+        os.path.join('brave', 'components', 'brave_sync', 'extension', 'brave-sync-android'),
     ]
     additional_paths = tuple(additional_list)
 
     return (prune_dirs, additional_paths)
+
+
+def CheckBraveMissingLicense(target_os, path, error):
+    if path.startswith('brave'):
+        if (target_os == 'android'):
+            if path in DESKTOP_ONLY_PATHS:
+                return  # Desktop failures are not relevant on Android.
+        else:
+            if path in ANDROID_ONLY_PATHS:
+                return  # Android failures are not relevant on desktop.
+        raise error

script/generate_licenses.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Brave Authors. All rights reserved.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import json
+import os
+import sys
+
+from io import open
+from lib.config import SOURCE_ROOT
+
+
+KNOWN_MISSING = [
+    # Emailed author (henrik@schack.dk) on 2019-11-05.
+    os.path.join('components', 'third_party', 'adblock', 'lists', 'adblock_dk'),
+    # https://github.com/gfmaster/adblock-korea-contrib/issues/47
+    os.path.join('components', 'third_party', 'adblock', 'lists', 'adblock_korea_contrib'),
+]
+
+
+def extract_license_info(directory):
+    readme_path = os.path.join(directory, 'README.chromium')
+    if not os.path.isfile(readme_path):
+        print('Missing README.chromium in %s' % directory)
+        sys.exit(1)
+
+    metadata = {
+        'slug': os.path.basename(directory),
+        'Name': None,
+        'URL': None,
+        'License': None,
+        'License File': None,
+        'License Text': None,
+    }
+
+    with open(readme_path, mode='rt', encoding='utf-8') as file_handle:
+        for line in file_handle:
+            for field in metadata:
+                if '%s:' % field in line:
+                    metadata[field] = line[len(field) + 1:].strip()
+                    break
+
+    if not metadata['License File']:
+        license_path = os.path.join(directory, 'LICENSE')
+        if os.path.isfile(license_path):
+            with open(license_path, mode='rt', encoding='utf-8') as file_handle:
+                metadata['License Text'] = file_handle.read()
+        elif metadata['License'] == 'unknown':
+            relative_dir = directory[len(SOURCE_ROOT) + 1:]
+            if relative_dir not in KNOWN_MISSING:
+                print('Unknown license is not whitelisted: %s' % relative_dir)
+                sys.exit(1)
+        else:
+            print(metadata)
+            print('Missing LICENSE in %s' % directory)
+            sys.exit(1)
+
+    return metadata
+
+
+def read_license_text(filename):
+    # README.chromium files assume UNIX path separators.
+    full_path = os.path.dirname(SOURCE_ROOT)
+    for piece in filename.split('/'):
+        full_path = os.path.join(full_path, piece)
+
+    if not os.path.isfile(full_path):
+        print('License file not found: %s' % full_path)
+        sys.exit(1)
+
+    with open(full_path, mode='rt', encoding='utf-8') as file_handle:
+        return file_handle.read()
+
+
+def external_component_license_file(preamble, components):
+    component_notices = ''
+    component_licenses = ''
+
+    licenses = {}
+    for component in components:
+        if component_notices:
+            component_notices += '\n'
+
+        license_id = component['License']
+        license_text = component['License Text']
+        if license_text:
+            # Custom license
+            license_id += '-' + component['slug']
+
+        component_notices += 'Name: %s\nURL: %s\nLicense: %s\n' % (component['Name'], component['URL'], license_id)
+
+        if license_id == 'unknown':
+            continue
+
+        if license_id not in licenses:
+            if license_text:
+                licenses[license_id] = license_text
+            else:
+                licenses[license_id] = read_license_text(component['License File'])
+
+    for license_id in licenses:
+        component_licenses += '--------------------------------------------------------------------------------\n'
+        component_licenses += '%s:\n\n' % license_id
+        component_licenses += '%s\n' % licenses[license_id]
+
+    return '%s\n\n%s\n%s' % (preamble, component_notices, component_licenses)
+
+
+def list_sub_components(base_dir):
+    found = []
+    for dirpath, dirs, dummy in os.walk(base_dir):
+        for dir_name in dirs:
+            found.append(extract_license_info(os.path.join(dirpath, dir_name)))
+    return found
+
+
+def write_license_file(directory, contents):
+    file_path = os.path.join(directory, 'LICENSE')
+    with open(file_path, mode='wt', encoding='utf-8') as file_handle:
+        file_handle.write(contents)
+
+
+def list_ntp_backgrounds(metadata_file):
+    json_metadata = ''
+    with open(metadata_file, mode='rt', encoding='utf-8') as file_handle:
+        # Strip out copyright header
+        metadata = file_handle.readlines()[3:]
+        # Hack to turn this TypeScript file into valid JSON
+        json_metadata = ''.join(metadata).replace('export const images: NewTab.Image[] = [', '[') \
+                                         .replace('"', '\"').replace("'", '"')
+
+    images = json.loads(json_metadata)
+    return images
+
+
+def validated_data_field(data, field_name):
+    field_value = data[field_name]
+    if not field_value:
+        print('Missing %s for background image %s' % (field_name, data['name']))
+        sys.exit(1)
+
+    return field_value
+
+
+def generate_backgrounds_license(preamble, backgrounds):
+    notices = ''
+
+    for background in backgrounds:
+        if notices:
+            notices += '\n'
+
+        filename = validated_data_field(background, 'source')
+        author_name = validated_data_field(background, 'author')
+        author_link = validated_data_field(background, 'link')
+        original_url = validated_data_field(background, 'originalUrl')
+        license_text = validated_data_field(background, 'license')
+        if license_text != 'used with permission' and license_text[0:8] != 'https://' \
+           and license_text[0:7] != 'http://':
+            print('Invalid license for background image %s. It needs to be a URL or the string "used with permission".'
+                  % background['name'])
+            sys.exit(1)
+
+        notices += 'File: %s\nAuthor: %s (%s)\nURL: %s\nLicense: %s\n' \
+                   % (filename, author_name, author_link, original_url, license_text)
+
+    return '%s\n\n%s' % (preamble, notices)
+
+
+def main():
+    components_dir = os.path.join(SOURCE_ROOT, 'components')
+    third_party_dir = os.path.join(components_dir, 'third_party')
+
+    # Brave Ad Block component
+    adblock_dir = os.path.join(third_party_dir, 'adblock')
+    adblock_lists_dir = os.path.join(adblock_dir, 'lists')
+    adblock_preamble = 'These licenses do not apply to any of the code shipped with the Brave Browser, but may ' \
+                       'apply to lists downloaded after installation for use with the Brave Shields feature. ' \
+                       'The Brave Browser and such lists are separate and independent works.'
+
+    adblock_components = list_sub_components(adblock_lists_dir)
+    write_license_file(adblock_dir, external_component_license_file(adblock_preamble, adblock_components))
+    print('  - %s sub-components added in adblock/LICENSE' % len(adblock_components))
+
+    # Brave Local Data component
+    local_data_dir = os.path.join(third_party_dir, 'local_data')
+    local_data_lists_dir = os.path.join(local_data_dir, 'lists')
+    local_data_preamble = 'These licenses do not apply to any of the code shipped with the Brave Browser, but may ' \
+                          'apply to data files downloaded after installation for use with various Brave features. ' \
+                          'The Brave Browser and such data files are separate and independent works.'
+
+    local_data_components = list_sub_components(local_data_lists_dir)
+    write_license_file(local_data_dir, external_component_license_file(local_data_preamble, local_data_components))
+    print('  - %s sub-components added in local_data/LICENSE' % len(local_data_components))
+
+    # Brave New Tab UI component
+    ntp_data_dir = os.path.join(components_dir, 'brave_new_tab_ui', 'data')
+    ntp_backgrounds_preamble = 'These licenses do not apply to any of the code shipped with the Brave Browser and ' \
+                               'instead apply to background images used on the new tab page. The Brave Browser and ' \
+                               'such data files are separate and independent works.'
+
+    ntp_backgrounds = list_ntp_backgrounds(os.path.join(ntp_data_dir, 'backgrounds.ts'))
+    write_license_file(ntp_data_dir, generate_backgrounds_license(ntp_backgrounds_preamble, ntp_backgrounds))
+    print('  - %s sub-components added in brave_new_tab_ui/data/LICENSE' %
+          len(ntp_backgrounds))
+
+
+if __name__ == '__main__':
+    sys.exit(main())