Roll out FOSSA scanning to all repositories

[trask]

FOSSA does both license and security scanning. Our initial focus is on license scanning since there may be overlap with other tools on the security scanning side.

I'm planning on asking @opentelemetrybot to fork and open PRs to all repos to add a fossa.yml workflow like https://github.com/open-telemetry/opentelemetry-java/blob/main/.github/workflows/fossa.yml.

Update: expand this for the script @opentelemetrybot is running

#!/usr/bin/env python3
from github3 import login
import os
import subprocess
import shutil

# Need to use a classic token with the following permissions:
# - repo
# - workflow
#
# I can't find any supporting documentation, but it doesn't seem that you can
# set "Allow edits and access to secrets by maintainers" on a PR from a fork
# when using a fine-grained token.
TOKEN = os.environ['GITHUB_TOKEN']

log_file=open('fossa.log', 'w')

gh = login(token=TOKEN)
temp_dir = 'repos'

def setup_local_repo(repo_name):
    upstream_repo = gh.repository('open-telemetry', repo_name)
    fork = upstream_repo.create_fork()

    repo_dir=os.path.join(temp_dir, repo_name)

    if not os.path.exists(repo_dir):
        subprocess.run(['git', 'clone', fork.clone_url], cwd=temp_dir)

    subprocess.run(['git', 'config', 'user.name', 'otelbot'], cwd=repo_dir)
    subprocess.run(['git', 'config', 'user.email', '197425009+otelbot@users.noreply.github.com'], cwd=repo_dir)

    subprocess.run(['git', 'stash'], cwd=repo_dir)
    subprocess.run(['git', 'clean', '-d', '-f'], cwd=repo_dir)

    existing_branches = subprocess.run(['git', 'branch', '--format=%(refname:short)'], cwd=repo_dir,
                                       capture_output=True, text=True).stdout.split()
    subprocess.run(['git', 'checkout', 'main'], cwd=repo_dir)
    for branch in existing_branches:
        if branch != 'main':
            subprocess.run(['git', 'branch', '-D', branch], cwd=repo_dir)

    existing_remotes = subprocess.run(['git', 'remote'], cwd=repo_dir, capture_output=True, text=True).stdout.split()
    if 'upstream' not in existing_remotes:
        subprocess.run(['git', 'remote', 'add', 'upstream', upstream_repo.clone_url], cwd=repo_dir)

    subprocess.run(['git', 'fetch', 'upstream'], cwd=repo_dir)

def create_fossa_pr(repo_name):

    # need a token with org permissions to create the PR
    upstream_repo = gh.repository('open-telemetry', repo_name)

    existing_prs = upstream_repo.pull_requests(state='open', head=f'opentelemetrybot:fossa')
    if any(True for _ in existing_prs):
        print("Found existing PR, skipping")
        return

    repo_dir=os.path.join(temp_dir, repo_name)

    subprocess.run(['git', 'checkout', '-b', 'fossa', 'upstream/main'], cwd=repo_dir)
    workflow_file_path = os.path.join(repo_dir, '.github', 'workflows', 'fossa.yml')
    if os.path.exists(workflow_file_path):
        print("Found existing FOSSA workflow file, skipping")
        return

    os.makedirs(os.path.dirname(workflow_file_path), exist_ok=True)
    shutil.copy('workflow-templates/fossa.yml', workflow_file_path)

    subprocess.run(['git', 'add', '.github/workflows/fossa.yml'], cwd=repo_dir)
    subprocess.run(['git', 'commit', '-a', '-m', 'Add FOSSA scanning workflow'], cwd=repo_dir)

    subprocess.run(['git', 'push', '-f', f'https://{TOKEN}@github.com/opentelemetrybot/{repo_name}.git', 'fossa'],
                   cwd=repo_dir)

    pr = upstream_repo.create_pull(title='Add FOSSA scanning workflow',
                                   body='See https://github.com/open-telemetry/community/issues/2574 for details',
                                   base='main',
                                   head='opentelemetrybot:fossa')

    print(f'PR created: {pr.html_url}')
    log_file.write(f'PR created: {pr.html_url}\n')

def process_all():

    org = gh.organization('open-telemetry')
    repos = [repo for repo in org.repositories() if not repo.archived]

    for repo in repos:
        setup_local_repo(repo.name)
        create_fossa_pr(repo.name)

process_all()

Maintainers: if you would like access to FOSSA, just DM me your email address on Slack (note: it needs to be an email address that is not already associated with a FOSSA account). I will collect the email addresses for a few days and then ask the CNCF to send out invites to the first batch (we are under the CNCF's FOSSA enterprise account and don't have access to send out invites ourselves).

Once we get a clean bill of health from FOSSA, we would like to publish the (passing) badges to https://github.com/open-telemetry/community/blob/main/reports/compliance.md. And folks can of course add the badges directly on their repos as well.

We haven't figured out yet the best way to track new licensing issues after initial compliance, though if you request access to FOSSA then you'll be able to have it send you notifications for a given repo(s).

[trask]

Sent a few PRs in the first batch, will wait until next week to send to the remaining repos.

[trask]

all PRs are created now

[trask]

Just a heads up for anyone watching this, we will come back to this work sometime after KubeCon