Plugin interface draft

Core maintainer perspective

Type definitions

export type CoreConfig = {
  /** list of plugins to be used (built-in, 3rd party, or custom) */
  plugins: PluginConfig[];
  /** portal configuration for uploading results */
  upload?: UploadConfig;
  /** categorization of individual audits */
  categories: CategoryConfig[];
  /** budget rules for assertion */
  budgets?: Budget[];
};

type UploadConfig = {
  /** URL of deployed portal API */
  server: string; // required if self-hosting (could default to our public cloud URL later)
  /** API key with write access to portal */
  apiKey: string; // should use environment variable
};

type CategoryConfig = {
  /** human-readable unique ID */
  slug: string;
  /** display name */
  title: string;
  /** description (Markdown) */
  description?: string;
  /** weighted references to plugin-specific audits/categories */
  metrics: {
    /** reference to a plugin's audit (e.g. 'eslint#max-lines') or category (e.g. 'categories:lhci#performance') */
    ref: string;
    /** coefficient for given score (use weight 0 if only for display) */
    weight: number;
  }[];
};

type Budget = {
  /** reference to audit ('eslint#max-lines') or category ('categories:performance') */
  ref: string;
  /** fail assertion if score too low */
  minScore?: number;
  /** fail assertion if too many warnings */
  maxWarnings?: number;
  /** fail assertion if value too high */
  maxValue?: number;
};

Usage example

const config: CoreConfig = {
  upload: {
    server: 'https://192.168.123.132/api/graphql',
    apiKey: process.env.SECRET_API_KEY!,
  },
  plugins: [
    eslintPlugin({ config: 'eslint.config.js' }),
    lhciPlugin({ config: '.lighthouserc.json' }),
    // ...
  ],
  categories: [
    {
      slug: 'performance',
      title: 'Performance',
      metrics: [
        {
          ref: 'groups:lhci#performance',
          weight: 3,
        },
        {
          ref: 'bundle-size#main',
          weight: 1,
        },
        {
          ref: 'eslint#@angular-eslint/template/no-call-expression',
          weight: 0,
        },
      ],
    },
    // ...
  ],
  budgets: [
    { ref: 'categories:performance', minScore: 0.6 },
    { ref: 'bundle-size#main', maxValue: 2_000_000 },
  ],
};

export default config;

Plugin author perspective

Type definitions

export type PluginConfig = {
  /** plugin metadata */
  meta: PluginMetadata;
  /** how to execute runner */
  runner: RunnerConfig;
  /** list of scorable metrics for given plugin */
  audits: AuditMetadata[];
  /** list of groups */
  groups?: Group[];
};

type PluginMetadata = {
  /** unique ID (human-readable, URL-safe) */
  slug: string;
  /** display name */
  name: string;
  /** plugin categorization */
  type: PluginType;
  /** icon from VSCode Material Icons extension */
  icon?: MaterialIcon; // see: https://github.com/flowup/quality-metrics/tree/main/libs/material-icons#readme
  /** plugin documentation site */
  docsUrl?: string;
};

type PluginType =
  | 'static-analysis' // eslint, stylelint, tsc, jscpd, ...
  | 'performance-measurements' // lhci collect, user-flow, ...
  | 'test-coverage' // jest --coverage, ...
  | 'dependency-audit'; // npm audit, ...

type RunnerConfig = {
  /** shell command to execute */
  command: string;
  /** path to runner artefact */
  outputPath: string
};

type AuditMetadata = {
  /** ID (unique within plugin) */
  slug: string;
  /** abbreviated name */
  label: string; // e.g. 'LCP', 'no-explicit-any' 'main.js'
  /** descriptive name */
  title: string; // e.g. 'Largest Contentful Paint', 'Disallow the `any` type', 'Size of main bundle'
  /** description (Markdown) */
  description?: string;
  /** link to documentation (rule rationale) */
  docsUrl?: string;
};

type Group = {
  /** human-readable unique ID */
  slug: string;
  /** display name */
  title: string;
  /** description (Markdown) */
  description?: string;
  /** weighted references to plugin-specific audits/categories */
  audits: {
    /** reference to a audit within plugin (e.g. 'max-lines') */
    ref: string;
    /** coefficient for given score (use weight 0 if only for display) */
    weight: number;
  }[];
};

Usage example

/* ESLint plugin */

type ESLintPluginOptions = {
  config: string | import('eslint').Linter.Config;
};

export function eslintPlugin(options: ESLintPluginOptions): PluginConfig {
  // ... load configuration, etc. ...
  return {
    meta: {
      slug: 'eslint',
      name: 'ESLint',
      type: 'static-analysis',
      icon: 'eslint',
      docsUrl: 'https://pushup.github.io/code-pushup/plugins/eslint',
    },
    runner: {
      // inputs via environment variables (e.g. process.env.CPU_AFFECTED_FILES)
      command: 'node node_modules/@cpu/eslint-plugin/bin/main.js',
    },
    audits: [
      {
        slug: '@typescript-eslint/no-explicit-any',
        label: 'no-explicit-any',
        title: 'Disallow the `any` type',
        docsUrl: 'https://typescript-eslint.io/rules/no-explicit-any/',
      },
      {
        slug: 'max-lines-200', // options part of ID (accurate comparisons)
        label: 'max-lines',
        title: 'Enforce a maximum number of lines per file',
        docsUrl: 'https://eslint.org/docs/latest/rules/max-lines',
      },
      // ...
    ],
  };
}

/* Lighthouse CI plugin */

type LHCIPluginOptions = {
  config: string;
};

export function lhciPlugin(options: LHCIPluginOptions): PluginConfig {
  // ... load configuration, etc. ...
  return {
    meta: {
      slug: 'lhci',
      name: 'Lighthouse CI',
      type: 'performance-measurements',
      icon: 'lighthouse',
      docsUrl: 'https://pushup.github.io/code-pushup/plugins/lhci'
    },
    runner: {
      outputPath: './dist/cpu/runner-outputs/lhci-plugin.json',
      command: 'npx lhci autorun',
    },
    audits: [
      {
        slug: 'largest-contentful-paint',
        label: 'LCP',
        title: 'Largest Contentful Paint',
        docsUrl:
          'https://developer.chrome.com/docs/lighthouse/performance/lighthouse-largest-contentful-paint/',
      },
      // ...
    ],
    groups: [
      {
        slug: 'performance',
        title: 'Lighthouse Performance',
        audits: [
          { ref: 'largest-contentful-paint', weight: 3 },
          // ...
        ],
      },
    ],
  };
}

Process definition of a plugin execution

To .... we need:

• outPath - name of /dist/plugin-a/runner-output.json
• format - JSON === RunnerOutput

Type definitions for plugin output

// JSON formatted output emitted by runner
export type RunnerOutput = {
  audits: Audit[];
};

type Audit = {
  /** references audit metadata */
  slug: string;
  /** formatted value (e.g. '0.9 s', '2.1 MB') */
  displayValue?: string;
  /** raw numeric value (defaults to score ?? details.warnings.length) */
  value?: number;
  /** value between 0 and 1 (defaults to Number(details.warnings.length === 0)) */
  score?: number;
  /** detailed information */
  details?: {
    /** list of findings */
    issues: Issue[];
  };
};

type Issue = {
  /** descriptive error message */
  message: string;
  /** severity level */
  severity: IssueSeverity;
  /** reference to source code */
  source?: SourceFileLocation; // if applicable (linter, unit test)
  // TODO: other context data
};

type IssueSeverity = 'info' | 'warning' | 'error';

type SourceFileLocation = {
  /** relative path to source file in Git repo */
  file: string;
  /** location in file */
  position?: {
    startLine: number;
    startColumn?: number;
    endLine?: number;
    endColumn?: number;
  };
};

Plugin output example

const eslintOutput: RunnerOutput = {
  audits: [
    {
      slug: '@typescript-eslint/no-explicit-any',
      score: 0,
      value: 2,
      details: {
        issues: [
          {
            message: 'Unexpected any. Specify a different type.',
            severity: 'warning',
            source: {
              file: 'src/utils.ts',
              position: {
                startLine: 5,
                startColumn: 10,
                endLine: 5,
                endColumn: 13,
              },
            },
          },
          {
            message: 'Unexpected any. Specify a different type.',
            severity: 'warning',
            source: {
              file: 'src/utils.ts',
              position: {
                startLine: 34,
                startColumn: 8,
                endLine: 34,
                endColumn: 11,
              },
            },
          },
        ],
      },
    },
    // ...
  ],
};

const lhciOutput: RunnerOutput = {
  audits: [
    {
      slug: 'largest-contentful-paint',
      value: 1100,
      score: 0.9,
      details: {
        issues: [
          { message: 'Avoid chaining critical requests', severity: 'info' },
        ],
      },
    },
    // ...
  ],
};