SCANNPM-10 Handle logic to process scanner properties from various so…

…urces with priority

src/constants.ts

@@ -18,6 +18,7 @@
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 import path from 'path';
+import { ScannerProperty } from './types';
 
 export const SCANNER_BOOTSTRAPPER_NAME = 'ScannerNpm';
 
@@ -33,3 +34,14 @@ export const SONAR_CACHE_DIR = path.join(
 );
 
 export const UNARCHIVE_SUFFIX = '_extracted';
+
+export const ENV_VAR_PREFIX = 'SONAR_SCANNER_';
+
+export const ENV_TO_PROPERTY_NAME: [string, ScannerProperty][] = [
+  ['SONAR_TOKEN', ScannerProperty.SonarToken],
+  ['SONAR_HOST_URL', ScannerProperty.SonarHostUrl],
+  ['SONAR_USER_HOME', ScannerProperty.SonarUserHome],
+  ['SONAR_ORGANIZATION', ScannerProperty.SonarOrganization],
+];
+
+export const SONAR_PROJECT_FILENAME = 'sonar-project.properties';

src/properties.ts

@@ -0,0 +1,292 @@
+/*
+ * sonar-scanner-npm
+ * Copyright (C) 2022-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+import { ScannerProperties, ScannerProperty } from './types';
+import {
+  ENV_TO_PROPERTY_NAME,
+  ENV_VAR_PREFIX,
+  SCANNER_BOOTSTRAPPER_NAME,
+  SONAR_PROJECT_FILENAME,
+} from './constants';
+import { version } from '../package.json';
+import { ScanOptions } from './scan';
+import path from 'path';
+import fs from 'fs';
+import { LogLevel, log } from './logging';
+import slugify from 'slugify';
+
+/**
+ * Convert the name of a sonar property from its environment variable form
+ * (eg SONAR_SCANNER_FOO_BAR) to its sonar form (eg sonar.scanner.fooBar).
+ */
+function envNameToSonarPropertyName(envName: string) {
+  // Extract the name and convert to camel case
+  const sonarScannerKey = envName
+    .substring(ENV_VAR_PREFIX.length)
+    .toLowerCase()
+    .replace(/_([a-z])/g, g => g[1].toUpperCase());
+  return `sonar.scanner.${sonarScannerKey}`;
+}
+
+/**
+ * Build the config.
+ */
+export function getPackageJsonProperties(
+  projectBaseDir: string,
+  sonarBaseExclusions?: string,
+): ScannerProperties {
+  const packageJsonParams: { [key: string]: string } = {};
+  const packageFile = path.join(projectBaseDir, 'package.json');
+  const packageData = fs.readFileSync(packageFile).toString();
+  const pkg = JSON.parse(packageData);
+  log(LogLevel.INFO, 'Retrieving info from "package.json" file');
+  function fileExistsInProjectSync(file: string) {
+    return fs.existsSync(path.resolve(projectBaseDir, file));
+  }
+  function dependenceExists(pkgName: string) {
+    return ['devDependencies', 'dependencies', 'peerDependencies'].some(function (prop) {
+      return pkg[prop] && pkgName in pkg[prop];
+    });
+  }
+  if (pkg) {
+    const invalidCharacterRegex = /[?$*+~.()'"!:@/]/g;
+    packageJsonParams['sonar.projectKey'] = slugify(pkg.name, {
+      remove: invalidCharacterRegex,
+    });
+    packageJsonParams['sonar.projectName'] = pkg.name;
+    packageJsonParams['sonar.projectVersion'] = pkg.version;
+    if (pkg.description) {
+      packageJsonParams['sonar.projectDescription'] = pkg.description;
+    }
+    if (pkg.homepage) {
+      packageJsonParams['sonar.links.homepage'] = pkg.homepage;
+    }
+    if (pkg.bugs?.url) {
+      packageJsonParams['sonar.links.issue'] = pkg.bugs.url;
+    }
+    if (pkg.repository?.url) {
+      packageJsonParams['sonar.links.scm'] = pkg.repository.url;
+    }
+
+    const potentialCoverageDirs = [
+      // jest coverage output directory
+      // See: http://facebook.github.io/jest/docs/en/configuration.html#coveragedirectory-string
+      pkg['nyc']?.['report-dir'],
+      // nyc coverage output directory
+      // See: https://github.com/istanbuljs/nyc#configuring-nyc
+      pkg['jest']?.['coverageDirectory'],
+    ]
+      .filter(Boolean)
+      .concat(
+        // default coverage output directory
+        'coverage',
+      );
+    const uniqueCoverageDirs = Array.from(new Set(potentialCoverageDirs));
+    packageJsonParams['sonar.exclusions'] = sonarBaseExclusions ?? '';
+    for (const lcovReportDir of uniqueCoverageDirs) {
+      const lcovReportPath = path.posix.join(lcovReportDir, 'lcov.info');
+      if (fileExistsInProjectSync(lcovReportPath)) {
+        packageJsonParams['sonar.exclusions'] += ',' + path.posix.join(lcovReportDir, '**');
+        // https://docs.sonarqube.org/display/PLUG/JavaScript+Coverage+Results+Import
+        packageJsonParams['sonar.javascript.lcov.reportPaths'] = lcovReportPath;
+        // TODO: use Generic Test Data to remove dependence of SonarJS, it is need transformation lcov to sonar generic coverage format
+      }
+    }
+
+    if (dependenceExists('mocha-sonarqube-reporter') && fileExistsInProjectSync('xunit.xml')) {
+      // https://docs.sonarqube.org/display/SONAR/Generic+Test+Data
+      packageJsonParams['sonar.testExecutionReportPaths'] = 'xunit.xml';
+    }
+    // TODO: use `glob` to lookup xunit format files and transformation to sonar generic report format
+  }
+  return packageJsonParams;
+}
+
+/**
+ * Convert CLI args into scanner properties.
+ */
+export function getCommandLineProperties(cliArgs?: string[]): ScannerProperties {
+  if (!cliArgs || cliArgs.length === 0) {
+    return {};
+  }
+
+  // Parse CLI args (eg: -Dsonar.token=xxx)
+  const properties: ScannerProperties = {};
+  for (const arg of cliArgs) {
+    if (!arg.startsWith('-D')) {
+      continue;
+    }
+    const [key, value] = arg.substring(2).split('=');
+    properties[key] = value;
+  }
+
+  return properties;
+}
+
+/**
+ * Parse properties stored in sonar project properties file, if it exists.
+ */
+export function getSonarFileProperties(projectBaseDir: string): ScannerProperties {
+  // Read sonar project properties file in project base dir
+  const sonarPropertiesFile = path.join(projectBaseDir, SONAR_PROJECT_FILENAME);
+  if (!fs.existsSync(sonarPropertiesFile)) {
+    return {};
+  }
+
+  try {
+    const properties: ScannerProperties = {};
+    const data = fs.readFileSync(sonarPropertiesFile).toString();
+    const lines = data.split('\n');
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (trimmedLine.length === 0 || trimmedLine.startsWith('#')) {
+        continue;
+      }
+      const [key, value] = trimmedLine.split('=');
+      properties[key] = value;
+    }
+
+    return properties;
+  } catch (error: any) {
+    log(LogLevel.WARN, `Failed to read ${SONAR_PROJECT_FILENAME} file: ${error.message}`);
+    return {};
+  }
+}
+
+/**
+ * Get scanner properties from scan option object (JS API).
+ */
+export function getScanOptionsProperties(scanOptions: ScanOptions): ScannerProperties {
+  const options = {
+    ...scanOptions.options,
+  };
+
+  if (typeof scanOptions.serverUrl !== 'undefined') {
+    options[ScannerProperty.SonarHostUrl] = scanOptions.serverUrl;
+  }
+
+  if (typeof scanOptions.token !== 'undefined') {
+    options[ScannerProperty.SonarToken] = scanOptions.token;
+  }
+
+  if (typeof scanOptions.verbose !== 'undefined') {
+    options[ScannerProperty.SonarVerbose] = scanOptions.verbose ? 'true' : 'false';
+  }
+
+  return options;
+}
+
+/**
+ * Automatically parse properties from environment variables.
+ */
+export function getEnvironmentProperties() {
+  const { env } = process;
+
+  let properties: ScannerProperties = {};
+
+  // Get known environment variables
+  for (const [envName, scannerProperty] of ENV_TO_PROPERTY_NAME) {
+    if (envName in env) {
+      properties[scannerProperty] = env[envName] as string;
+    }
+  }
+
+  // Get generic environment variables
+  properties = {
+    ...properties,
+    ...Object.fromEntries(
+      Object.entries(env)
+        .filter(([key]) => key.startsWith(ENV_VAR_PREFIX))
+        .map(([key, value]) => [envNameToSonarPropertyName(key), value as string]),
+    ),
+  };
+
+  // Get JSON parameters from env
+  try {
+    const jsonParams = env.SONAR_SCANNER_JSON_PARAMS ?? env.SONARQUBE_SCANNER_PARAMS;
+    if (jsonParams) {
+      properties = {
+        ...properties,
+        ...JSON.parse(jsonParams),
+      };
+    }
+    // TODO: Do we need to keep this warning, as we're likely doing a breaking change anyway?
+    if (!env.SONAR_SCANNER_JSON_PARAMS && env.SONARQUBE_SCANNER_PARAMS) {
+      console.warn(
+        'SONARQUBE_SCANNER_PARAMS is deprecated, please use SONAR_SCANNER_JSON_PARAMS instead',
+      );
+    }
+  } catch (e) {
+    // Ignore
+    console.warn(`Failed to parse JSON parameters from ENV: ${e}`);
+  }
+
+  return properties;
+}
+
+/**
+ * Get bootstrapper properties, that can not be overridden.
+ */
+export function getBootstrapperProperties(startTimestampMs: number): ScannerProperties {
+  return {
+    [ScannerProperty.SonarScannerApp]: SCANNER_BOOTSTRAPPER_NAME,
+    [ScannerProperty.SonarScannerAppVersion]: version,
+    [ScannerProperty.SonarScannerBootstrapStartTime]: startTimestampMs.toString(),
+    // Bootstrap cache hit/miss is set later after the bootstrapper has run and before scanner engine is started
+  };
+}
+
+export function getProperties(
+  scanOptions: ScanOptions,
+  startTimestampMs: number,
+  cliArgs?: string[],
+): ScannerProperties {
+  const bootstrapperProperties = getBootstrapperProperties(startTimestampMs);
+  const cliProperties = getCommandLineProperties(cliArgs);
+  const scanOptionsProperties = getScanOptionsProperties(scanOptions);
+  const envProperties = getEnvironmentProperties();
+
+  // Compute default base dir respecting order of precedence we use for the final merge
+  const projectBaseDir =
+    cliProperties[ScannerProperty.SonarProjectBaseDir] ??
+    scanOptionsProperties[ScannerProperty.SonarProjectBaseDir] ??
+    envProperties[ScannerProperty.SonarProjectBaseDir] ??
+    process.cwd();
+
+  const sonarFileProperties = getSonarFileProperties(projectBaseDir);
+
+  const baseSonarExclusions =
+    cliProperties[ScannerProperty.SonarExclusions] ??
+    scanOptionsProperties[ScannerProperty.SonarExclusions] ??
+    sonarFileProperties[ScannerProperty.SonarExclusions] ??
+    envProperties[ScannerProperty.SonarExclusions];
+  const packageJsonProperties = getPackageJsonProperties(projectBaseDir, baseSonarExclusions);
+
+  // Merge properties respecting order of precedence
+  return [
+    bootstrapperProperties, // Can't be overridden
+    cliProperties, // Highest precedence
+    scanOptionsProperties,
+    sonarFileProperties,
+    packageJsonProperties,
+    envProperties, // Lowest precedence
+  ]
+    .reverse()
+    .reduce((acc, curr) => ({ ...acc, ...curr }), {});
+}

src/scan.ts

@@ -18,6 +18,10 @@
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
+import { setLogLevel } from './logging';
+import { getProperties } from './properties';
+import { ScannerProperty } from './types';
+
 export type ScanOptions = {
   serverUrl: string;
   token: string;
@@ -29,5 +33,13 @@ export type ScanOptions = {
 };
 
 export async function scan(scanOptions: ScanOptions, cliArgs?: string[]) {
+  const startTimestampMs = Date.now();
+  const properties = getProperties(scanOptions, startTimestampMs, cliArgs);
+
+  setLogLevel(properties[ScannerProperty.SonarVerbose]);
+
   // TODO: NPMSCAN-2 new bootstrapper sequence
+  // ...
+  properties[ScannerProperty.SonarScannerBootstrapEngineCacheHit] = 'false';
+  // ...
 }

src/types.ts

@@ -38,4 +38,20 @@ export type ScannerLogEntry = {
   throwable?: string;
 };
 
-export type ScannerParams = { [key: string]: string };
+export enum ScannerProperty {
+  SonarVerbose = 'sonar.verbose',
+  SonarToken = 'sonar.token',
+  SonarExclusions = 'sonar.exclusions',
+  SonarHostUrl = 'sonar.host.url',
+  SonarUserHome = 'sonar.userHome',
+  SonarOrganization = 'sonar.organization',
+  SonarScannerApp = 'sonar.scanner.app',
+  SonarScannerAppVersion = 'sonar.scanner.appVersion',
+  SonarScannerBootstrapStartTime = 'sonar.scanner.bootstrapStartTime',
+  SonarProjectBaseDir = 'sonar.projectBaseDir',
+  SonarScannerBootstrapEngineCacheHit = 'sonar.scanner.bootstrapEngineCacheHit',
+}
+
+export type ScannerProperties = {
+  [key: string]: string;
+};