Skip to content

Commit

Permalink
feat: Implement Yarn Berry Analyser
Browse files Browse the repository at this point in the history
  • Loading branch information
@segovia
segovia committed Jan 28, 2025
1 parent deb5b6d commit 04ebeb5
Show file tree
Hide file tree
Showing 17 changed files with 11,275 additions and 128 deletions.
172 changes: 56 additions & 116 deletions ...encycheck/analyzer/YarnAuditAnalyzer.java → ...k/analyzer/AbstractYarnAuditAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,36 @@

import org.apache.commons.collections4.MultiValuedMap;
import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.analyzer.exception.SearchException;
import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException;
import org.owasp.dependencycheck.data.nodeaudit.Advisory;
import org.owasp.dependencycheck.data.nodeaudit.NpmPayloadBuilder;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.exception.InitializationException;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.URLConnectionFailureException;
import org.owasp.dependencycheck.utils.processing.ProcessReader;
import org.semver4j.Semver;
import org.semver4j.SemverException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.springett.parsers.cpe.exceptions.CpeValidationException;

import javax.annotation.concurrent.ThreadSafe;
import jakarta.json.Json;
import jakarta.json.JsonException;
import jakarta.json.JsonObject;
import jakarta.json.JsonReader;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@ThreadSafe
public class YarnAuditAnalyzer extends AbstractNpmAnalyzer {
public abstract class AbstractYarnAuditAnalyzer extends AbstractNpmAnalyzer {

/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(YarnAuditAnalyzer.class);
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractYarnAuditAnalyzer.class);
protected static final int YARN_CLASSIC_MAJOR_VERSION = 1;

/**
* The file name to scan.
Expand All @@ -71,23 +62,39 @@ public class YarnAuditAnalyzer extends AbstractNpmAnalyzer {
.addFilenames(YARN_PACKAGE_LOCK).build();

/**
* An expected error from `yarn audit --offline --verbose --json` that will
* be ignored.
* The path to the `yarn` executable.
*/
private static final String EXPECTED_ERROR = "{\"type\":\"error\",\"data\":\"Can't make a request in "
+ "offline mode (\\\"https://registry.yarnpkg.com/-/npm/v1/security/audits\\\")\"}\n";
private String yarnPath;

/**
* The path to the `yarn` executable.
* The version of the `yarn` executable.
*/
private String yarnPath;
private String yarnVersion;


/**
* Extracts the major version from a version string.
*
* @return the major version (e.g., `4` from "4.2.1")
*/
protected int getYarnMajorVersion() {
if (StringUtils.isBlank(yarnVersion)) {
throw new IllegalArgumentException("Version string cannot be null or empty");
}
try {
var semver = new Semver(yarnVersion);
return semver.getMajor();
} catch (SemverException e) {
throw new IllegalArgumentException("Invalid version string format", e);
}
}

/**
* Analyzes the yarn lock file to determine vulnerable dependencies. Uses
* yarn audit --offline to generate the payload to be sent to the NPM API.
*
* @param dependency the yarn lock file
* @param engine the analysis engine
* @param engine the analysis engine
* @throws AnalysisException thrown if there is an error analyzing the file
*/
@Override
Expand All @@ -110,6 +117,20 @@ protected void analyzeDependency(Dependency dependency, Engine engine) throws An
}
}

/**
* Analyzes the package.
*
* @param lockFile a reference to the package-lock.json
* @param packageFile a reference to the package.json
* @param dependency a reference to the dependency-object for the yarn.lock
* @param dependencyMap a collection of module/version pairs; during
* creation of the payload the dependency map is populated with the
* module/version information.
* @return a list of advisories
* @throws AnalysisException thrown when there is an error creating or submitting the npm audit
*/
protected abstract List<Advisory> analyzePackage(File lockFile, File packageFile, Dependency dependency, MultiValuedMap<String, String> dependencyMap) throws AnalysisException;

@Override
protected String getAnalyzerEnabledSettingKey() {
return Settings.KEYS.ANALYZER_YARN_AUDIT_ENABLED;
Expand All @@ -120,11 +141,6 @@ protected FileFilter getFileFilter() {
return LOCK_FILE_FILTER;
}

@Override
public String getName() {
return "Yarn Audit Analyzer";
}

@Override
public AnalysisPhase getAnalysisPhase() {
return AnalysisPhase.FINDING_ANALYSIS;
Expand All @@ -145,7 +161,7 @@ protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationExcep
}
final List<String> args = new ArrayList<>();
args.add(getYarn());
args.add("--help");
args.add("--version");
final ProcessBuilder builder = new ProcessBuilder(args);
LOGGER.debug("Launching: {}", args);
try {
Expand All @@ -158,6 +174,11 @@ protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationExcep
switch (exitValue) {
case expectedExitValue:
LOGGER.debug("{} is enabled.", getName());
yarnVersion = processReader.getOutput();
if (StringUtils.isBlank(yarnVersion)) {
this.setEnabled(false);
LOGGER.warn("The {} has been disabled. Yarn version could not be determined.", getName());
}
break;
case yarnExecutableNotFoundExitValue:
default:
Expand All @@ -177,7 +198,7 @@ protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationExcep
*
* @return the path to `yarn`
*/
private String getYarn() {
protected String getYarn() {
final String value;
synchronized (this) {
if (yarnPath == null) {
Expand All @@ -199,54 +220,24 @@ private String getYarn() {
return value;
}

private JsonObject fetchYarnAuditJson(Dependency dependency, boolean skipDevDependencies) throws AnalysisException {
final File folder = dependency.getActualFile().getParentFile();
if (!folder.isDirectory()) {
throw new AnalysisException(String.format("%s should have been a directory.", folder.getAbsolutePath()));
}
/**
* Workaround 64k limitation of InputStream, redirect stdout to a file that we will read later
* instead of reading directly stdout from Process's InputStream which is topped at 64k
*/
protected String startAndReadStdoutToString(ProcessBuilder builder) throws AnalysisException {
try {
final List<String> args = new ArrayList<>();

args.add(getYarn());
args.add("audit");
//offline audit is not supported - but the audit request is generated in the verbose output
args.add("--offline");
if (skipDevDependencies) {
args.add("--groups");
args.add("dependencies");
}
args.add("--json");
args.add("--verbose");
final ProcessBuilder builder = new ProcessBuilder(args);
builder.directory(folder);
LOGGER.debug("Launching: {}", args);
// Workaround 64k limitation of InputStream, redirect stdout to a file that we will read later
// instead of reading directly stdout from Process's InputStream which is topped at 64k

final File tmpFile = getSettings().getTempFile("yarn_audit", "json");
builder.redirectOutput(tmpFile);
final Process process = builder.start();
try (ProcessReader processReader = new ProcessReader(process)) {
processReader.readAll();
final String errOutput = processReader.getError();

if (!StringUtils.isBlank(errOutput) && !EXPECTED_ERROR.equals(errOutput)) {
if (!StringUtils.isBlank(errOutput)) {
LOGGER.debug("Process Error Out: {}", errOutput);
LOGGER.debug("Process Out: {}", processReader.getOutput());
}
final String verboseJson = new String(Files.readAllBytes(tmpFile.toPath()), StandardCharsets.UTF_8);
final String auditRequestJson = Arrays.stream(verboseJson.split("\n"))
.filter(line -> line.contains("Audit Request"))
.findFirst().get();
String auditRequest;
try (JsonReader reader = Json.createReader(IOUtils.toInputStream(auditRequestJson, StandardCharsets.UTF_8))) {
final JsonObject jsonObject = reader.readObject();
auditRequest = jsonObject.getString("data");
auditRequest = auditRequest.substring(15);
}
LOGGER.debug("Audit Request: {}", auditRequest);

return Json.createReader(IOUtils.toInputStream(auditRequest, StandardCharsets.UTF_8)).readObject();
return new String(Files.readAllBytes(tmpFile.toPath()), StandardCharsets.UTF_8);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new AnalysisException("Yarn audit process was interrupted.", ex);
Expand All @@ -255,55 +246,4 @@ private JsonObject fetchYarnAuditJson(Dependency dependency, boolean skipDevDepe
throw new AnalysisException("yarn audit failure; this error can be ignored if you are not analyzing projects with a yarn lockfile.", ioe);
}
}

/**
* Analyzes the package and yarn lock files by extracting dependency
* information, creating a payload to submit to the npm audit API,
* submitting the payload, and returning the identified advisories.
*
* @param lockFile a reference to the package-lock.json
* @param packageFile a reference to the package.json
* @param dependency a reference to the dependency-object for the yarn.lock
* @param dependencyMap a collection of module/version pairs; during
* creation of the payload the dependency map is populated with the
* module/version information.
* @return a list of advisories
* @throws AnalysisException thrown when there is an error creating or
* submitting the npm audit API payload
*/
private List<Advisory> analyzePackage(final File lockFile, final File packageFile,
Dependency dependency, MultiValuedMap<String, String> dependencyMap)
throws AnalysisException {
try {
final boolean skipDevDependencies = getSettings().getBoolean(Settings.KEYS.ANALYZER_NODE_AUDIT_SKIPDEV, false);
// Retrieves the contents of package-lock.json from the Dependency
final JsonObject lockJson = fetchYarnAuditJson(dependency, skipDevDependencies);
// Retrieves the contents of package-lock.json from the Dependency
final JsonObject packageJson;
try (JsonReader packageReader = Json.createReader(Files.newInputStream(packageFile.toPath()))) {
packageJson = packageReader.readObject();
}
// Modify the payload to meet the NPM Audit API requirements
final JsonObject payload = NpmPayloadBuilder.build(lockJson, packageJson, dependencyMap, skipDevDependencies);

// Submits the package payload to the nsp check service
return getSearcher().submitPackage(payload);

} catch (URLConnectionFailureException e) {
this.setEnabled(false);
throw new AnalysisException("Failed to connect to the NPM Audit API (YarnAuditAnalyzer); the analyzer "
+ "is being disabled and may result in false negatives.", e);
} catch (IOException e) {
LOGGER.debug("Error reading dependency or connecting to NPM Audit API", e);
this.setEnabled(false);
throw new AnalysisException("Failed to read results from the NPM Audit API (YarnAuditAnalyzer); "
+ "the analyzer is being disabled and may result in false negatives.", e);
} catch (JsonException e) {
throw new AnalysisException(String.format("Failed to parse %s file from the NPM Audit API "
+ "(YarnAuditAnalyzer).", lockFile.getPath()), e);
} catch (SearchException ex) {
LOGGER.error("YarnAuditAnalyzer failed on {}", dependency.getActualFilePath());
throw ex;
}
}
}
3 changes: 2 additions & 1 deletion core/src/main/java/org/owasp/dependencycheck/analyzer/AnalysisPhase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ public enum AnalysisPhase {
* {@link NvdCveAnalyzer}
* {@link PnpmAuditAnalyzer}
* {@link RetireJsAnalyzer}
* {@link YarnAuditAnalyzer}
* {@link YarnClassicAuditAnalyzer}
* {@link YarnBerryAuditAnalyzer}
*
*/
FINDING_ANALYSIS,
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ protected String getAnalyzerEnabledSettingKey() {
*/
private boolean isNodeAuditEnabled(Engine engine) {
for (Analyzer a : engine.getAnalyzers()) {
if (a instanceof NodeAuditAnalyzer || a instanceof YarnAuditAnalyzer || a instanceof PnpmAuditAnalyzer) {
if (a instanceof NodeAuditAnalyzer || a instanceof AbstractYarnAuditAnalyzer || a instanceof PnpmAuditAnalyzer) {
if (a.isEnabled()) {
try {
((AbstractNpmAnalyzer) a).prepareFileTypeAnalyzer(engine);
Expand Down
Loading

0 comments on commit 04ebeb5

Please sign in to comment.