Skip to content

Commit

Permalink
exim: Use har-reader
Browse files Browse the repository at this point in the history 
- CHANGELOG > Added change note.
- build file > Updated/added dependencies.
- HarImporter > Re-worked to use har-reader lib.
- HarUtils > Extensively added to to facilitate use of the new lib.
- MenuImportHar > Re-worked and simplified to take advantage of the new
utils, importer, and lib.
- HarImporterUnitTest > Tweaked to accommodate HarImporter changes.

Signed-off-by: kingthorin <kingthorin@users.noreply.github.com>
  • Loading branch information
@kingthorin
kingthorin committed May 23, 2024
1 parent 54f0e51 commit d3ef666
Show file tree
Hide file tree
Showing 11 changed files with 1,160 additions and 61 deletions.
3 changes: 2 additions & 1 deletion addOns/exim/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ All notable changes to this add-on will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## Unreleased

### Changed
- HAR importing now uses Sebastian Stöhr's har-reader library. It should be much more tolerant of 'weird' HAR things, and thus be able to import more samples. (If you come across HAR that won't import please open an issue and provide a sample so we can work on further improvements!)

## [0.9.0] - 2024-05-07
### Added
Expand Down
2 changes: 2 additions & 0 deletions addOns/exim/exim.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ dependencies {
zapAddOn("commonlib")

implementation(files("lib/org.jwall.web.audit-0.2.15.jar"))
implementation("de.sstoehr:har-reader:2.3.0")

testImplementation(parent!!.childProjects.get("commonlib")!!.sourceSets.test.get().output)
testImplementation(project(":testutils"))
testImplementation(libs.log4j.core)
}
184 changes: 148 additions & 36 deletions addOns/exim/src/main/java/org/zaproxy/addon/exim/har/HarImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,59 +19,160 @@
*/
package org.zaproxy.addon.exim.har;

import edu.umass.cs.benchlab.har.HarContent;
import edu.umass.cs.benchlab.har.HarEntries;
import edu.umass.cs.benchlab.har.HarEntry;
import edu.umass.cs.benchlab.har.HarHeader;
import edu.umass.cs.benchlab.har.HarLog;
import edu.umass.cs.benchlab.har.HarResponse;
import edu.umass.cs.benchlab.har.tools.HarFileReader;
import de.sstoehr.harreader.HarReader;
import de.sstoehr.harreader.HarReaderException;
import de.sstoehr.harreader.model.HarContent;
import de.sstoehr.harreader.model.HarEntry;
import de.sstoehr.harreader.model.HarHeader;
import de.sstoehr.harreader.model.HarLog;
import de.sstoehr.harreader.model.HarResponse;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.parosproxy.paros.Constant;
import org.parosproxy.paros.control.Control;
import org.parosproxy.paros.extension.history.ExtensionHistory;
import org.parosproxy.paros.model.HistoryReference;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.network.HttpHeader;
import org.parosproxy.paros.network.HttpMalformedHeaderException;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpResponseHeader;
import org.zaproxy.addon.commonlib.http.HttpFieldsNames;
import org.zaproxy.addon.commonlib.ui.ProgressPaneListener;
import org.zaproxy.addon.exim.ExtensionExim;
import org.zaproxy.zap.network.HttpResponseBody;
import org.zaproxy.zap.utils.HarUtils;
import org.zaproxy.zap.utils.Stats;
import org.zaproxy.zap.utils.ThreadUtils;

public class HarImporter {

private static final Logger LOGGER = LogManager.getLogger(HarImporter.class);
private static final String STATS_HAR_FILE = "import.har.file";
private static final String STATS_HAR_FILE_ERROR = "import.har.file.errors";
private static final String STATS_HAR_FILE_MSG = "import.har.file.message";
private static final String STATS_HAR_FILE_MSG_ERROR = "import.har.file.message.errors";
// The following list is ordered to hopefully match quickly
private static final List<String> ACCEPTED_VERSIONS =
List.of(HttpHeader.HTTP11, HttpHeader.HTTP2, HttpHeader.HTTP10, HttpHeader.HTTP09);

protected static final String STATS_HAR_FILE_ERROR = "import.har.file.errors";

private static ExtensionHistory extHistory;

private ProgressPaneListener progressListener;
private boolean success;
private static ExtensionHistory extHistory;

public HarImporter(File file) {
this(file, null);
}

public HarImporter(File file, ProgressPaneListener listener) {
this.progressListener = listener;
importHarFile(file);
HarLog log = null;
try {
log = new HarReader().readFromFile(file).getLog();
importHarLog(log);
} catch (HarReaderException e) {
LOGGER.warn(
"Failed to read HAR file: {} \n {}", file.getAbsolutePath(), e.getMessage());
Stats.incCounter(ExtensionExim.STATS_PREFIX + STATS_HAR_FILE_ERROR);
success = false;
completed();
return;
}
completed();
}

static List<HttpMessage> getHttpMessages(HarLog log) throws HttpMalformedHeaderException {
public HarImporter(HarLog harLog, ProgressPaneListener listener) {
this.progressListener = listener;
importHarLog(harLog);
completed();
}

private void importHarLog(HarLog log) {
processMessages(log);
Stats.incCounter(ExtensionExim.STATS_PREFIX + STATS_HAR_FILE);
success = true;
}

private void processMessages(HarLog log) {
if (log == null) {
return;
}

List<HttpMessage> messages = null;
int before = log.getEntries().size();
preProcessHarLog(log);
int adjustment = before - log.getEntries().size();
try {
messages = HarImporter.getHttpMessages(log);
} catch (HttpMalformedHeaderException e) {
LOGGER.warn("Failed to process HAR entries. {}", e.getMessage());
LOGGER.debug(e, e);
Stats.incCounter(ExtensionExim.STATS_PREFIX + STATS_HAR_FILE_ERROR);
completed();
return;
}
int count = adjustment;
for (HttpMessage msg : messages) {
if (msg == null) {
continue;
}
persistMessage(msg);
updateProgress(++count, msg.getRequestHeader().getURI().toString());
}
}

private HarLog preProcessHarLog(HarLog log) {
List<HarEntry> entries =
log.getEntries().stream()
.filter(e -> entryHasUsableHttpVersion(e))
.filter(f -> entryIsNotLocalPrivate(f))
.collect(Collectors.toList());
for (HarEntry entry : entries) {
entry.getResponse().getHeaders().forEach(this::preProcessHarHeaders);
}
log.setEntries(entries);
return log;
}

private static boolean entryHasUsableHttpVersion(HarEntry entry) {
preProcessEntryMissingHttpVersion(entry, entry.getRequest().getHttpVersion(), false);
preProcessEntryMissingHttpVersion(entry, entry.getResponse().getHttpVersion(), true);

if (!containsIgnoreCase(ACCEPTED_VERSIONS, entry.getRequest().getHttpVersion())
|| !containsIgnoreCase(ACCEPTED_VERSIONS, entry.getResponse().getHttpVersion())) {
LOGGER.warn(
"Message with unsupported HTTP version (Req version: {}, Resp version: {}) will be dropped: {}",
entry.getRequest().getHttpVersion(),
entry.getResponse().getHttpVersion(),
entry.getRequest().getUrl());
return false;
}
return true;
}

private static boolean entryIsNotLocalPrivate(HarEntry entry) {
String url = entry.getRequest().getUrl();
if (StringUtils.startsWithIgnoreCase(url, "about")
|| StringUtils.startsWithIgnoreCase(url, "chrome")
|| StringUtils.startsWithIgnoreCase(url, "edge")) {
LOGGER.debug("Skipping local private entry: {}", url);
return false;
}
return true;
}

protected static List<HttpMessage> getHttpMessages(HarLog log)
throws HttpMalformedHeaderException {
List<HttpMessage> result = new ArrayList<>();
HarEntries entries = log.getEntries();
for (HarEntry entry : entries.getEntries()) {
List<HarEntry> entries = log.getEntries();
for (HarEntry entry : entries) {
result.add(getHttpMessage(entry));
}
return result;
Expand Down Expand Up @@ -101,7 +202,7 @@ private static void setHttpResponse(HarResponse harResponse, HttpMessage message
.append(harResponse.getStatusText())
.append("\r\n");

for (HarHeader harHeader : harResponse.getHeaders().getHeaders()) {
for (HarHeader harHeader : harResponse.getHeaders()) {
strBuilderResHeader
.append(harHeader.getName())
.append(": ")
Expand All @@ -118,29 +219,40 @@ private static void setHttpResponse(HarResponse harResponse, HttpMessage message
}
}

private void importHarFile(File file) {
try {
processMessages(file);
Stats.incCounter(ExtensionExim.STATS_PREFIX + STATS_HAR_FILE);
success = true;
} catch (IOException e) {
LOGGER.warn(
Constant.messages.getString(
ExtensionExim.EXIM_OUTPUT_ERROR, file.getAbsolutePath()));
LOGGER.warn(e);
Stats.incCounter(ExtensionExim.STATS_PREFIX + STATS_HAR_FILE_ERROR);
success = false;
private static boolean containsIgnoreCase(List<String> checkList, String candidate) {
return checkList.stream().anyMatch(e -> e.equalsIgnoreCase(candidate));
}

private static void preProcessEntryMissingHttpVersion(
HarEntry entry, String vers, boolean response) {
if (vers == null || vers.isEmpty() || vers.equalsIgnoreCase(HttpHeader.HTTP)) {
if (response) {
entry.getResponse().setHttpVersion(HttpHeader.HTTP11);
} else {
entry.getRequest().setHttpVersion(HttpHeader.HTTP11);
}
LOGGER.info(
"Setting {} version to {} for {}",
response ? "response" : "request",
response
? entry.getResponse().getHttpVersion()
: entry.getRequest().getHttpVersion(),
entry.getRequest().getUrl());
}
}

private void processMessages(File file) throws IOException {
List<HttpMessage> messages =
HarImporter.getHttpMessages(new HarFileReader().readHarFile(file));
int count = 1;
for (HttpMessage msg : messages) {
persistMessage(msg);
updateProgress(count, msg.getRequestHeader().getURI().toString());
count++;
private void preProcessHarHeaders(HarHeader header) {
String name = header.getName();
String value = header.getValue();
if ((name.equalsIgnoreCase(HttpFieldsNames.CACHE_CONTROL)
|| name.equalsIgnoreCase(HttpFieldsNames.SET_COOKIE))
&& (value.contains("\n") || value.contains("\r"))) {
header.setValue(header.getValue().replaceAll("[\r\n]", ", "));
// Escaped so the CRLF actually show
LOGGER.debug(
"Removed CRLF from \"{}\" header value: {}",
name,
StringEscapeUtils.escapeJava(value));
}
}

Expand Down
Loading

0 comments on commit d3ef666

Please sign in to comment.