Skip to content

Commit

Permalink
Implements #157
Browse files Browse the repository at this point in the history
  • Loading branch information
@CoreyD97
CoreyD97 committed Oct 19, 2023
1 parent 12ad2fa commit 8b9781a
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 129 deletions.
6 changes: 3 additions & 3 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ dependencies {
implementation 'net.portswigger.burp.extensions:montoya-api:2023.5'
implementation 'org.swinglabs:swingx:1.6.1'
implementation 'com.github.CoreyD97:Burp-Montoya-Utilities:54678c64'
// implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.9'
implementation 'co.elastic.clients:elasticsearch-java:8.6.2'
implementation 'co.elastic.clients:elasticsearch-java:8.8.2'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.3'
implementation 'org.apache.httpcomponents:httpclient:4.5.13'
implementation 'org.apache.commons:commons-text:1.10.0'
implementation 'org.apache.logging.log4j:log4j-core:2.19.0'

testRuntimeOnly files("${System.properties['user.home']}/BurpSuitePro/burpsuite_pro.jar")
testRuntimeOnly files("${System.properties['user.home']}/BurpSuiteCommunity/burpsuite_community.jar")
}

jar {
Expand Down
110 changes: 74 additions & 36 deletions src/main/java/com/nccgroup/loggerplusplus/exports/ElasticExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,20 @@
import co.elastic.clients.transport.endpoints.BooleanResponse;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.coreyd97.BurpExtenderUtilities.Preferences;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.nccgroup.loggerplusplus.LoggerPlusPlus;
import com.nccgroup.loggerplusplus.filter.logfilter.LogTableFilter;
import com.nccgroup.loggerplusplus.filter.parser.ParseException;
import com.nccgroup.loggerplusplus.logentry.LogEntry;
import com.nccgroup.loggerplusplus.logentry.LogEntryField;
import com.nccgroup.loggerplusplus.logentry.LogEntrySerializer;
import com.nccgroup.loggerplusplus.logentry.Status;
import com.nccgroup.loggerplusplus.util.Globals;
import lombok.extern.log4j.Log4j2;
Expand All @@ -40,6 +47,7 @@
import java.net.ConnectException;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
Expand All @@ -59,16 +67,20 @@ public class ElasticExporter extends AutomaticLogExporter implements ExportPanel

private final ScheduledExecutorService executorService;
private final ElasticExporterControlPanel controlPanel;
private final Gson gson;
private final ObjectMapper mapper;

private Logger logger = LogManager.getLogger(this);

protected ElasticExporter(ExportController exportController, Preferences preferences) {
super(exportController, preferences);
this.fields = new ArrayList<>(preferences.getSetting(Globals.PREF_PREVIOUS_ELASTIC_FIELDS));
this.gson = LoggerPlusPlus.gsonProvider.getGson();
executorService = Executors.newScheduledThreadPool(1);

this.mapper = new ObjectMapper();
SimpleModule module = new SimpleModule("LogEntry Serializer", new Version(0,1,0,"",null, null));
module.addSerializer(LogEntry.class, new ElasticExporter.EntrySerializer(LogEntry.class));
mapper.registerModule(module);

if ((boolean) preferences.getSetting(Globals.PREF_ELASTIC_AUTOSTART_GLOBAL)
|| (boolean) preferences.getSetting(Globals.PREF_ELASTIC_AUTOSTART_PROJECT)) {
//Autostart exporter.
Expand All @@ -91,22 +103,23 @@ void setup() throws Exception {
String projectPreviousFilterString = preferences.getSetting(Globals.PREF_ELASTIC_FILTER_PROJECT_PREVIOUS);
String filterString = preferences.getSetting(Globals.PREF_ELASTIC_FILTER);

if (!Objects.equals(projectPreviousFilterString, filterString)) {
//The current filter isn't what we used to export last time.
int res = JOptionPane.showConfirmDialog(LoggerPlusPlus.instance.getLoggerFrame(),
"Heads up! Looks like the filter being used to select which logs to export to " +
"ElasticSearch has changed since you last ran the exporter for this project.\n" +
"Do you want to continue?", "ElasticSearch Export Log Filter", JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE);
if (res == JOptionPane.NO_OPTION) {
throw new Exception("Export cancelled.");
}
}
// if (!Objects.equals(projectPreviousFilterString, filterString)) {
// //The current filter isn't what we used to export last time.
// int res = JOptionPane.showConfirmDialog(LoggerPlusPlus.instance.getLoggerFrame(),
// "Heads up! Looks like the filter being used to select which logs to export to " +
// "ElasticSearch has changed since you last ran the exporter for this project.\n" +
// "Do you want to continue?", "ElasticSearch Export Log Filter", JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE);
// if (res == JOptionPane.NO_OPTION) {
// throw new Exception("Export cancelled.");
// }
// }

if (!StringUtils.isBlank(filterString)) {
try {
logFilter = new LogTableFilter(filterString);
} catch (ParseException ex) {
logger.error("The log filter configured for the Elastic exporter is invalid!", ex);
throw new Exception("The log filter configured for the Elastic exporter is invalid!", ex);
}
}

Expand Down Expand Up @@ -140,7 +153,7 @@ void setup() throws Exception {
}


ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper());
ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper(this.mapper));

elasticClient = new ElasticsearchClient(transport);

Expand Down Expand Up @@ -195,21 +208,21 @@ private void createIndices() throws IOException {
}
}

public JsonObject serializeLogEntry(LogEntry logEntry) {
//Todo Better serialization of entries
JsonObject jsonObject = new JsonObject();
for (LogEntryField field : this.fields) {
Object value = formatValue(logEntry.getValueByKey(field));
try {
jsonObject.addProperty(field.getFullLabel(), gson.toJson(value));
}catch (Exception e){
log.error("ElasticExporter: " + value);
log.error("ElasticExporter: " + e.getMessage());
throw e;
}
}
return jsonObject;
}
// public JsonObject serializeLogEntry(LogEntry logEntry) {
// //Todo Better serialization of entries
// JsonObject jsonObject = new JsonObject();
// for (LogEntryField field : this.fields) {
// Object value = formatValue(logEntry.getValueByKey(field));
// try {
// jsonObject.addProperty(field.getFullLabel(), gson.toJson(value));
// }catch (Exception e){
// log.error("ElasticExporter: " + value);
// log.error("ElasticExporter: " + e.getMessage());
// throw e;
// }
// }
// return jsonObject;
// }

private void indexPendingEntries(){
try {
Expand All @@ -228,7 +241,7 @@ private void indexPendingEntries(){
bulkBuilder.operations(op -> op
.index(idx -> idx
.index(this.indexName)
.document(serializeLogEntry(logEntry))
.document(logEntry)
)
);

Expand All @@ -255,18 +268,13 @@ private void indexPendingEntries(){
shutdown();
}
}catch (IOException e) {
e.printStackTrace();
log.error(e);
}
}catch (Exception e){
e.printStackTrace();
log.error(e);
}
}

private Object formatValue(Object value){
if (value instanceof java.net.URL) return String.valueOf((java.net.URL) value);
else return value;
}

public ExportController getExportController() {
return this.exportController;
}
Expand All @@ -279,4 +287,34 @@ public void setFields(List<LogEntryField> fields) {
preferences.setSetting(Globals.PREF_PREVIOUS_ELASTIC_FIELDS, fields);
this.fields = fields;
}

private class EntrySerializer extends StdSerializer<LogEntry> {

public EntrySerializer(Class<LogEntry> t) {
super(t);
}

@Override
public void serialize(LogEntry logEntry, JsonGenerator gen, SerializerProvider provider) throws IOException {
gen.writeStartObject();
for (LogEntryField field : ElasticExporter.this.fields) {
Object value = logEntry.getValueByKey(field);
if(value == null) continue;
try {
switch (field.getType().getSimpleName()){
case "Integer": gen.writeNumberField(field.getFullLabel(), (Integer) value); break;
case "Short": gen.writeNumberField(field.getFullLabel(), (Short) value); break;
case "Double": gen.writeNumberField(field.getFullLabel(), (Double) value); break;
case "String": gen.writeStringField(field.getFullLabel(), value.toString()); break;
case "Boolean": gen.writeBooleanField(field.getFullLabel(), (Boolean) value); break;
case "Date": gen.writeNumberField(field.getFullLabel(), ((Date) value).getTime()); break;
default: log.error("Unhandled field type: " + field.getType().getSimpleName());
}
}catch (Exception e){
log.error("ElasticExporter: Couldn't serialize field. The field was ommitted from the export.");
}
}
gen.writeEndObject();
}
}
}
90 changes: 5 additions & 85 deletions src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public class LogEntry {
private String requestHttpVersion = "";
private String requestContentType = "";
private String protocol = "";
private int targetPort = -1;
private short targetPort = -1;
private int requestBodyLength = -1;
private String clientIP = "";
private boolean hasSetCookies = false;
Expand Down Expand Up @@ -179,9 +179,7 @@ private Status processRequest() {

requestHeaders = request.headers();

// Get HTTP Version, which would be the last token in "GET /admin/login/?next\u003d/admin/ HTTP/1.1"
String[] httpRequestTokens = requestHeaders.get(0).value().split(" ");
this.requestHttpVersion = httpRequestTokens[httpRequestTokens.length - 1];
this.requestHttpVersion = request.httpVersion();

this.parameters = request.parameters().stream()
.filter(param -> param.type() != HttpParameterType.COOKIE)
Expand All @@ -192,7 +190,7 @@ private Status processRequest() {
this.hostname = this.request.httpService().host();
this.protocol = this.request.httpService().secure() ? "https" : "http";
this.isSSL = this.request.httpService().secure();
this.targetPort = this.request.httpService().port();
this.targetPort = (short) this.request.httpService().port();

boolean isDefaultPort = (this.protocol.equals("https") && this.targetPort == 443)
|| (this.protocol.equals("http") && this.targetPort == 80);
Expand Down Expand Up @@ -361,11 +359,8 @@ private Status processResponse() {
this.redirectURL = headers.get("Location");
}

// Extract HTTP Status message
HttpHeader httpStatusTokens = response.headers().get(0);
//TODO FixMe
// this.responseStatusText = httpStatusTokens[httpStatusTokens.length - 1];
// this.responseHttpVersion = httpStatusTokens[0];
this.responseStatusText = response.reasonPhrase();
this.responseHttpVersion = response.httpVersion();


if (headers.containsKey("content-type")) {
Expand Down Expand Up @@ -422,7 +417,6 @@ private Status processResponse() {
.filter(parameter -> !reflectionController.isParameterFiltered(parameter) && reflectionController.validReflection(response.bodyToString(), parameter))
.map(HttpParameter::name).collect(Collectors.toList());

// this.requestResponse = LoggerPlusPlus.montoya.saveBuffersToTempFiles(requestResponse);
} else {
//Just look for reflections in the headers.
ReflectionController reflectionController = LoggerPlusPlus.instance.getReflectionController();
Expand All @@ -439,60 +433,6 @@ private Status processResponse() {
this.complete = true;

return Status.PROCESSED;

// RegEx processing for responses - should be available only when we have a
// RegEx rule!
// There are 5 RegEx rule for requests
// for(int i=0;i<5;i++){
// String regexVarName = "regex"+(i+1)+"Resp";
// if(logTable.getColumnModel().isColumnEnabled(regexVarName)){
// // so this rule is enabled!
// // check to see if the RegEx is not empty
// LogTableColumn regexColumn =
// logTable.getColumnModel().getColumnByName(regexVarName);
// String regexString = regexColumn.getRegExData().getRegExString();
// if(!regexString.isEmpty()){
// // now we can process it safely!
// Pattern p = null;
// try{
// if(regexColumn.getRegExData().isRegExCaseSensitive())
// p = Pattern.compile(regexString);
// else
// p = Pattern.compile(regexString, Pattern.CASE_INSENSITIVE);
//
// Matcher m = p.matcher(strFullResponse);
// StringBuilder allMatches = new StringBuilder();
//
// int counter = 1;
// while (m.find()) {
// if(counter==2){
// allMatches.insert(0, "X");
// allMatches.append("X");
// }
// if(counter > 1){
// allMatches.append("X"+m.group()+"X"); //TODO investigate unicode use
// }else{
// allMatches.append(m.group());
// }
// counter++;
//
// }
//
// this.regexAllResp[i] = allMatches.toString();
//
// }catch(Exception e){
// LoggerPlusPlus.montoya.printError("Error in regular expression: " +
// regexString);
// }
//
// }
// }
// }

// if(!logTable.getColumnModel().isColumnEnabled("response") &&
// !logTable.getColumnModel().isColumnEnabled("request")){
// this.requestResponse = null;
// }
}

public byte[] getRequestBytes() {
Expand Down Expand Up @@ -613,26 +553,6 @@ public Object getValueByKey(LogEntryField columnName) {
return this.usesCookieJar.toString();
case ORIGIN:
return this.origin;
// case REGEX1REQ:
// return this.regexAllReq[0];
// case REGEX2REQ:
// return this.regexAllReq[1];
// case REGEX3REQ:
// return this.regexAllReq[2];
// case REGEX4REQ:
// return this.regexAllReq[3];
// case REGEX5REQ:
// return this.regexAllReq[4];
// case REGEX1RESP:
// return this.regexAllResp[0];
// case REGEX2RESP:
// return this.regexAllResp[1];
// case REGEX3RESP:
// return this.regexAllResp[2];
// case REGEX4RESP:
// return this.regexAllResp[3];
// case REGEX5RESP:
// return this.regexAllResp[4];
case REFLECTED_PARAMS:
return reflectedParameters;
case REFLECTION_COUNT:
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntryField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,14 @@ public enum LogEntryField {
HOST(FieldGroup.REQUEST, String.class, "The protocol and hostname of the requested URL.", "Host"),
PORT(FieldGroup.REQUEST, Short.class, "The port the request was sent to.", "Port"),
REQUEST_CONTENT_TYPE(FieldGroup.REQUEST, String.class, "The content-type header sent to the server.", "ContentType", "Content_Type"),
REQUEST_HTTP_VERSION(FieldGroup.REQUEST, Short.class, "The HTTP version sent in the request.", "RequestHttpVersion", "RequestHttpVersion"),
REQUEST_HTTP_VERSION(FieldGroup.REQUEST, String.class, "The HTTP version sent in the request.", "RequestHttpVersion", "RequestHttpVersion"),
EXTENSION(FieldGroup.REQUEST, String.class, "The URL extension used in the request.", "Extension"),
REFERRER(FieldGroup.REQUEST, String.class, "The referrer header value of the request.", "Referrer"),
HASPARAMS(FieldGroup.REQUEST, Boolean.class, "Did the request contain parameters?", "HasParams"),
HASGETPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain get parameters?", "HasGetParam", "HasGetParams", "HasQueryString"),
HASPOSTPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain post parameters?", "HasPostParam", "HasPayload", "Payload"),
HASCOOKIEPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain cookies?", "HasSentCookies"),
SENTCOOKIES(FieldGroup.REQUEST, Boolean.class, "The value of the cookies header sent to the server.", "CookieString", "SentCookies", "Cookies"),
HASCOOKIEPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain cookies?", "HasSentCookies", "HasCookies"),
SENTCOOKIES(FieldGroup.REQUEST, String.class, "The value of the cookies header sent to the server.", "CookieString", "SentCookies", "Cookies"),
PARAMETER_COUNT(FieldGroup.REQUEST, Integer.class, "The number of parameters in the request.", "ParameterCount", "ParamCount"),
PARAMETERS(FieldGroup.REQUEST, String.class, "The parameters in the request.", "Parameters", "Params"),
ORIGIN(FieldGroup.REQUEST, String.class, "The Origin header", "Origin"),
Expand All @@ -61,8 +61,8 @@ public enum LogEntryField {
RESPONSE_LENGTH(FieldGroup.RESPONSE, Integer.class, "The length of the received response.", "Length"),
REDIRECT_URL(FieldGroup.RESPONSE, URL.class, "The URL the response redirects to.", "Redirect", "RedirectURL"),
STATUS(FieldGroup.RESPONSE, Short.class, "The status code received in the response.", "Status", "StatusCode"),
STATUS_TEXT(FieldGroup.RESPONSE, Short.class, "The status text received in the response.", "StatusText", "StatusText"),
RESPONSE_HTTP_VERSION(FieldGroup.RESPONSE, Short.class, "The HTTP version received in the response.", "ResponseHttpVersion", "ResponseHttpVersion"),
STATUS_TEXT(FieldGroup.RESPONSE, String.class, "The status text received in the response.", "StatusText", "StatusText"),
RESPONSE_HTTP_VERSION(FieldGroup.RESPONSE, String.class, "The HTTP version received in the response.", "ResponseHttpVersion", "ResponseHttpVersion"),
RTT(FieldGroup.RESPONSE, Integer.class, "The round trip time (as calculated by L++, not 100% accurate).", "RTT", "TimeTaken"),
TITLE(FieldGroup.RESPONSE, String.class, "The HTTP response title.", "Title"),
RESPONSE_CONTENT_TYPE(FieldGroup.RESPONSE, String.class, "The content-type header sent by the server.", "ContentType", "Content_Type"),
Expand Down

0 comments on commit 8b9781a

Please sign in to comment.