diff --git a/build.gradle b/build.gradle index 12ec103e..9e127ee8 100644 --- a/build.gradle +++ b/build.gradle @@ -17,13 +17,13 @@ dependencies { implementation 'net.portswigger.burp.extensions:montoya-api:2023.5' implementation 'org.swinglabs:swingx:1.6.1' implementation 'com.github.CoreyD97:Burp-Montoya-Utilities:54678c64' -// implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.9' - implementation 'co.elastic.clients:elasticsearch-java:8.6.2' + implementation 'co.elastic.clients:elasticsearch-java:8.8.2' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.3' implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'org.apache.commons:commons-text:1.10.0' implementation 'org.apache.logging.log4j:log4j-core:2.19.0' - testRuntimeOnly files("${System.properties['user.home']}/BurpSuitePro/burpsuite_pro.jar") + testRuntimeOnly files("${System.properties['user.home']}/BurpSuiteCommunity/burpsuite_community.jar") } jar { diff --git a/src/main/java/com/nccgroup/loggerplusplus/exports/ElasticExporter.java b/src/main/java/com/nccgroup/loggerplusplus/exports/ElasticExporter.java index a67e323d..8b2050fe 100644 --- a/src/main/java/com/nccgroup/loggerplusplus/exports/ElasticExporter.java +++ b/src/main/java/com/nccgroup/loggerplusplus/exports/ElasticExporter.java @@ -15,6 +15,12 @@ import co.elastic.clients.transport.endpoints.BooleanResponse; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.coreyd97.BurpExtenderUtilities.Preferences; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.nccgroup.loggerplusplus.LoggerPlusPlus; @@ -22,6 +28,7 @@ import com.nccgroup.loggerplusplus.filter.parser.ParseException; import com.nccgroup.loggerplusplus.logentry.LogEntry; import com.nccgroup.loggerplusplus.logentry.LogEntryField; +import com.nccgroup.loggerplusplus.logentry.LogEntrySerializer; import com.nccgroup.loggerplusplus.logentry.Status; import com.nccgroup.loggerplusplus.util.Globals; import lombok.extern.log4j.Log4j2; @@ -40,6 +47,7 @@ import java.net.ConnectException; import java.net.InetAddress; import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -59,16 +67,20 @@ public class ElasticExporter extends AutomaticLogExporter implements ExportPanel private final ScheduledExecutorService executorService; private final ElasticExporterControlPanel controlPanel; - private final Gson gson; + private final ObjectMapper mapper; private Logger logger = LogManager.getLogger(this); protected ElasticExporter(ExportController exportController, Preferences preferences) { super(exportController, preferences); this.fields = new ArrayList<>(preferences.getSetting(Globals.PREF_PREVIOUS_ELASTIC_FIELDS)); - this.gson = LoggerPlusPlus.gsonProvider.getGson(); executorService = Executors.newScheduledThreadPool(1); + this.mapper = new ObjectMapper(); + SimpleModule module = new SimpleModule("LogEntry Serializer", new Version(0,1,0,"",null, null)); + module.addSerializer(LogEntry.class, new ElasticExporter.EntrySerializer(LogEntry.class)); + mapper.registerModule(module); + if ((boolean) preferences.getSetting(Globals.PREF_ELASTIC_AUTOSTART_GLOBAL) || (boolean) preferences.getSetting(Globals.PREF_ELASTIC_AUTOSTART_PROJECT)) { //Autostart exporter. @@ -91,22 +103,23 @@ void setup() throws Exception { String projectPreviousFilterString = preferences.getSetting(Globals.PREF_ELASTIC_FILTER_PROJECT_PREVIOUS); String filterString = preferences.getSetting(Globals.PREF_ELASTIC_FILTER); - if (!Objects.equals(projectPreviousFilterString, filterString)) { - //The current filter isn't what we used to export last time. - int res = JOptionPane.showConfirmDialog(LoggerPlusPlus.instance.getLoggerFrame(), - "Heads up! Looks like the filter being used to select which logs to export to " + - "ElasticSearch has changed since you last ran the exporter for this project.\n" + - "Do you want to continue?", "ElasticSearch Export Log Filter", JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE); - if (res == JOptionPane.NO_OPTION) { - throw new Exception("Export cancelled."); - } - } +// if (!Objects.equals(projectPreviousFilterString, filterString)) { +// //The current filter isn't what we used to export last time. +// int res = JOptionPane.showConfirmDialog(LoggerPlusPlus.instance.getLoggerFrame(), +// "Heads up! Looks like the filter being used to select which logs to export to " + +// "ElasticSearch has changed since you last ran the exporter for this project.\n" + +// "Do you want to continue?", "ElasticSearch Export Log Filter", JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE); +// if (res == JOptionPane.NO_OPTION) { +// throw new Exception("Export cancelled."); +// } +// } if (!StringUtils.isBlank(filterString)) { try { logFilter = new LogTableFilter(filterString); } catch (ParseException ex) { logger.error("The log filter configured for the Elastic exporter is invalid!", ex); + throw new Exception("The log filter configured for the Elastic exporter is invalid!", ex); } } @@ -140,7 +153,7 @@ void setup() throws Exception { } - ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper()); + ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper(this.mapper)); elasticClient = new ElasticsearchClient(transport); @@ -195,21 +208,21 @@ private void createIndices() throws IOException { } } - public JsonObject serializeLogEntry(LogEntry logEntry) { - //Todo Better serialization of entries - JsonObject jsonObject = new JsonObject(); - for (LogEntryField field : this.fields) { - Object value = formatValue(logEntry.getValueByKey(field)); - try { - jsonObject.addProperty(field.getFullLabel(), gson.toJson(value)); - }catch (Exception e){ - log.error("ElasticExporter: " + value); - log.error("ElasticExporter: " + e.getMessage()); - throw e; - } - } - return jsonObject; - } +// public JsonObject serializeLogEntry(LogEntry logEntry) { +// //Todo Better serialization of entries +// JsonObject jsonObject = new JsonObject(); +// for (LogEntryField field : this.fields) { +// Object value = formatValue(logEntry.getValueByKey(field)); +// try { +// jsonObject.addProperty(field.getFullLabel(), gson.toJson(value)); +// }catch (Exception e){ +// log.error("ElasticExporter: " + value); +// log.error("ElasticExporter: " + e.getMessage()); +// throw e; +// } +// } +// return jsonObject; +// } private void indexPendingEntries(){ try { @@ -228,7 +241,7 @@ private void indexPendingEntries(){ bulkBuilder.operations(op -> op .index(idx -> idx .index(this.indexName) - .document(serializeLogEntry(logEntry)) + .document(logEntry) ) ); @@ -255,18 +268,13 @@ private void indexPendingEntries(){ shutdown(); } }catch (IOException e) { - e.printStackTrace(); + log.error(e); } }catch (Exception e){ - e.printStackTrace(); + log.error(e); } } - private Object formatValue(Object value){ - if (value instanceof java.net.URL) return String.valueOf((java.net.URL) value); - else return value; - } - public ExportController getExportController() { return this.exportController; } @@ -279,4 +287,34 @@ public void setFields(List fields) { preferences.setSetting(Globals.PREF_PREVIOUS_ELASTIC_FIELDS, fields); this.fields = fields; } + + private class EntrySerializer extends StdSerializer { + + public EntrySerializer(Class t) { + super(t); + } + + @Override + public void serialize(LogEntry logEntry, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeStartObject(); + for (LogEntryField field : ElasticExporter.this.fields) { + Object value = logEntry.getValueByKey(field); + if(value == null) continue; + try { + switch (field.getType().getSimpleName()){ + case "Integer": gen.writeNumberField(field.getFullLabel(), (Integer) value); break; + case "Short": gen.writeNumberField(field.getFullLabel(), (Short) value); break; + case "Double": gen.writeNumberField(field.getFullLabel(), (Double) value); break; + case "String": gen.writeStringField(field.getFullLabel(), value.toString()); break; + case "Boolean": gen.writeBooleanField(field.getFullLabel(), (Boolean) value); break; + case "Date": gen.writeNumberField(field.getFullLabel(), ((Date) value).getTime()); break; + default: log.error("Unhandled field type: " + field.getType().getSimpleName()); + } + }catch (Exception e){ + log.error("ElasticExporter: Couldn't serialize field. The field was ommitted from the export."); + } + } + gen.writeEndObject(); + } + } } diff --git a/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntry.java b/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntry.java index b67e6eaf..8a029167 100644 --- a/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntry.java +++ b/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntry.java @@ -79,7 +79,7 @@ public class LogEntry { private String requestHttpVersion = ""; private String requestContentType = ""; private String protocol = ""; - private int targetPort = -1; + private short targetPort = -1; private int requestBodyLength = -1; private String clientIP = ""; private boolean hasSetCookies = false; @@ -179,9 +179,7 @@ private Status processRequest() { requestHeaders = request.headers(); - // Get HTTP Version, which would be the last token in "GET /admin/login/?next\u003d/admin/ HTTP/1.1" - String[] httpRequestTokens = requestHeaders.get(0).value().split(" "); - this.requestHttpVersion = httpRequestTokens[httpRequestTokens.length - 1]; + this.requestHttpVersion = request.httpVersion(); this.parameters = request.parameters().stream() .filter(param -> param.type() != HttpParameterType.COOKIE) @@ -192,7 +190,7 @@ private Status processRequest() { this.hostname = this.request.httpService().host(); this.protocol = this.request.httpService().secure() ? "https" : "http"; this.isSSL = this.request.httpService().secure(); - this.targetPort = this.request.httpService().port(); + this.targetPort = (short) this.request.httpService().port(); boolean isDefaultPort = (this.protocol.equals("https") && this.targetPort == 443) || (this.protocol.equals("http") && this.targetPort == 80); @@ -361,11 +359,8 @@ private Status processResponse() { this.redirectURL = headers.get("Location"); } - // Extract HTTP Status message - HttpHeader httpStatusTokens = response.headers().get(0); - //TODO FixMe -// this.responseStatusText = httpStatusTokens[httpStatusTokens.length - 1]; -// this.responseHttpVersion = httpStatusTokens[0]; + this.responseStatusText = response.reasonPhrase(); + this.responseHttpVersion = response.httpVersion(); if (headers.containsKey("content-type")) { @@ -422,7 +417,6 @@ private Status processResponse() { .filter(parameter -> !reflectionController.isParameterFiltered(parameter) && reflectionController.validReflection(response.bodyToString(), parameter)) .map(HttpParameter::name).collect(Collectors.toList()); -// this.requestResponse = LoggerPlusPlus.montoya.saveBuffersToTempFiles(requestResponse); } else { //Just look for reflections in the headers. ReflectionController reflectionController = LoggerPlusPlus.instance.getReflectionController(); @@ -439,60 +433,6 @@ private Status processResponse() { this.complete = true; return Status.PROCESSED; - - // RegEx processing for responses - should be available only when we have a - // RegEx rule! - // There are 5 RegEx rule for requests - // for(int i=0;i<5;i++){ - // String regexVarName = "regex"+(i+1)+"Resp"; - // if(logTable.getColumnModel().isColumnEnabled(regexVarName)){ - // // so this rule is enabled! - // // check to see if the RegEx is not empty - // LogTableColumn regexColumn = - // logTable.getColumnModel().getColumnByName(regexVarName); - // String regexString = regexColumn.getRegExData().getRegExString(); - // if(!regexString.isEmpty()){ - // // now we can process it safely! - // Pattern p = null; - // try{ - // if(regexColumn.getRegExData().isRegExCaseSensitive()) - // p = Pattern.compile(regexString); - // else - // p = Pattern.compile(regexString, Pattern.CASE_INSENSITIVE); - // - // Matcher m = p.matcher(strFullResponse); - // StringBuilder allMatches = new StringBuilder(); - // - // int counter = 1; - // while (m.find()) { - // if(counter==2){ - // allMatches.insert(0, "X"); - // allMatches.append("X"); - // } - // if(counter > 1){ - // allMatches.append("X"+m.group()+"X"); //TODO investigate unicode use - // }else{ - // allMatches.append(m.group()); - // } - // counter++; - // - // } - // - // this.regexAllResp[i] = allMatches.toString(); - // - // }catch(Exception e){ - // LoggerPlusPlus.montoya.printError("Error in regular expression: " + - // regexString); - // } - // - // } - // } - // } - - // if(!logTable.getColumnModel().isColumnEnabled("response") && - // !logTable.getColumnModel().isColumnEnabled("request")){ - // this.requestResponse = null; - // } } public byte[] getRequestBytes() { @@ -613,26 +553,6 @@ public Object getValueByKey(LogEntryField columnName) { return this.usesCookieJar.toString(); case ORIGIN: return this.origin; - // case REGEX1REQ: - // return this.regexAllReq[0]; - // case REGEX2REQ: - // return this.regexAllReq[1]; - // case REGEX3REQ: - // return this.regexAllReq[2]; - // case REGEX4REQ: - // return this.regexAllReq[3]; - // case REGEX5REQ: - // return this.regexAllReq[4]; - // case REGEX1RESP: - // return this.regexAllResp[0]; - // case REGEX2RESP: - // return this.regexAllResp[1]; - // case REGEX3RESP: - // return this.regexAllResp[2]; - // case REGEX4RESP: - // return this.regexAllResp[3]; - // case REGEX5RESP: - // return this.regexAllResp[4]; case REFLECTED_PARAMS: return reflectedParameters; case REFLECTION_COUNT: diff --git a/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntryField.java b/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntryField.java index 11f69443..fbd881ed 100644 --- a/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntryField.java +++ b/src/main/java/com/nccgroup/loggerplusplus/logentry/LogEntryField.java @@ -39,14 +39,14 @@ public enum LogEntryField { HOST(FieldGroup.REQUEST, String.class, "The protocol and hostname of the requested URL.", "Host"), PORT(FieldGroup.REQUEST, Short.class, "The port the request was sent to.", "Port"), REQUEST_CONTENT_TYPE(FieldGroup.REQUEST, String.class, "The content-type header sent to the server.", "ContentType", "Content_Type"), - REQUEST_HTTP_VERSION(FieldGroup.REQUEST, Short.class, "The HTTP version sent in the request.", "RequestHttpVersion", "RequestHttpVersion"), + REQUEST_HTTP_VERSION(FieldGroup.REQUEST, String.class, "The HTTP version sent in the request.", "RequestHttpVersion", "RequestHttpVersion"), EXTENSION(FieldGroup.REQUEST, String.class, "The URL extension used in the request.", "Extension"), REFERRER(FieldGroup.REQUEST, String.class, "The referrer header value of the request.", "Referrer"), HASPARAMS(FieldGroup.REQUEST, Boolean.class, "Did the request contain parameters?", "HasParams"), HASGETPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain get parameters?", "HasGetParam", "HasGetParams", "HasQueryString"), HASPOSTPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain post parameters?", "HasPostParam", "HasPayload", "Payload"), - HASCOOKIEPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain cookies?", "HasSentCookies"), - SENTCOOKIES(FieldGroup.REQUEST, Boolean.class, "The value of the cookies header sent to the server.", "CookieString", "SentCookies", "Cookies"), + HASCOOKIEPARAM(FieldGroup.REQUEST, Boolean.class, "Did the request contain cookies?", "HasSentCookies", "HasCookies"), + SENTCOOKIES(FieldGroup.REQUEST, String.class, "The value of the cookies header sent to the server.", "CookieString", "SentCookies", "Cookies"), PARAMETER_COUNT(FieldGroup.REQUEST, Integer.class, "The number of parameters in the request.", "ParameterCount", "ParamCount"), PARAMETERS(FieldGroup.REQUEST, String.class, "The parameters in the request.", "Parameters", "Params"), ORIGIN(FieldGroup.REQUEST, String.class, "The Origin header", "Origin"), @@ -61,8 +61,8 @@ public enum LogEntryField { RESPONSE_LENGTH(FieldGroup.RESPONSE, Integer.class, "The length of the received response.", "Length"), REDIRECT_URL(FieldGroup.RESPONSE, URL.class, "The URL the response redirects to.", "Redirect", "RedirectURL"), STATUS(FieldGroup.RESPONSE, Short.class, "The status code received in the response.", "Status", "StatusCode"), - STATUS_TEXT(FieldGroup.RESPONSE, Short.class, "The status text received in the response.", "StatusText", "StatusText"), - RESPONSE_HTTP_VERSION(FieldGroup.RESPONSE, Short.class, "The HTTP version received in the response.", "ResponseHttpVersion", "ResponseHttpVersion"), + STATUS_TEXT(FieldGroup.RESPONSE, String.class, "The status text received in the response.", "StatusText", "StatusText"), + RESPONSE_HTTP_VERSION(FieldGroup.RESPONSE, String.class, "The HTTP version received in the response.", "ResponseHttpVersion", "ResponseHttpVersion"), RTT(FieldGroup.RESPONSE, Integer.class, "The round trip time (as calculated by L++, not 100% accurate).", "RTT", "TimeTaken"), TITLE(FieldGroup.RESPONSE, String.class, "The HTTP response title.", "Title"), RESPONSE_CONTENT_TYPE(FieldGroup.RESPONSE, String.class, "The content-type header sent by the server.", "ContentType", "Content_Type"),