Skip to content

Commit

Permalink
'#43 As leapp scripts declare the output temp report folder as a a
Browse files Browse the repository at this point in the history 
static variable, multiple plugins concurrent execution were scrambling
device info between their specific file Output. As the necessary result
is the merging of those all info, the concurrency is not a problem,
being it done at the end of all individual plugins processing (4th
queue).
  • Loading branch information
@patrickdalla
patrickdalla committed Oct 9, 2023
1 parent 63d4b19 commit 1f650aa
Showing 1 changed file with 73 additions and 42 deletions.
115 changes: 73 additions & 42 deletions iped-engine/src/main/java/iped/engine/task/leappbridge/LeappBridgeTask.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package iped.engine.task.leappbridge;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
Expand All @@ -13,9 +14,11 @@
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.FileUtils;
import org.apache.lucene.document.Document;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;

import com.google.common.io.Files;

Expand All @@ -29,6 +32,7 @@
import iped.engine.data.Item;
import iped.engine.search.IPEDSearcher;
import iped.engine.task.AbstractPythonTask;
import iped.engine.task.ExportFileTask;
import iped.engine.task.index.IndexItem;
import iped.engine.util.ParentInfo;
import iped.properties.BasicProps;
Expand All @@ -39,9 +43,9 @@

public class LeappBridgeTask extends AbstractPythonTask {

private static final String DEVICE_DETAILS_HTML = "DeviceDetails";
public static MediaType DEVICEDETAILS = MediaType.application("x-leapp-devicedetails");

private static final String DEVICE_INFO_PROPERTY = "DEVICE_INFO";
private static final String DEVICE_DETAILS_HTML = "DeviceDetails.html";

private static final String ALEAPP_PLUGIN = "ALEAPP:PLUGIN";

Expand All @@ -54,6 +58,7 @@ public class LeappBridgeTask extends AbstractPythonTask {
static AtomicInteger count = new AtomicInteger(0);

HashMap<String, Item> mappedEvidences = new HashMap<String, Item>();
static HashMap<Integer, List<File>> tempDirs = new HashMap<Integer, List<File>>();

public LeappBridgeTask() {
}
Expand Down Expand Up @@ -91,15 +96,20 @@ public void init(ConfigurationManager configurationManager) throws Exception {
Jep jep = super.getJep();
pluginsManager.init(jep);
jep.eval("from scripts.ilapfuncs import OutputParameters");

}

@Override
public void finish() throws Exception {

// removes temporary folders
for (List<File> dirs : tempDirs.values()) {
for (File tempDir : dirs) {
if (tempDir.exists()) {
FileUtils.deleteDirectory(tempDir);
}
}
}
}


public void executePlugin(IItem evidence, LeapArtifactsPlugin p, List<String> filesFound, File reportDumpPath) {
Jep jep = getJep();
try {
Expand All @@ -124,18 +134,27 @@ public void executePlugin(IItem evidence, LeapArtifactsPlugin p, List<String> fi
jep.eval("sys.path.append('" + scriptsDir.getCanonicalPath().replace("\\", "\\\\") + "\\\\ALEAPP')");

// temp directory to save log and device info files
if (tmp == null) {
tmp = Files.createTempDir();
tmp.mkdirs();
tmp.deleteOnExit();
tmp = Files.createTempDir();
tmp.mkdirs();
tmp.deleteOnExit();

int leappRepEvidence = evidence.getParentId();

synchronized (tempDirs) {
List<File> tempFolders = tempDirs.get(leappRepEvidence);
if (tempFolders == null) {
tempFolders = new ArrayList<File>();
tempDirs.put(leappRepEvidence, tempFolders);
}
tempFolders.add(tmp);
}

jep.eval("import scripts.artifact_report");
jep.eval("from multiprocessing import Process");
jep.eval("import os");
jep.eval("import sys");
jep.eval("from java.lang import System");
jep.eval("from iped.engine.task.jleapp import ArtifactJavaReport");
jep.eval("from iped.engine.task.leappbridge import ArtifactJavaReport");
jep.eval("from " + p.getModuleName() + " import " + p.getMethodName() + " as parse");


Expand All @@ -162,26 +181,19 @@ public void executePlugin(IItem evidence, LeapArtifactsPlugin p, List<String> fi
try {
jep.eval("OutputParameters('" + tmp.getCanonicalPath() + "')");
} catch (Exception e) {
e.printStackTrace();
if (e.getMessage().contains("<class 'FileExistsError'>")) {
// skips as the dir already exists so it there is no need to create it but we
// can continue without it
} else {
throw e;
}
}
jep.eval("logfunc('" + PLUGIN_EXECUTION_MESSAGE + ":" + p.getModuleName() + "')");
jep.eval("parse(" + lists + ",'" + reportPath.getCanonicalPath() + "',dumb,True)");
}

File[] fs = new File(tmp.getCanonicalPath()).listFiles();
if (fs.length > 0) {
File deviceInfo = new File(new File(fs[0], "Script Logs"), "DeviceInfo.html");
if (deviceInfo.exists()) {
List<String> deviceInfoList = new ArrayList<String>();
deviceInfoList.addAll(Files.readLines(deviceInfo, Charset.forName("UTF-8")));
for (String info : deviceInfoList) {
// collected device info
evidence.getMetadata().add(DEVICE_INFO_PROPERTY, info);
}
}
}
} catch (Exception e) {
if (e.getMessage().contains("FileNotFoundError1")) {
if (e.getMessage().contains("FileNotFoundError")) {
if (e.getStackTrace()[0].getLineNumber() == 138) {
System.out.println();
}
Expand All @@ -205,7 +217,6 @@ public String getName() {
public void process(IItem evidence) throws Exception {
if (evidence.getName().equals("Dump")) {
Item subItem = (Item) evidence.createChildItem();
// Item fileEvidence = mappedEvidences.get(file.toString());
ParentInfo parentInfo = new ParentInfo(evidence);

String name = REPORT_EVIDENCE_NAME;
Expand All @@ -217,9 +228,9 @@ public void process(IItem evidence) throws Exception {
subItem.setExtraAttribute(ExtraProperties.DECODED_DATA, true);
worker.processNewItem(subItem);

// creates on subitem for each plugin execution
for (LeapArtifactsPlugin p : pluginsManager.getPlugins()) {
Item psubItem = (Item) subItem.createChildItem();
// Item fileEvidence = mappedEvidences.get(file.toString());
ParentInfo pparentInfo = new ParentInfo(subItem);

String moduleName = p.moduleName;
Expand All @@ -233,15 +244,14 @@ public void process(IItem evidence) throws Exception {
worker.processNewItem(psubItem);
}

// creates subitem to hold device info collected
Item devDetailsSubItem = (Item) subItem.createChildItem();
// Item fileEvidence = mappedEvidences.get(file.toString());
ParentInfo pparentInfo = new ParentInfo(subItem);

devDetailsSubItem.setName(DEVICE_DETAILS_HTML);
devDetailsSubItem.setMediaType(DEVICEDETAILS);
devDetailsSubItem.setPath(parentInfo.getPath() + "/" + DEVICE_DETAILS_HTML);
devDetailsSubItem.setSubItem(true);
devDetailsSubItem.setSubitemId(1);
devDetailsSubItem.getMetadata().set(ALEAPP_DEVICE_DETAILS, "true");
devDetailsSubItem.setExtraAttribute(ExtraProperties.DECODED_DATA, true);
worker.processNewItem(devDetailsSubItem);
}
Expand All @@ -258,10 +268,10 @@ public void process(IItem evidence) throws Exception {
}
}

// the device details should be the last to process as it gets output from every
// the device details must be the last to process as it gets output from every
// plugin execution
String isDevDetail = evidence.getMetadata().get(ALEAPP_DEVICE_DETAILS);
if (isDevDetail != null && isDevDetail.equals("true")) {
MediaType mt = evidence.getMediaType();
if (mt != null && mt.equals(DEVICEDETAILS)) {
int priority = worker.manager.getProcessingQueues().getCurrentQueuePriority();
if (priority < START_QUEUE_PRIORITY + 1) {
reEnqueueItem(evidence, START_QUEUE_PRIORITY + 1);
Expand All @@ -273,20 +283,41 @@ public void process(IItem evidence) throws Exception {
}

private void processDeviceDetails(IItem evidence) {
IPEDSearcher filesSearcher = new IPEDSearcher(ipedCase);
String query = "parentId:\"" + evidence.getParentId() + "\"";
/*
* Leapp declares static references to temporary "device details" html
* OutputParameters. This causes a new concurrent plugin execution to overwrite
* this static variable. So, there is no garantee that the content become
* scrambled. It is not a problem though, as all this writings are to be merged
* anyway in a single DeviceInfo.html.
*/

Integer leappRepEvidence = evidence.getParentId();
List<File> tempFolders = tempDirs.get(leappRepEvidence);

filesSearcher.setQuery(query);
try {
SearchResult filesResult = filesSearcher.search();
for (int j = 0; j < filesResult.getLength(); j++) {
int artLuceneId = ipedCase.getLuceneId(filesResult.getId(j));
Document artdoc = ipedCase.getReader().document(artLuceneId);
String info = artdoc.get(DEVICE_INFO_PROPERTY);
if (info != null) {
evidence.getMetadata().add(DEVICE_INFO_PROPERTY, info);
ExportFileTask extractor = new ExportFileTask();
extractor.setWorker(worker);
StringBuffer sb = new StringBuffer();
sb.append("<html><body>");
for (File tmp : tempFolders) {
File[] fs = new File(tmp.getCanonicalPath()).listFiles();
if (fs.length > 0) {
File deviceInfo = new File(new File(fs[0], "Script Logs"), "DeviceInfo.html");
if (deviceInfo.exists()) {
List<String> deviceInfoList = new ArrayList<String>();
deviceInfoList.addAll(Files.readLines(deviceInfo, Charset.forName("UTF-8")));
for (String info : deviceInfoList) {
// collected device info
sb.append(info);
}
}
}
}
sb.append("</body></html>");

// export thumb data to internal database
ByteArrayInputStream is = new ByteArrayInputStream(sb.toString().getBytes());
extractor.extractFile(is, evidence, evidence.getLength());

} catch (Exception e) {
e.printStackTrace();
Expand Down

0 comments on commit 1f650aa

Please sign in to comment.