Skip to content

Commit

Permalink
Merge pull request #6017 from thc202/client/spider/elements
Browse files Browse the repository at this point in the history 
client: add element interactions to the spider
  • Loading branch information
@kingthorin
kingthorin authored Dec 23, 2024
2 parents fd24dea + ff2ddce commit feb0f3a
Show file tree
Hide file tree
Showing 14 changed files with 593 additions and 33 deletions.
10 changes: 9 additions & 1 deletion addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
import org.zaproxy.addon.client.ui.PopupMenuClientHistoryCopy;
import org.zaproxy.addon.client.ui.PopupMenuClientOpenInBrowser;
import org.zaproxy.addon.client.ui.PopupMenuClientShowInSites;
import org.zaproxy.addon.commonlib.ExtensionCommonlib;
import org.zaproxy.addon.network.ExtensionNetwork;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.eventBus.Event;
Expand Down Expand Up @@ -114,6 +115,7 @@ public class ExtensionClientIntegration extends ExtensionAdaptor {
private static final List<Class<? extends Extension>> EXTENSION_DEPENDENCIES =
List.of(
ExtensionAlert.class,
ExtensionCommonlib.class,
ExtensionHistory.class,
ExtensionNetwork.class,
ExtensionSelenium.class);
Expand Down Expand Up @@ -155,7 +157,13 @@ public void initModel(Model model) {
new ClientSideDetails(
Constant.messages.getString("client.tree.title"), null),
this.getModel().getSession()));
spiderScanController = new SpiderScanController(this);
spiderScanController =
new SpiderScanController(
this,
Control.getSingleton()
.getExtensionLoader()
.getExtension(ExtensionCommonlib.class)
.getValueProvider());
passiveScanController = new ClientPassiveScanController();
}

Expand Down
11 changes: 10 additions & 1 deletion addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
public class ClientMap extends SortedTreeModel implements EventPublisher {

public static final String MAP_NODE_ADDED_EVENT = "client.mapNode.added";
public static final String MAP_COMPONENT_ADDED_EVENT = "client.mapComponent.added";
public static final String DEPTH_KEY = "depth";
public static final String SIBLINGS_KEY = "siblings";
public static final String URL_KEY = "url";
Expand All @@ -48,7 +49,7 @@ public class ClientMap extends SortedTreeModel implements EventPublisher {
public ClientMap(ClientNode root) {
super(root);
this.root = root;
ZAP.getEventBus().registerPublisher(this, MAP_NODE_ADDED_EVENT);
ZAP.getEventBus().registerPublisher(this, MAP_NODE_ADDED_EVENT, MAP_COMPONENT_ADDED_EVENT);
}

@Override
Expand Down Expand Up @@ -164,6 +165,13 @@ public boolean addComponentToNode(ClientNode node, ClientSideComponent component
boolean componentAdded = details.addComponent(component);
if (!wasVisited || componentAdded) {
details.setVisited(true);

Map<String, String> map = new HashMap<>(component.getData());
map.put(DEPTH_KEY, Integer.toString(node.getLevel()));
map.put(SIBLINGS_KEY, Integer.toString(node.getChildCount()));
ZAP.getEventBus()
.publishSyncEvent(
this, new Event(this, MAP_COMPONENT_ADDED_EVENT, new Target(), map));
}
return componentAdded;
}
Expand All @@ -176,6 +184,7 @@ public ClientNode setRedirect(String originalUrl, String redirectedUrl) {
node.getUserObject()
.addComponent(
new ClientSideComponent(
Map.of(),
ClientSideComponent.REDIRECT,
null,
originalUrl,
Expand Down
13 changes: 13 additions & 0 deletions addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
*/
package org.zaproxy.addon.client.internal;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import lombok.AllArgsConstructor;
import lombok.Getter;
Expand All @@ -32,6 +34,8 @@ public class ClientSideComponent {

public static String REDIRECT = "Redirect";

private final Map<String, String> data;

private String tagName;
private String id;
private String parentUrl;
Expand All @@ -42,6 +46,11 @@ public class ClientSideComponent {
private int formId = -1;

public ClientSideComponent(JSONObject json) {
data = new HashMap<>();
for (Object key : json.keySet()) {
data.put(key.toString(), json.get(key).toString());
}

this.tagName = json.getString("tagName");
this.id = json.getString("id");
this.parentUrl = json.getString("url");
Expand All @@ -60,6 +69,10 @@ public ClientSideComponent(JSONObject json) {
}
}

public Map<String, String> getData() {
return data;
}

public String getTypeForDisplay() {
switch (tagName) {
case "A":
Expand Down
93 changes: 77 additions & 16 deletions addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,18 @@
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import javax.swing.table.TableModel;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -39,6 +43,10 @@
import org.zaproxy.addon.client.ExtensionClientIntegration;
import org.zaproxy.addon.client.internal.ClientMap;
import org.zaproxy.addon.client.internal.ClientNode;
import org.zaproxy.addon.client.spider.actions.ClickElement;
import org.zaproxy.addon.client.spider.actions.OpenUrl;
import org.zaproxy.addon.client.spider.actions.SubmitForm;
import org.zaproxy.addon.commonlib.ValueProvider;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.eventBus.Event;
import org.zaproxy.zap.eventBus.EventConsumer;
Expand All @@ -60,8 +68,6 @@ public class ClientSpider implements EventConsumer, GenericScanner2 {
* Help pages
*
* The following features should be implemented in future releases:
* Filling in forms
* Clicking on buttons
* Clicking on likely navigation elements
* Preventing reqs to out of scope sites (via navigation elements)
* Automation framework support
Expand All @@ -73,6 +79,7 @@ public class ClientSpider implements EventConsumer, GenericScanner2 {

private ExecutorService threadPool;

private final ValueProvider valueProvider;
private ClientOptions options;
private int scanId;
private String displayName;
Expand All @@ -94,7 +101,7 @@ public class ClientSpider implements EventConsumer, GenericScanner2 {
private boolean stopped;

private int tasksDoneCount;
private int tasksTotalCount;
private final AtomicInteger tasksTotalCount;

private UrlTableModel addedNodesModel;
private ScanListenner2 listener;
Expand All @@ -105,13 +112,16 @@ public ClientSpider(
String targetUrl,
ClientOptions options,
int id,
User user) {
User user,
ValueProvider valueProvider) {
this.extClient = extClient;
this.displayName = displayName;
this.targetUrl = targetUrl;
this.options = options;
this.scanId = id;
this.tasksTotalCount = new AtomicInteger();
this.user = user;
this.valueProvider = valueProvider;
this.addedNodesModel = new UrlTableModel();

ZAP.getEventBus().registerConsumer(this, ClientMap.class.getCanonicalName());
Expand All @@ -126,7 +136,7 @@ public ClientSpider(
String targetUrl,
ClientOptions options,
int id) {
this(extClient, displayName, targetUrl, options, id, null);
this(extClient, displayName, targetUrl, options, id, null, null);
}

@Override
Expand All @@ -152,10 +162,35 @@ public void run() {

List<String> unvisitedUrls = getUnvisitedUrls();

addTask(targetUrl, options.getInitialLoadTimeInSecs());
addInitialOpenUrlTask(targetUrl);

// Add all of the known but unvisited URLs otherwise these will get ignored
unvisitedUrls.forEach(url -> addTask(url, options.getInitialLoadTimeInSecs()));
unvisitedUrls.forEach(this::addInitialOpenUrlTask);
}

private void addInitialOpenUrlTask(String url) {
addOpenUrlTask(url, options.getInitialLoadTimeInSecs());
}

private void addOpenUrlTask(String url, int loadTimeInSecs) {
addTask(openAction(url), loadTimeInSecs);
}

private List<SpiderAction> openAction(String url, SpiderAction... additionalActions) {
List<SpiderAction> actions = new ArrayList<>(5);
actions.add(new OpenUrl(url));
actions.add(wd -> checkRedirect(url, wd));
if (additionalActions != null) {
Stream.of(additionalActions).forEach(actions::add);
}
return actions;
}

private void checkRedirect(String url, WebDriver wd) {
String actualUrl = wd.getCurrentUrl();
if (!url.equals(actualUrl)) {
setRedirect(url, actualUrl);
}
}

private List<String> getUnvisitedUrls() {
Expand Down Expand Up @@ -203,10 +238,10 @@ public void returnWebDriver(WebDriver wd) {
}
}

private void addTask(String url, int loadTimeInSecs) {
this.tasksTotalCount++;
private void addTask(List<SpiderAction> actions, int loadTimeInSecs) {
int id = tasksTotalCount.incrementAndGet();
try {
ClientSpiderTask task = new ClientSpiderTask(this, url, loadTimeInSecs);
ClientSpiderTask task = new ClientSpiderTask(id, this, actions, loadTimeInSecs);
if (paused) {
this.pausedTasks.add(task);
} else {
Expand Down Expand Up @@ -246,12 +281,13 @@ public void eventReceived(Event event) {
return;
}

String url = event.getParameters().get(ClientMap.URL_KEY);
Map<String, String> parameters = event.getParameters();
String url = parameters.get(ClientMap.URL_KEY);
if (url.startsWith(targetUrl)) {
addUriToAddedNodesModel(url);

if (options.getMaxDepth() > 0) {
int depth = Integer.parseInt(event.getParameters().get(ClientMap.DEPTH_KEY));
int depth = Integer.parseInt(parameters.get(ClientMap.DEPTH_KEY));
if (depth > options.getMaxDepth()) {
LOGGER.debug(
"Ignoring URL - too deep {} > {} : {}",
Expand All @@ -262,7 +298,7 @@ public void eventReceived(Event event) {
}
}
if (options.getMaxChildren() > 0) {
int siblings = Integer.parseInt(event.getParameters().get(ClientMap.SIBLINGS_KEY));
int siblings = Integer.parseInt(parameters.get(ClientMap.SIBLINGS_KEY));
if (siblings > options.getMaxChildren()) {
LOGGER.debug(
"Ignoring URL - too wide {} > {} : {}",
Expand All @@ -272,8 +308,33 @@ public void eventReceived(Event event) {
return;
}
}
addTask(url, options.getPageLoadTimeInSecs());

if (ClientMap.MAP_COMPONENT_ADDED_EVENT.equals(event.getEventType())) {
if (ClickElement.isSupported(href -> href.startsWith(targetUrl), parameters)) {
addTask(
openAction(
url,
new ClickElement(valueProvider, createURI(url), parameters)),
options.getPageLoadTimeInSecs());
} else if (SubmitForm.isSupported(parameters)) {
addTask(
openAction(
url, new SubmitForm(valueProvider, createURI(url), parameters)),
options.getPageLoadTimeInSecs());
}
} else {
addOpenUrlTask(url, options.getPageLoadTimeInSecs());
}
}
}

private URI createURI(String value) {
try {
return new URI(value, true);
} catch (URIException | NullPointerException e) {
LOGGER.warn("Failed to create URI from {}", value, e);
}
return null;
}

private void addUriToAddedNodesModel(final String uri) {
Expand All @@ -292,11 +353,11 @@ protected void setRedirect(String originalUrl, String redirectedUrl) {
public int getProgress() {
if (finished && !stopped) {
return 100;
} else if (this.tasksTotalCount <= 1) {
} else if (tasksTotalCount.get() <= 1) {
// Still waiting for the first request to be processed
return 0;
}
return (this.tasksDoneCount * 100) / this.tasksTotalCount;
return (this.tasksDoneCount * 100) / tasksTotalCount.get();
}

@Override
Expand Down
22 changes: 11 additions & 11 deletions addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.zaproxy.addon.client.spider;

import java.time.Duration;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.openqa.selenium.WebDriver;
Expand All @@ -28,14 +29,17 @@ public class ClientSpiderTask implements Runnable {

private static final Logger LOGGER = LogManager.getLogger(ClientSpiderTask.class);

private final int id;
private ClientSpider clientSpider;
private String url;
private List<SpiderAction> actions;
private int timeout;
private WebDriver wd;

public ClientSpiderTask(ClientSpider clientSpider, String url, int timeout) {
public ClientSpiderTask(
int id, ClientSpider clientSpider, List<SpiderAction> actions, int timeout) {
this.id = id;
this.clientSpider = clientSpider;
this.url = url;
this.actions = actions;
this.timeout = timeout;
}

Expand All @@ -57,21 +61,17 @@ public void run() {
wd = this.clientSpider.getWebDriver();
startTime = System.currentTimeMillis();
wd.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(this.timeout));
wd.get(url);
String actualUrl = wd.getCurrentUrl();
if (!url.equals(actualUrl)) {
clientSpider.setRedirect(url, actualUrl);
}
actions.forEach(e -> e.run(wd));
ok = true;
} catch (Exception e) {
LOGGER.warn("Task failed {} {}", url, e.getMessage(), e);
LOGGER.warn("Task {} failed {}", id, e.getMessage(), e);
}
if (wd != null) {
this.clientSpider.returnWebDriver(wd);
}
LOGGER.debug(
"Task completed {} {} in {} secs",
url,
"Task {} completed {} in {} secs",
id,
ok,
(System.currentTimeMillis() - startTime) / 1000);
this.clientSpider.postTaskExecution(this);
Expand Down
Loading

0 comments on commit feb0f3a

Please sign in to comment.