diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java index 6250d0f6a6..0c13dd5a52 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java @@ -48,6 +48,7 @@ public class ClientOptions extends VersionedAbstractParam { private static final String MAX_DEPTH_KEY = CLIENT_BASE_KEY + ".maxDepth"; private static final String MAX_DURATION_KEY = CLIENT_BASE_KEY + ".maxDuration"; private static final String MAX_CHILDREN_KEY = CLIENT_BASE_KEY + ".maxChildren"; + private static final String MAX_SCANS_IN_UI_KEY = CLIENT_BASE_KEY + ".maxScansInUI"; private static final String DEFAULT_BROWSER_ID = Browser.FIREFOX_HEADLESS.getId(); @@ -62,6 +63,7 @@ public class ClientOptions extends VersionedAbstractParam { private int maxChildren; private int maxDepth = 5; private int maxDuration; + private int maxScansInUi = 5; @Override public ClientOptions clone() { @@ -79,6 +81,7 @@ protected void parseImpl() { this.maxChildren = getInt(MAX_CHILDREN_KEY, 0); this.maxDepth = getInt(MAX_DEPTH_KEY, 5); this.maxDuration = getInt(MAX_DURATION_KEY, 0); + this.maxScansInUi = getInt(MAX_SCANS_IN_UI_KEY, 5); try { pscanRulesDisabled = @@ -216,4 +219,13 @@ public void setMaxDuration(int maxDuration) { this.maxDuration = maxDuration; getConfig().setProperty(MAX_DURATION_KEY, maxDuration); } + + public int getMaxScansInUi() { + return this.maxScansInUi; + } + + public void setMaxScansInUi(int maxScansInUi) { + this.maxScansInUi = maxScansInUi; + getConfig().setProperty(MAX_SCANS_IN_UI_KEY, maxScansInUi); + } } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java index f3d5203bf8..833b19ffb2 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java @@ -19,6 +19,7 @@ */ package org.zaproxy.addon.client; +import java.awt.EventQueue; import java.awt.event.InputEvent; import java.awt.event.KeyEvent; import java.io.File; @@ -34,6 +35,9 @@ import java.util.Collections; import java.util.List; import javax.swing.ImageIcon; +import javax.swing.SwingUtilities; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -64,7 +68,9 @@ import org.zaproxy.addon.client.spider.AuthenticationHandler; import org.zaproxy.addon.client.spider.ClientSpider; import org.zaproxy.addon.client.spider.ClientSpiderDialog; +import org.zaproxy.addon.client.spider.ClientSpiderPanel; import org.zaproxy.addon.client.spider.PopupMenuSpider; +import org.zaproxy.addon.client.spider.SpiderScanController; import org.zaproxy.addon.client.ui.ClientDetailsPanel; import org.zaproxy.addon.client.ui.ClientHistoryPanel; import org.zaproxy.addon.client.ui.ClientMapPanel; @@ -85,8 +91,10 @@ import org.zaproxy.zap.extension.selenium.ExtensionSelenium; import org.zaproxy.zap.extension.selenium.ProfileManager; import org.zaproxy.zap.model.ScanEventPublisher; +import org.zaproxy.zap.model.Target; import org.zaproxy.zap.users.User; import org.zaproxy.zap.utils.DisplayUtils; +import org.zaproxy.zap.utils.ThreadUtils; import org.zaproxy.zap.view.ZapMenuItem; public class ExtensionClientIntegration extends ExtensionAdaptor { @@ -114,17 +122,18 @@ public class ExtensionClientIntegration extends ExtensionAdaptor { private ClientMapPanel clientMapPanel; private ClientDetailsPanel clientDetailsPanel; private ClientHistoryPanel clientHistoryPanel; + private ClientSpiderPanel clientSpiderPanel; private ClientHistoryTableModel clientHistoryTableModel; private RedirectScript redirectScript; private ClientZestRecorder clientHandler; - private ClientPassiveScanController scanController; + private SpiderScanController spiderScanController; + private ClientPassiveScanController passiveScanController; private ClientPassiveScanHelper pscanHelper; private ClientOptions clientParam; private ClientIntegrationAPI api; private EventConsumer eventConsumer; private Event lastAjaxSpiderStartEvent; - private List spiders = Collections.synchronizedList(new ArrayList<>()); - private ImageIcon icon; + private static ImageIcon icon; private ClientSpiderDialog spiderDialog; private ZapMenuItem menuItemCustomScan; @@ -146,15 +155,17 @@ public void initModel(Model model) { new ClientSideDetails( Constant.messages.getString("client.tree.title"), null), this.getModel().getSession())); + spiderScanController = new SpiderScanController(this); + passiveScanController = new ClientPassiveScanController(); } @Override public void hook(ExtensionHook extensionHook) { super.hook(extensionHook); - scanController = new ClientPassiveScanController(); this.api = new ClientIntegrationAPI(this); + extensionHook.addSessionListener(new SessionChangedListenerImpl()); extensionHook.addOptionsParamSet(getClientParam()); extensionHook.addApiImplementor(this.api); extensionHook.addSessionListener(new SessionChangeListener()); @@ -172,6 +183,7 @@ public void hook(ExtensionHook extensionHook) { .addPopupMenuItem( new PopupMenuSpider( Constant.messages.getString("client.attack.spider"), this)); + extensionHook.getHookView().addStatusPanel(getClientSpiderPanel()); } // Client Map menu items @@ -253,14 +265,16 @@ public void hook(ExtensionHook extensionHook) { Constant.messages.getString("client.details.popup.copy.texts"), ClientSideComponent::getText)); - extensionHook.getHookView().addOptionPanel(new OptionsPassiveScan(scanController)); + extensionHook + .getHookView() + .addOptionPanel(new OptionsPassiveScan(passiveScanController)); } } @Override public void optionsLoaded() { - scanController.setEnabled(getClientParam().isPscanEnabled()); - scanController.setDisabledScanRules(getClientParam().getPscanRulesDisabled()); + passiveScanController.setEnabled(getClientParam().isPscanEnabled()); + passiveScanController.setDisabledScanRules(getClientParam().getPscanRulesDisabled()); } @Override @@ -403,6 +417,9 @@ public void unload() { if (eventConsumer != null) { ZAP.getEventBus().unregisterConsumer(eventConsumer); } + if (hasView()) { + getClientSpiderPanel().unload(); + } } @Override @@ -412,11 +429,7 @@ public boolean canUnload() { @Override public void destroy() { - stopAllSpiders(); - } - - private void stopAllSpiders() { - spiders.forEach(ClientSpider::stop); + this.spiderScanController.stopAllScans(); } public ClientNode getOrAddClientNode(String url, boolean visited, boolean storage) { @@ -476,6 +489,20 @@ private ClientHistoryPanel getClientHistoryPanel() { return clientHistoryPanel; } + private ClientSpiderPanel getClientSpiderPanel() { + if (clientSpiderPanel == null) { + clientSpiderPanel = + new ClientSpiderPanel(this, this.spiderScanController, this.getClientParam()); + } + return clientSpiderPanel; + } + + public void updateAddedCount() { + if (getView() != null) { + getClientSpiderPanel().updateAddedCount(); + } + } + public void addReportedObject(ReportedObject obj) { if (obj instanceof ReportedEvent) { ReportedEvent ev = (ReportedEvent) obj; @@ -493,7 +520,7 @@ public void addReportedObject(ReportedObject obj) { } } this.clientHistoryTableModel.addReportedObject(obj); - this.scanController + this.passiveScanController .getEnabledScanRules() .forEach( s -> { @@ -506,7 +533,7 @@ public void addReportedObject(ReportedObject obj) { } public ClientPassiveScanController getPassiveScanController() { - return this.scanController; + return this.passiveScanController; } @Override @@ -539,12 +566,12 @@ public void sessionChanged(Session session) { if (clientHistoryTableModel != null) { clientHistoryTableModel.clear(); } - spiders.clear(); + spiderScanController.reset(); } @Override public void sessionAboutToChange(Session session) { - stopAllSpiders(); + spiderScanController.stopAllScans(); } @Override @@ -577,54 +604,12 @@ protected static boolean isApiUrl(String url) { return url.startsWith(API.API_URL) || url.startsWith(API.API_URL_S); } - /** - * Run the client spider with the configured options - * - * @param url The inital URL to request - * @return an id which can be used to reference the specific scan. - */ - public int runSpider(String url) { - return this.runSpider(url, this.getClientParam()); - } - - /** - * Run the client spider with the specified options - * - * @param url The inital URL to request - * @param options Custom options. - * @param user the user to be used for authentication. - * @return an id which can be used to reference the specific scan. - */ - public int runSpider(String url, ClientOptions options, User user) { - synchronized (spiders) { - ClientSpider cs = new ClientSpider(this, url, options, spiders.size(), user); - spiders.add(cs); - cs.start(); - return spiders.indexOf(cs); - } - } - - /** - * Run the client spider with the specified options - * - * @param url The initial URL to request - * @param options Custom options. - * @return an id which can be used to reference the specific scan. - */ - public int runSpider(String url, ClientOptions options) { - return this.runSpider(url, options, null); - } - - public ClientSpider getSpider(int id) { - return this.spiders.get(id); - } - @Override public List getActiveActions() { List activeActions = new ArrayList<>(); String actionPrefix = Constant.messages.getString("client.activeActionPrefix"); - spiders.stream() - .filter(cs -> cs.isRunning()) + this.spiderScanController + .getActiveScans() .forEach( cs -> activeActions.add( @@ -671,7 +656,7 @@ private ZapMenuItem getMenuItemCustomScan() { return menuItemCustomScan; } - public ImageIcon getIcon() { + public static ImageIcon getIcon() { if (icon == null) { icon = getIcon("spiderClient.png"); } @@ -687,4 +672,128 @@ public static ImageIcon getIcon(String name) { } return DisplayUtils.getScaledIcon(url); } + + /** + * Abbreviates (the middle of) the given display name if greater than 30 characters. + * + * @param displayName the display name that might be abbreviated + * @return the, possibly, abbreviated display name + */ + private static String abbreviateDisplayName(String displayName) { + return StringUtils.abbreviateMiddle(displayName, "..", 30); + } + + public int startScan(String url, ClientOptions options, User user) + throws URIException, NullPointerException { + return this.startScan( + abbreviateDisplayName(url), null, user, new Object[] {new URI(url, true), options}); + } + + public int startScan( + String displayName, Target target, User user, Object[] contextSpecificObjects) { + int id = + this.spiderScanController.startScan( + displayName, target, user, contextSpecificObjects); + if (hasView()) { + addScanToUi(this.spiderScanController.getScan(id)); + } + return id; + } + + public List getAllScans() { + return this.spiderScanController.getAllScans(); + } + + public List getActiveScans() { + return this.spiderScanController.getActiveScans(); + } + + public ClientSpider getScan(int id) { + return this.spiderScanController.getScan(id); + } + + public void stopScan(int id) { + this.spiderScanController.stopScan(id); + } + + public void stopAllScans() { + this.spiderScanController.stopAllScans(); + } + + public void pauseScan(int id) { + this.spiderScanController.pauseScan(id); + } + + public void pauseAllScans() { + this.spiderScanController.pauseAllScans(); + } + + public void resumeScan(int id) { + this.spiderScanController.resumeScan(id); + } + + public void resumeAllScans() { + this.spiderScanController.resumeAllScans(); + } + + private void addScanToUi(final ClientSpider scan) { + if (!Constant.isDevBuild()) { + return; + } + + if (!EventQueue.isDispatchThread()) { + SwingUtilities.invokeLater(() -> addScanToUi(scan)); + return; + } + + this.getClientSpiderPanel().scannerStarted(scan); + scan.setListener(getClientSpiderPanel()); + this.getClientSpiderPanel().switchView(scan); + this.getClientSpiderPanel().setTabFocus(); + } + + private class SessionChangedListenerImpl implements SessionChangedListener { + + @Override + public void sessionAboutToChange(Session session) { + spiderScanController.reset(); + + if (hasView()) { + if (Constant.isDevBuild()) { + getClientSpiderPanel().reset(); + } + if (spiderDialog != null) { + spiderDialog.reset(); + } + } + } + + @Override + public void sessionChanged(final Session session) { + if (hasView() && Constant.isDevBuild()) { + ThreadUtils.invokeAndWaitHandled(getClientSpiderPanel()::reset); + } + } + + @Override + public void sessionScopeChanged(Session session) { + if (hasView() && Constant.isDevBuild()) { + getClientSpiderPanel().sessionScopeChanged(session); + } + } + + @Override + public void sessionModeChanged(Mode mode) { + if (Mode.safe.equals(mode)) { + spiderScanController.stopAllScans(); + } + + if (hasView()) { + if (Constant.isDevBuild()) { + getClientSpiderPanel().sessionModeChanged(mode); + } + getMenuItemCustomScan().setEnabled(!Mode.safe.equals(mode)); + } + } + } } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java index 1c4bcfc1fc..9a962ae5a0 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java @@ -29,12 +29,12 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.swing.table.TableModel; import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.openqa.selenium.WebDriver; import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.view.View; import org.zaproxy.addon.client.ClientOptions; import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.addon.client.internal.ClientMap; @@ -43,16 +43,18 @@ import org.zaproxy.zap.eventBus.Event; import org.zaproxy.zap.eventBus.EventConsumer; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; +import org.zaproxy.zap.model.GenericScanner2; +import org.zaproxy.zap.model.ScanListenner2; import org.zaproxy.zap.users.User; +import org.zaproxy.zap.utils.ThreadUtils; -public class ClientSpider implements EventConsumer { +public class ClientSpider implements EventConsumer, GenericScanner2 { /* * Client Spider status - Work In Progress. * This functionality has not yet been officially released, so do not rely on any of the classes or methods for now. * * TODO The following features will need to be implemented before the first release: - * GUI (!) * Separate proxy (or maybe even one proxy per browser?) * Support for modes * Help pages @@ -71,8 +73,9 @@ public class ClientSpider implements EventConsumer { private ExecutorService threadPool; - private int id; private ClientOptions options; + private int scanId; + private String displayName; private String targetUrl; private User user; @@ -93,17 +96,23 @@ public class ClientSpider implements EventConsumer { private int tasksDoneCount; private int tasksTotalCount; + private UrlTableModel addedNodesModel; + private ScanListenner2 listener; + public ClientSpider( ExtensionClientIntegration extClient, + String displayName, String targetUrl, ClientOptions options, int id, User user) { this.extClient = extClient; + this.displayName = displayName; this.targetUrl = targetUrl; this.options = options; - this.id = id; + this.scanId = id; this.user = user; + this.addedNodesModel = new UrlTableModel(); ZAP.getEventBus().registerConsumer(this, ClientMap.class.getCanonicalName()); @@ -112,11 +121,16 @@ public ClientSpider( } public ClientSpider( - ExtensionClientIntegration extClient, String targetUrl, ClientOptions options, int id) { - this(extClient, targetUrl, options, id, null); + ExtensionClientIntegration extClient, + String displayName, + String targetUrl, + ClientOptions options, + int id) { + this(extClient, displayName, targetUrl, options, id, null); } - public void start() { + @Override + public void run() { startTime = System.currentTimeMillis(); lastEventReceivedtime = startTime; if (options.getMaxDuration() > 0) { @@ -134,7 +148,7 @@ public void start() { Executors.newFixedThreadPool( options.getThreadCount(), new ClientSpiderThreadFactory( - "ZAP-ClientSpiderThreadPool-" + id + "-thread-")); + "ZAP-ClientSpiderThreadPool-" + scanId + "-thread-")); List unvisitedUrls = getUnvisitedUrls(); @@ -199,7 +213,7 @@ private void addTask(String url, int loadTimeInSecs) { executeTask(task); } } catch (RejectedExecutionException e) { - tempLogProgress("Failed to add task: " + e.getMessage()); + LOGGER.debug("Failed to add task", e.getMessage()); } } @@ -211,26 +225,31 @@ private void executeTask(ClientSpiderTask task) { protected synchronized void postTaskExecution(ClientSpiderTask task) { this.tasksDoneCount++; this.spiderTasks.remove(task); - if (this.spiderTasks.isEmpty()) { - this.tempLogProgress("No running tasks, starting shutdown timer"); + if (listener != null) { + listener.scanProgress(scanId, displayName, this.getProgress(), this.getMaximum()); + } + if (this.spiderTasks.isEmpty() && !paused) { + LOGGER.debug("No running tasks, starting shutdown timer"); new ShutdownThread(options.getShutdownTimeInSecs()).start(); } } @Override public void eventReceived(Event event) { - if (finished) { + if (finished || stopped) { return; } this.lastEventReceivedtime = System.currentTimeMillis(); if (maxTime > 0 && this.lastEventReceivedtime > maxTime) { - this.tempLogProgress("Exceeded max time, stopping"); - this.stop(); + LOGGER.debug("Exceeded max time, stopping"); + this.stopScan(); return; } String url = event.getParameters().get(ClientMap.URL_KEY); if (url.startsWith(targetUrl)) { + addUriToAddedNodesModel(url); + if (options.getMaxDepth() > 0) { int depth = Integer.parseInt(event.getParameters().get(ClientMap.DEPTH_KEY)); if (depth > options.getMaxDepth()) { @@ -257,6 +276,15 @@ public void eventReceived(Event event) { } } + private void addUriToAddedNodesModel(final String uri) { + ThreadUtils.invokeLater( + () -> { + addedNodesModel.addScanResult(uri); + extClient.updateAddedCount(); + }); + } + + @Override public int getProgress() { if (finished && !stopped) { return 100; @@ -267,7 +295,8 @@ public int getProgress() { return (this.tasksDoneCount * 100) / this.tasksTotalCount; } - public void stop() { + @Override + public void stopScan() { this.stopped = true; if (paused) { this.pausedTasks.clear(); @@ -277,23 +306,28 @@ public void stop() { ZAP.getEventBus().unregisterConsumer(this, ClientMap.class.getCanonicalName()); } - public void pause() { + @Override + public void pauseScan() { this.paused = true; } - public void resume() { + @Override + public void resumeScan() { this.pausedTasks.forEach(this::executeTask); this.paused = false; } - protected boolean isStopped() { - return stopped; + @Override + public boolean isStopped() { + return finished || stopped; } + @Override public boolean isPaused() { return paused; } + @Override public boolean isRunning() { return !finished; } @@ -302,23 +336,11 @@ public String getTargetUrl() { return targetUrl; } - /** - * TODO This is a temporary method used to record progress. It will be removed once the GUI has - * been implemented. Messages are not expected to be i18n'ed. - */ - protected void tempLogProgress(String msg) { - if (View.isInitialised()) { - View.getSingleton().getOutputPanel().appendAsync(msg + "\n"); - } else { - LOGGER.debug(msg); - } - } - private void finished() { finished = true; long timeTaken = System.currentTimeMillis() - startTime; - tempLogProgress( - "Spider finished " + DurationFormatUtils.formatDuration(timeTaken, "HH:MM:SS")); + LOGGER.debug( + "Spider finished {}", DurationFormatUtils.formatDuration(timeTaken, "HH:MM:SS")); if (this.user != null) { synchronized (extClient.getAuthenticationHandlers()) { extClient @@ -336,6 +358,9 @@ private void finished() { } this.webDriverActive.clear(); } + if (listener != null) { + listener.scanFinshed(scanId, displayName); + } } private class ShutdownThread extends Thread { @@ -357,10 +382,9 @@ public void run() { // Ignore } if (lastEventReceivedtime > starttime) { - // New event, don't shutdown - tempLogProgress("Spider not finished.."); + LOGGER.debug("Spider not finished.."); if (spiderTasks.isEmpty()) { - tempLogProgress("No running tasks, restarting shutdown timer"); + LOGGER.debug("No running tasks, restarting shutdown timer"); new ShutdownThread(options.getShutdownTimeInSecs()).start(); } return; @@ -395,4 +419,37 @@ public Thread newThread(Runnable r) { return t; } } + + @Override + public void setScanId(int id) { + this.scanId = id; + } + + @Override + public int getScanId() { + return this.scanId; + } + + @Override + public void setDisplayName(String name) { + this.displayName = name; + } + + @Override + public String getDisplayName() { + return this.displayName; + } + + @Override + public int getMaximum() { + return 100; + } + + public TableModel getAddedNodesTableModel() { + return this.addedNodesModel; + } + + public void setListener(ScanListenner2 listener) { + this.listener = listener; + } } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java index 125300d40f..8b9179e40c 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java @@ -368,7 +368,12 @@ public void save() { subtreeOnlyPreviousCheckedState = getBoolValue(FIELD_SUBTREE_ONLY); User user = this.getSelectedUser(); - this.extension.runSpider(getStartUrl(), clientParams, user); + try { + this.extension.startScan(getStartUrl(), clientParams, user); + } catch (Exception e) { + // Should not happen as we will already have checked the URL + LOGGER.debug("Failed to start client spider", e); + } } /** diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java new file mode 100644 index 0000000000..9a5e028886 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java @@ -0,0 +1,213 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.awt.BorderLayout; +import java.awt.EventQueue; +import javax.swing.JButton; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.JScrollPane; +import javax.swing.JTabbedPane; +import javax.swing.JToolBar; +import javax.swing.ListSelectionModel; +import javax.swing.SwingUtilities; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jdesktop.swingx.JXTable; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.model.SiteNode; +import org.parosproxy.paros.view.View; +import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.ExtensionClientIntegration; +import org.zaproxy.zap.model.ScanController; +import org.zaproxy.zap.model.ScanListenner2; +import org.zaproxy.zap.view.ScanPanel2; +import org.zaproxy.zap.view.ZapTable; + +@SuppressWarnings("serial") +public class ClientSpiderPanel extends ScanPanel2> + implements ScanListenner2 { + + public static final String HTTP_MESSAGE_CONTAINER_NAME = "ClientSpiderHttpMessageContainer"; + + private static final long serialVersionUID = 1L; + + private static final Logger LOGGER = LogManager.getLogger(ClientSpiderPanel.class); + + private static final String ZERO_REQUESTS_LABEL_TEXT = "0"; + + private static final UrlTableModel EMPTY_TABLE_MODEL = new UrlTableModel(); + + public static final String PANEL_NAME = "ClientPanel"; + + private static final String ADDED_NODES_CONTAINER_NAME = "ClientAddedNodesContainer"; + + private JPanel mainPanel; + private JTabbedPane tabbedPane; + private JButton scanButton; + private ZapTable addedNodesTable; + private JScrollPane addedNodesTableScrollPane; + private JLabel addedCountNameLabel; + private JLabel addedCountValueLabel; + + private ExtensionClientIntegration extension; + private ClientOptions clientOptions; + + public ClientSpiderPanel( + ExtensionClientIntegration extension, + SpiderScanController controller, + ClientOptions clientOptions) { + super("client.spider", ExtensionClientIntegration.getIcon(), controller); + this.extension = extension; + this.clientOptions = clientOptions; + } + + @Override + protected JPanel getWorkPanel() { + if (mainPanel == null) { + mainPanel = new JPanel(new BorderLayout()); + + tabbedPane = new JTabbedPane(); + tabbedPane.addTab( + Constant.messages.getString("client.spider.panel.tab.addednodes"), + getAddedNodesTableScrollPane()); + tabbedPane.setSelectedIndex(0); + + mainPanel.add(tabbedPane); + } + return mainPanel; + } + + private JScrollPane getAddedNodesTableScrollPane() { + if (addedNodesTableScrollPane == null) { + addedNodesTableScrollPane = new JScrollPane(); + addedNodesTableScrollPane.setName("ClientSpiderAddedUrlsPane"); + addedNodesTableScrollPane.setViewportView(getAddedNodesTable()); + } + return addedNodesTableScrollPane; + } + + private JXTable getAddedNodesTable() { + if (addedNodesTable == null) { + addedNodesTable = new ZapTable(EMPTY_TABLE_MODEL); + addedNodesTable.setColumnSelectionAllowed(false); + addedNodesTable.setCellSelectionEnabled(false); + addedNodesTable.setRowSelectionAllowed(true); + addedNodesTable.setAutoCreateRowSorter(true); + + addedNodesTable.setAutoCreateColumnsFromModel(false); + + addedNodesTable.setName(ADDED_NODES_CONTAINER_NAME); + addedNodesTable.setDoubleBuffered(true); + addedNodesTable.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); + } + return addedNodesTable; + } + + private JLabel getAddedCountNameLabel() { + if (addedCountNameLabel == null) { + addedCountNameLabel = new JLabel(); + addedCountNameLabel.setText( + Constant.messages.getString("client.spider.toolbar.added.label")); + } + return addedCountNameLabel; + } + + private JLabel getAddedCountValueLabel() { + if (addedCountValueLabel == null) { + addedCountValueLabel = new JLabel(); + addedCountValueLabel.setText(ZERO_REQUESTS_LABEL_TEXT); + } + return addedCountValueLabel; + } + + @Override + protected int addToolBarElements(JToolBar toolBar, Location location, int gridX) { + if (ScanPanel2.Location.afterProgressBar == location) { + toolBar.add(new JToolBar.Separator(), getGBC(gridX++, 0)); + toolBar.add(getAddedCountNameLabel(), getGBC(gridX++, 0)); + toolBar.add(getAddedCountValueLabel(), getGBC(gridX++, 0)); + + toolBar.add(new JToolBar.Separator(), getGBC(gridX++, 0)); + } + return gridX; + } + + /** Update the count of added nodes. */ + public void updateAddedCount() { + ClientSpider sc = this.getSelectedScanner(); + if (sc != null) { + this.getAddedCountValueLabel() + .setText(Integer.toString(sc.getAddedNodesTableModel().getRowCount())); + } else { + this.getAddedCountValueLabel().setText(ZERO_REQUESTS_LABEL_TEXT); + } + } + + @Override + public void switchView(final ClientSpider scanner) { + if (View.isInitialised() && !EventQueue.isDispatchThread()) { + SwingUtilities.invokeLater(() -> switchView(scanner)); + return; + } + if (scanner != null) { + getAddedNodesTable().setModel(scanner.getAddedNodesTableModel()); + } else { + getAddedNodesTable().setModel(EMPTY_TABLE_MODEL); + } + this.updateAddedCount(); + } + + @Override + public JButton getNewScanButton() { + if (scanButton == null) { + scanButton = + new JButton(Constant.messages.getString("client.spider.toolbar.button.new")); + scanButton.setIcon(ExtensionClientIntegration.getIcon()); + scanButton.addActionListener(e -> extension.showScanDialog(getSiteTreeTarget())); + } + return scanButton; + } + + @Override + protected int getNumberOfScansToShow() { + return clientOptions.getMaxScansInUi(); + } + + private SiteNode getSiteTreeTarget() { + if (!extension.getView().getSiteTreePanel().getTreeSite().isSelectionEmpty()) { + return (SiteNode) + extension + .getView() + .getSiteTreePanel() + .getTreeSite() + .getSelectionPath() + .getLastPathComponent(); + } + return null; + } + + // Overridden to expose the method to the extension + @Override + public void unload() { + super.unload(); + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderTask.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderTask.java index 8c1d706c29..01d3cdc309 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderTask.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderTask.java @@ -20,10 +20,14 @@ package org.zaproxy.addon.client.spider; import java.time.Duration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.openqa.selenium.WebDriver; public class ClientSpiderTask implements Runnable { + private static final Logger LOGGER = LogManager.getLogger(ClientSpiderTask.class); + private ClientSpider clientSpider; private String url; private int timeout; @@ -56,19 +60,16 @@ public void run() { wd.get(url); ok = true; } catch (Exception e) { - clientSpider.tempLogProgress("Task failed " + url + " " + e.getMessage()); + LOGGER.warn("Task failed {} {}", url, e.getMessage(), e); } if (wd != null) { this.clientSpider.returnWebDriver(wd); } - clientSpider.tempLogProgress( - "Task completed " - + url - + " " - + ok - + " in " - + (System.currentTimeMillis() - startTime) / 1000 - + " secs"); + LOGGER.debug( + "Task completed {} {} in {} secs", + url, + ok, + (System.currentTimeMillis() - startTime) / 1000); this.clientSpider.postTaskExecution(this); } } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/PopupMenuSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/PopupMenuSpider.java index c6a4189d64..b4b5b5d967 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/PopupMenuSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/PopupMenuSpider.java @@ -34,7 +34,7 @@ public class PopupMenuSpider extends PopupMenuItemSiteNodeContainer { public PopupMenuSpider(String label, ExtensionClientIntegration extension) { super(label); - this.setIcon(extension.getIcon()); + this.setIcon(ExtensionClientIntegration.getIcon()); this.extension = extension; } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java new file mode 100644 index 0000000000..3a5139f983 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java @@ -0,0 +1,317 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.httpclient.URI; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.ExtensionClientIntegration; +import org.zaproxy.zap.model.ScanController; +import org.zaproxy.zap.model.Target; +import org.zaproxy.zap.users.User; + +public class SpiderScanController implements ScanController { + + private static final Logger LOGGER = LogManager.getLogger(SpiderScanController.class); + + private ExtensionClientIntegration extension; + + /** + * The {@code Lock} for exclusive access of instance variables related to multiple active scans. + * + * @see #clientSpiderMap + * @see #scanIdCounter + */ + private final Lock clientSpidersLock; + + /** + * The counter used to give an unique ID to active scans. + * + *

Note: All accesses (both write and read) should be done while holding the + * {@code Lock} {@code clientSpidersLock}. + * + * @see #clientSpidersLock + * @see #startScan(String, Target, User, Object[]) + */ + private int scanIdCounter; + + /** + * A map that contains all {@code ClientSpider}s created (and not yet removed). Used to control + * (i.e. pause/resume and stop) the multiple active scans and get its results. The instance + * variable is never {@code null}. The map key is the ID of the scan. + * + *

Note: All accesses (both write and read) should be done while holding the + * {@code Lock} {@code clientSpidersLock}. + * + * @see #clientSpidersLock + * @see #startScan(String, Target, User, Object[]) + * @see #scanIdCounter + */ + private Map clientSpiderMap; + + /** + * An ordered list of all of the {@code ClientSpider}s created (and not yet removed). Used to + * get provide the 'last' scan for client using the 'old' API that didn't support concurrent + * scans. + */ + private List clientSpiderList; + + public SpiderScanController(ExtensionClientIntegration extension) { + this.clientSpidersLock = new ReentrantLock(); + this.extension = extension; + this.clientSpiderMap = new HashMap<>(); + this.clientSpiderList = new ArrayList<>(); + } + + @Override + public int startScan(String name, Target target, User user, Object[] contextSpecificObjects) { + clientSpidersLock.lock(); + try { + int id = this.scanIdCounter++; + + ClientOptions clientOptions = extension.getClientParam(); + URI startUri = null; + + if (contextSpecificObjects != null) { + for (Object obj : contextSpecificObjects) { + if (obj instanceof ClientOptions) { + LOGGER.debug("Setting custom spider params"); + clientOptions = (ClientOptions) obj; + } else if (obj instanceof URI) { + startUri = (URI) obj; + } else { + LOGGER.error( + "Unexpected contextSpecificObject: {}", + obj.getClass().getCanonicalName()); + } + } + } + + ClientSpider scan = + new ClientSpider(extension, name, startUri.toString(), clientOptions, id, user); + + this.clientSpiderMap.put(id, scan); + this.clientSpiderList.add(scan); + scan.run(); + + return id; + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public ClientSpider getScan(int id) { + return this.clientSpiderMap.get(id); + } + + @Override + public ClientSpider getLastScan() { + clientSpidersLock.lock(); + try { + if (clientSpiderList.isEmpty()) { + return null; + } + return clientSpiderList.get(clientSpiderList.size() - 1); + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public List getAllScans() { + List list = new ArrayList<>(); + clientSpidersLock.lock(); + try { + for (ClientSpider scan : clientSpiderList) { + list.add(scan); + } + return list; + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public List getActiveScans() { + List list = new ArrayList<>(); + clientSpidersLock.lock(); + try { + for (ClientSpider scan : clientSpiderList) { + if (!scan.isStopped()) { + list.add(scan); + } + } + return list; + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public ClientSpider removeScan(int id) { + clientSpidersLock.lock(); + + try { + ClientSpider ascan = this.clientSpiderMap.get(id); + if (!clientSpiderMap.containsKey(id)) { + return null; + } + ascan.stopScan(); + clientSpiderMap.remove(id); + clientSpiderList.remove(ascan); + return ascan; + } finally { + clientSpidersLock.unlock(); + } + } + + public int getTotalNumberScans() { + return clientSpiderMap.size(); + } + + @Override + public void stopAllScans() { + clientSpidersLock.lock(); + try { + for (ClientSpider scan : clientSpiderMap.values()) { + scan.stopScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public void pauseAllScans() { + clientSpidersLock.lock(); + try { + for (ClientSpider scan : clientSpiderMap.values()) { + scan.pauseScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public void resumeAllScans() { + clientSpidersLock.lock(); + try { + for (ClientSpider scan : clientSpiderMap.values()) { + scan.resumeScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public int removeAllScans() { + clientSpidersLock.lock(); + try { + int count = 0; + for (Iterator it = clientSpiderMap.values().iterator(); it.hasNext(); ) { + ClientSpider ascan = it.next(); + ascan.stopScan(); + it.remove(); + clientSpiderList.remove(ascan); + count++; + } + return count; + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public int removeFinishedScans() { + clientSpidersLock.lock(); + try { + int count = 0; + for (Iterator it = clientSpiderMap.values().iterator(); it.hasNext(); ) { + ClientSpider scan = it.next(); + if (scan.isStopped()) { + scan.stopScan(); + it.remove(); + clientSpiderList.remove(scan); + count++; + } + } + return count; + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public void stopScan(int id) { + clientSpidersLock.lock(); + try { + if (this.clientSpiderMap.containsKey(id)) { + this.clientSpiderMap.get(id).stopScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public void pauseScan(int id) { + clientSpidersLock.lock(); + try { + if (this.clientSpiderMap.containsKey(id)) { + this.clientSpiderMap.get(id).pauseScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + @Override + public void resumeScan(int id) { + clientSpidersLock.lock(); + try { + if (this.clientSpiderMap.containsKey(id)) { + this.clientSpiderMap.get(id).resumeScan(); + } + } finally { + clientSpidersLock.unlock(); + } + } + + public void reset() { + this.removeAllScans(); + clientSpidersLock.lock(); + try { + this.scanIdCounter = 0; + } finally { + clientSpidersLock.unlock(); + } + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/UrlTableModel.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/UrlTableModel.java new file mode 100644 index 0000000000..1c8b5c63d2 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/UrlTableModel.java @@ -0,0 +1,130 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.util.ArrayList; +import java.util.List; +import javax.swing.table.AbstractTableModel; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.parosproxy.paros.Constant; + +@SuppressWarnings("serial") +public class UrlTableModel extends AbstractTableModel { + + private static final long serialVersionUID = -6380136823410869457L; + + private static final String[] COLUMN_NAMES = { + Constant.messages.getString("client.spider.panel.table.header.uri"), + }; + + private static final int COLUMN_COUNT = COLUMN_NAMES.length; + + private List scanResults; + + public UrlTableModel() { + super(); + + scanResults = new ArrayList<>(); + } + + @Override + public String getColumnName(int column) { + return COLUMN_NAMES[column]; + } + + @Override + public int getColumnCount() { + return COLUMN_COUNT; + } + + @Override + public int getRowCount() { + return scanResults.size(); + } + + @Override + public Object getValueAt(int row, int col) { + UrlScanResult result = scanResults.get(row); + switch (col) { + case 0: + return result.getUri(); + default: + return null; + } + } + + public void removeAllElements() { + scanResults.clear(); + fireTableDataChanged(); + } + + public void addScanResult(String uri) { + UrlScanResult result = new UrlScanResult(uri); + scanResults.add(result); + fireTableRowsInserted(scanResults.size() - 1, scanResults.size() - 1); + } + + public void removesScanResult(String uri) { + UrlScanResult toRemove = new UrlScanResult(uri); + int index = scanResults.indexOf(toRemove); + if (index >= 0) { + scanResults.remove(index); + fireTableRowsDeleted(index, index); + } + } + + /** + * Returns the type of column for given column index. + * + * @param columnIndex the column index + * @return the column class + */ + @Override + public Class getColumnClass(int columnIndex) { + switch (columnIndex) { + case 0: + return String.class; + } + return null; + } + + /** + * The Class UrlScanResult that stores an entry in the table (a result for the spidering + * process). + */ + @EqualsAndHashCode + @Getter + private static class UrlScanResult { + private String uri; + + protected UrlScanResult(String uri) { + this.uri = uri; + } + } + + public List getAddedNodes() { + List list = new ArrayList<>(this.scanResults.size()); + for (UrlScanResult res : this.scanResults) { + list.add(res.uri); + } + return list; + } +} diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties index 61c827672e..929c657b79 100644 --- a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties +++ b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties @@ -91,6 +91,20 @@ client.scandialog.tab.scope = Scope client.scandialog.title = Client Spider client.spider.menu.tools.label = Client Spider +client.spider.options.title = Client Options +client.spider.panel.tab.addednodes = Added Nodes +client.spider.panel.tab.urls = URLs +client.spider.panel.table.header.uri = URI +client.spider.panel.title = Client Spider +client.spider.toolbar.added.label = Nodes Added: +client.spider.toolbar.ascans.label = Current Scans: +client.spider.toolbar.button.new = New Scan +client.spider.toolbar.button.options = Client Options +client.spider.toolbar.button.pause = Pause Spider +client.spider.toolbar.button.stop = Stop Spider +client.spider.toolbar.button.unpause = Resume Spider +client.spider.toolbar.progress.label = Progress: +client.spider.toolbar.progress.select = --Select Scan-- client.tree.popup.attack = Attack client.tree.popup.browser = Open in Browser... diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java index 89248e60f4..f847aa75bf 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java @@ -149,10 +149,10 @@ void shouldStartSpider() throws IOException { try { // When - int spiderId = extClient.runSpider("https://www.example.com", options); - ClientSpider spider = extClient.getSpider(spiderId); + int spiderId = extClient.startScan("https://www.example.com", options, null); + ClientSpider spider = extClient.getScan(spiderId); boolean isRunning = spider.isRunning(); - spider.stop(); + spider.stopScan(); // Then assertEquals(true, isRunning); diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java index 56cdc785b5..0d8a8a4d43 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java @@ -89,7 +89,7 @@ void tearDown() { void shouldRequestInScopeUrls() { // Given ClientSpider spider = - new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); + new ClientSpider(extClient, "", "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -97,7 +97,7 @@ void shouldRequestInScopeUrls() { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); // When - spider.start(); + spider.run(); map.getOrAddNode("https://www.example.com/test#1", false, false); // Note the ".org" - this should not be requested map.getOrAddNode("https://www.example.org/test#2", false, false); @@ -107,7 +107,7 @@ void shouldRequestInScopeUrls() { } catch (InterruptedException e) { // Ignore } - spider.stop(); + spider.stopScan(); // Then verify(wd, atLeastOnce()).get(argument.capture()); @@ -126,7 +126,7 @@ void shouldIgnoreRequestAfterStopped() throws Exception { // Given CountDownLatch cdl = new CountDownLatch(1); String seedUrl = "https://www.example.com/"; - ClientSpider spider = new ClientSpider(extClient, seedUrl, clientOptions, 1); + ClientSpider spider = new ClientSpider(extClient, "", seedUrl, clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -134,7 +134,7 @@ void shouldIgnoreRequestAfterStopped() throws Exception { String urlAfterStop = "https://www.example.com/test#1"; doAnswer( invocation -> { - spider.stop(); + spider.stopScan(); map.getOrAddNode(urlAfterStop, false, false); cdl.countDown(); return null; @@ -143,7 +143,7 @@ void shouldIgnoreRequestAfterStopped() throws Exception { .get(seedUrl); // When - spider.start(); + spider.run(); // and stopped on URL access // Then @@ -156,20 +156,20 @@ void shouldIgnoreRequestAfterStopped() throws Exception { void shouldStartPauseResumeStopSpider() { // Given ClientSpider spider = - new ClientSpider(extClient, "https://www.example.com", clientOptions, 1); + new ClientSpider(extClient, "", "https://www.example.com", clientOptions, 1); SpiderStatus statusPostStart; SpiderStatus statusPostPause; SpiderStatus statusPostResume; SpiderStatus statusPostStop; // When - spider.start(); + spider.run(); statusPostStart = new SpiderStatus(spider); - spider.pause(); + spider.pauseScan(); statusPostPause = new SpiderStatus(spider); - spider.resume(); + spider.resumeScan(); statusPostResume = new SpiderStatus(spider); - spider.stop(); + spider.stopScan(); statusPostStop = new SpiderStatus(spider); // Then @@ -195,7 +195,7 @@ void shouldIgnoreUrlsTooDeep() { // Given clientOptions.setMaxDepth(5); ClientSpider spider = - new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); + new ClientSpider(extClient, "", "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -203,7 +203,7 @@ void shouldIgnoreUrlsTooDeep() { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); // When - spider.start(); + spider.run(); map.getOrAddNode("https://www.example.com/l1", false, false); map.getOrAddNode("https://www.example.com/l1/l2", false, false); map.getOrAddNode("https://www.example.com/l1/l2/l3", false, false); @@ -215,7 +215,7 @@ void shouldIgnoreUrlsTooDeep() { } catch (InterruptedException e) { // Ignore } - spider.stop(); + spider.stopScan(); ClientNode l6Node = map.getNode("https://www.example.com/l1/l2/l3/l4/l5/l6", false, false); // Then @@ -238,7 +238,7 @@ void shouldIgnoreUrlsTooWide() { // Given clientOptions.setMaxChildren(4); ClientSpider spider = - new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); + new ClientSpider(extClient, "", "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -246,7 +246,7 @@ void shouldIgnoreUrlsTooWide() { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); // When - spider.start(); + spider.run(); map.getOrAddNode("https://www.example.com/l1", false, false); map.getOrAddNode("https://www.example.com/l2", false, false); map.getOrAddNode("https://www.example.com/l3", false, false); @@ -258,7 +258,7 @@ void shouldIgnoreUrlsTooWide() { } catch (InterruptedException e) { // Ignore } - spider.stop(); + spider.stopScan(); ClientNode l6Node = map.getNode("https://www.example.com/l6", false, false); // Then @@ -280,7 +280,7 @@ void shouldIgnoreUrlsTooWide() { void shouldVisitKnownUnvisitedUrls() { // Given ClientSpider spider = - new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); + new ClientSpider(extClient, "", "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -300,14 +300,14 @@ void shouldVisitKnownUnvisitedUrls() { .thenReturn(exampleSlashNode); // When - spider.start(); + spider.run(); try { Thread.sleep(200); } catch (InterruptedException e) { // Ignore } - spider.stop(); + spider.stopScan(); // Then verify(wd, atLeastOnce()).get(argument.capture());