Add Code Coverage framework

VERSION

@@ -1 +1 @@
-2.6.23
+2.6.24

core/src/org/testar/monkey/alayer/actions/ActionRoles.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
 *
-* Copyright (c) 2013 - 2020 Universitat Politecnica de Valencia - www.upv.es
-* Copyright (c) 2018 - 2020 Open Universiteit - www.ou.nl
+* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es
+* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
@@ -42,6 +42,7 @@ private ActionRoles(){}
 	public static final Role
 
 	Action = Role.from("Action"),
+	NOPAction = Role.from("NOPAction", Action),
 	MouseAction = Role.from("MouseAction", Action),
 	KeyboardAction = Role.from("KeyboardAction", Action),
 	CompoundAction = Role.from("CompoundAction", Action),

core/src/org/testar/monkey/alayer/actions/NOP.java

@@ -1,6 +1,7 @@
 /***************************************************************************************************
 *
-* Copyright (c) 2013, 2014, 2015, 2016, 2017 Universitat Politecnica de Valencia - www.upv.es
+* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es
+* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
@@ -27,10 +28,6 @@
 * POSSIBILITY OF SUCH DAMAGE.
 *******************************************************************************************************/
 
-
-/**
- *  @author Sebastian Bauersfeld
- */
 package org.testar.monkey.alayer.actions;
 
 import org.testar.monkey.Util;
@@ -47,19 +44,22 @@
 public class NOP extends TaggableBase implements Action {
 	private static final long serialVersionUID = 8622084462407313716L;
 
-	public static final String NOP_ID = "No Operation"; // by urueda
-
+	public static final String NOP_ID = "No Operation";
+
+	public NOP() {
+		this.set(Tags.Desc, NOP_ID);
+		this.set(Tags.Role, ActionRoles.NOPAction);
+	}
+
 	public void run(SUT system, State state, double duration){ Util.pause(duration); }
 
 	public String toString(){ return NOP_ID; }
 
-	// by urueda
 	@Override
 	public String toString(Role... discardParameters) {
 		return toString();
 	}
 
-	// by urueda
 	@Override
 	public String toShortString() {
 		Role r = get(Tags.Role, null);
@@ -69,7 +69,6 @@ public String toShortString() {
 			return toString();
 	}
 
-	// by urueda
 	@Override
 	public String toParametersString() {
 		return "";

core/src/org/testar/monkey/alayer/actions/StdActionCompiler.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
  *
- * Copyright (c) 2013 - 2020 Universitat Politecnica de Valencia - www.upv.es
- * Copyright (c) 2018 - 2020 Open Universiteit - www.ou.nl
+ * Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es
+ * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -40,6 +40,7 @@
 import org.testar.monkey.alayer.Action;
 import org.testar.monkey.alayer.Finder;
 import org.testar.monkey.alayer.Position;
+import org.testar.monkey.alayer.State;
 import org.testar.monkey.alayer.StdAbstractor;
 import org.testar.monkey.alayer.Tags;
 import org.testar.monkey.alayer.Widget;
@@ -375,7 +376,13 @@ public Action hitShortcutKey(List<KBKeys> keys) {
 		builder.add(NOP, 1.0);
 		return builder.build();
 	}
-
+
+	public Action noOperationalState(State state) {
+		Action ret = new NOP();
+		ret.mapActionToWidget(state);
+		return ret;
+	}
+
 	public Action killProcessByPID(long pid){ return killProcessByPID(pid, 0); }
 	public Action killProcessByName(String name){ return killProcessByName(name, 0); }
 	public Action killProcessByPID(long pid, double timeToWaitBeforeKilling){ return KillProcess.byPID(pid, timeToWaitBeforeKilling); }

testar/build.gradle

@@ -122,6 +122,12 @@ dependencies {
     implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.1'
     implementation group: 'com.orientechnologies', name: 'orientdb-graphdb', version: '3.0.34'
     implementation group: 'com.sikulix', name: 'sikulixapi', version: '2.0.5'
+    // jacoco and maven dependencies for jacoco coverage
+    implementation group: 'org.apache.maven', name: 'maven-core', version: '3.9.8'
+    implementation group: 'org.apache.maven.reporting', name: 'maven-reporting-api', version: '3.1.1'
+    implementation group: 'org.jacoco', name: 'jacoco-maven-plugin', version: '0.8.12'
+    // https://mvnrepository.com/artifact/org.apache.commons/commons-csv
+    implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.11.0'
 }
 
 task prepareSettings(type: Copy) {

testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java

@@ -0,0 +1,203 @@
+/***************************************************************************************************
+ *
+ * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es
+ * Copyright (c) 2024 Open Universiteit - www.ou.nl
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *******************************************************************************************************/
+
+import org.testar.DerivedActions;
+import org.testar.SutVisualization;
+import org.testar.coverage.CodeCoverageManager;
+import org.testar.monkey.alayer.Action;
+import org.testar.monkey.alayer.SUT;
+import org.testar.monkey.alayer.State;
+import org.testar.monkey.alayer.Verdict;
+import org.testar.monkey.alayer.exceptions.ActionBuildException;
+import org.testar.monkey.alayer.exceptions.StateBuildException;
+import org.testar.monkey.alayer.exceptions.SystemStartException;
+import org.testar.protocols.DesktopProtocol;
+import org.testar.settings.Settings;
+
+import java.util.Set;
+
+public class Protocol_desktop_java_coverage extends DesktopProtocol {
+
+	private CodeCoverageManager codeCoverageManager;
+
+	/**
+	 * Called once during the life time of TESTAR
+	 * This method can be used to perform initial setup work
+	 * @param   settings  the current TESTAR settings as specified by the user.
+	 */
+	@Override
+	protected void initialize(Settings settings){
+		super.initialize(settings);
+		// Initialize the code coverage extractor using the test settings values
+		codeCoverageManager = new CodeCoverageManager(settings);
+	}
+
+
+	/**
+	 * This method is called when TESTAR starts the System Under Test (SUT). The method should
+	 * take care of
+	 *   1) starting the SUT (you can use TESTAR's settings obtainable from <code>settings()</code> to find
+	 *      out what executable to run)
+	 *   2) waiting until the system is fully loaded and ready to be tested (with large systems, you might have to wait several
+	 *      seconds until they have finished loading)
+	 * @return  a started SUT, ready to be tested.
+	 */
+	@Override
+	protected SUT startSystem() throws SystemStartException{
+		return super.startSystem();
+	}
+
+	/**
+	 * This method is invoked each time the TESTAR starts the SUT to generate a new sequence.
+	 * This can be used for example for bypassing a login screen by filling the username and password
+	 * or bringing the system into a specific start state which is identical on each start (e.g. one has to delete or restore
+	 * the SUT's configuration files etc.)
+	 */
+	@Override
+	protected void beginSequence(SUT system, State state){
+		// Before executing the first SUT action, extract the initial coverage
+		codeCoverageManager.getActionCoverage("0");
+		super.beginSequence(system, state);
+	}
+
+	/**
+	 * This method is called when the TESTAR requests the state of the SUT.
+	 * Here you can add additional information to the SUT's state or write your
+	 * own state fetching routine. The state should have attached an oracle
+	 * (TagName: <code>Tags.OracleVerdict</code>) which describes whether the
+	 * state is erroneous and if so why.
+	 *
+	 * super.getState(system) puts the state information also to the HTML sequence report
+	 *
+	 * @return  the current state of the SUT with attached oracle.
+	 */
+	@Override
+	protected State getState(SUT system) throws StateBuildException{
+		return super.getState(system);
+	}
+
+	/**
+	 * The getVerdict methods implements the online state oracles that
+	 * examine the SUT's current state and returns an oracle verdict.
+	 * @return oracle verdict, which determines whether the state is erroneous and why.
+	 */
+	@Override
+	protected Verdict getVerdict(State state){
+		// The super methods implements the implicit online state oracles for:
+		// system crashes
+		// non-responsiveness
+		// suspicious tags
+		Verdict verdict = super.getVerdict(state);
+
+		//--------------------------------------------------------
+		// MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE
+		//--------------------------------------------------------
+
+		return verdict;
+	}
+
+	/**
+	 * This method is used by TESTAR to determine the set of currently available actions.
+	 * You can use the SUT's current state, analyze the widgets and their properties to create
+	 * a set of sensible actions, such as: "Click every Button which is enabled" etc.
+	 * The return value is supposed to be non-null. If the returned set is empty, TESTAR
+	 * will stop generation of the current action and continue with the next one.
+	 * @param system the SUT
+	 * @param state the SUT's current state
+	 * @return  a set of actions
+	 */
+	@Override
+	protected Set<Action> deriveActions(SUT system, State state) throws ActionBuildException{
+		//The super method returns a ONLY actions for killing unwanted processes if needed, or bringing the SUT to
+		//the foreground. You should add all other actions here yourself.
+		// These "special" actions are prioritized over the normal GUI actions in selectAction() / preSelectAction().
+		Set<Action> actions = super.deriveActions(system,state);
+
+		// Derive left-click actions, click and type actions, and scroll actions from
+
+		// all widgets of the GUI:
+		//DerivedActions derived = deriveClickTypeScrollActionsFromAllWidgets(actions, state);
+
+		// the top level widgets of the GUI such as menu items:
+		DerivedActions derived = deriveClickTypeScrollActionsFromTopLevelWidgets(actions, state);
+
+		Set<Action> filteredActions = derived.getFilteredActions();
+		actions = derived.getAvailableActions();
+
+		//Showing the grey dots for filtered actions if visualization is on:
+		if(visualizationOn || mode() == Modes.Spy) SutVisualization.visualizeFilteredActions(cv, state, filteredActions);
+
+		//return the set of derived actions
+		return actions;
+	}
+
+	/**
+	 * Select one of the available actions using an action selection algorithm (for example random action selection)
+	 *
+	 * super.selectAction(state, actions) updates information to the HTML sequence report
+	 *
+	 * @param state the SUT's current state
+	 * @param actions the set of derived actions
+	 * @return  the selected action (non-null!)
+	 */
+	@Override
+	protected Action selectAction(State state, Set<Action> actions){
+		return super.selectAction(state, actions);
+	}
+
+	/**
+	 * Execute the selected action.
+	 *
+	 * super.executeAction(system, state, action) is updating the HTML sequence report with selected action
+	 *
+	 * @param system the SUT
+	 * @param state the SUT's current state
+	 * @param action the action to execute
+	 * @return whether or not the execution succeeded
+	 */
+	@Override
+	protected boolean executeAction(SUT system, State state, Action action){
+		boolean executed = super.executeAction(system, state, action);
+		// After executing the SUT action, extract the action coverage
+		codeCoverageManager.getActionCoverage(String.valueOf(actionCount));
+		return executed;
+	}
+
+	/**
+	 * This method is invoked each time after TESTAR finished the generation of a sequence.
+	 */
+	@Override
+	protected void finishSequence() {
+		// Before finishing the sequence and closing the SUT, extract the sequence coverage
+		codeCoverageManager.getSequenceCoverage();
+		super.finishSequence();
+	}
+
+}