diff --git a/VERSION b/VERSION index b52e74001..61460b606 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6.23 \ No newline at end of file +2.6.24 \ No newline at end of file diff --git a/core/src/org/testar/monkey/alayer/actions/ActionRoles.java b/core/src/org/testar/monkey/alayer/actions/ActionRoles.java index c82e7e113..b451d364a 100644 --- a/core/src/org/testar/monkey/alayer/actions/ActionRoles.java +++ b/core/src/org/testar/monkey/alayer/actions/ActionRoles.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013 - 2020 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2018 - 2020 Open Universiteit - www.ou.nl +* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,6 +42,7 @@ private ActionRoles(){} public static final Role Action = Role.from("Action"), + NOPAction = Role.from("NOPAction", Action), MouseAction = Role.from("MouseAction", Action), KeyboardAction = Role.from("KeyboardAction", Action), CompoundAction = Role.from("CompoundAction", Action), diff --git a/core/src/org/testar/monkey/alayer/actions/NOP.java b/core/src/org/testar/monkey/alayer/actions/NOP.java index 0905e5bdb..a98ca3246 100644 --- a/core/src/org/testar/monkey/alayer/actions/NOP.java +++ b/core/src/org/testar/monkey/alayer/actions/NOP.java @@ -1,6 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013, 2014, 2015, 2016, 2017 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -27,10 +28,6 @@ * POSSIBILITY OF SUCH DAMAGE. *******************************************************************************************************/ - -/** - * @author Sebastian Bauersfeld - */ package org.testar.monkey.alayer.actions; import org.testar.monkey.Util; @@ -47,19 +44,22 @@ public class NOP extends TaggableBase implements Action { private static final long serialVersionUID = 8622084462407313716L; - public static final String NOP_ID = "No Operation"; // by urueda - + public static final String NOP_ID = "No Operation"; + + public NOP() { + this.set(Tags.Desc, NOP_ID); + this.set(Tags.Role, ActionRoles.NOPAction); + } + public void run(SUT system, State state, double duration){ Util.pause(duration); } public String toString(){ return NOP_ID; } - // by urueda @Override public String toString(Role... discardParameters) { return toString(); } - // by urueda @Override public String toShortString() { Role r = get(Tags.Role, null); @@ -69,7 +69,6 @@ public String toShortString() { return toString(); } - // by urueda @Override public String toParametersString() { return ""; diff --git a/core/src/org/testar/monkey/alayer/actions/StdActionCompiler.java b/core/src/org/testar/monkey/alayer/actions/StdActionCompiler.java index e7d04d4fb..3b33adfbf 100644 --- a/core/src/org/testar/monkey/alayer/actions/StdActionCompiler.java +++ b/core/src/org/testar/monkey/alayer/actions/StdActionCompiler.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2020 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2020 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,7 @@ import org.testar.monkey.alayer.Action; import org.testar.monkey.alayer.Finder; import org.testar.monkey.alayer.Position; +import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.StdAbstractor; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Widget; @@ -375,7 +376,13 @@ public Action hitShortcutKey(List keys) { builder.add(NOP, 1.0); return builder.build(); } - + + public Action noOperationalState(State state) { + Action ret = new NOP(); + ret.mapActionToWidget(state); + return ret; + } + public Action killProcessByPID(long pid){ return killProcessByPID(pid, 0); } public Action killProcessByName(String name){ return killProcessByName(name, 0); } public Action killProcessByPID(long pid, double timeToWaitBeforeKilling){ return KillProcess.byPID(pid, timeToWaitBeforeKilling); } diff --git a/testar/build.gradle b/testar/build.gradle index a4f66f9d3..89010c4dc 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -122,6 +122,12 @@ dependencies { implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.1' implementation group: 'com.orientechnologies', name: 'orientdb-graphdb', version: '3.0.34' implementation group: 'com.sikulix', name: 'sikulixapi', version: '2.0.5' + // jacoco and maven dependencies for jacoco coverage + implementation group: 'org.apache.maven', name: 'maven-core', version: '3.9.8' + implementation group: 'org.apache.maven.reporting', name: 'maven-reporting-api', version: '3.1.1' + implementation group: 'org.jacoco', name: 'jacoco-maven-plugin', version: '0.8.12' + // https://mvnrepository.com/artifact/org.apache.commons/commons-csv + implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.11.0' } task prepareSettings(type: Copy) { diff --git a/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java new file mode 100644 index 000000000..0310d1480 --- /dev/null +++ b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java @@ -0,0 +1,203 @@ +/*************************************************************************************************** + * + * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +import org.testar.DerivedActions; +import org.testar.SutVisualization; +import org.testar.coverage.CodeCoverageManager; +import org.testar.monkey.alayer.Action; +import org.testar.monkey.alayer.SUT; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.exceptions.ActionBuildException; +import org.testar.monkey.alayer.exceptions.StateBuildException; +import org.testar.monkey.alayer.exceptions.SystemStartException; +import org.testar.protocols.DesktopProtocol; +import org.testar.settings.Settings; + +import java.util.Set; + +public class Protocol_desktop_java_coverage extends DesktopProtocol { + + private CodeCoverageManager codeCoverageManager; + + /** + * Called once during the life time of TESTAR + * This method can be used to perform initial setup work + * @param settings the current TESTAR settings as specified by the user. + */ + @Override + protected void initialize(Settings settings){ + super.initialize(settings); + // Initialize the code coverage extractor using the test settings values + codeCoverageManager = new CodeCoverageManager(settings); + } + + + /** + * This method is called when TESTAR starts the System Under Test (SUT). The method should + * take care of + * 1) starting the SUT (you can use TESTAR's settings obtainable from settings() to find + * out what executable to run) + * 2) waiting until the system is fully loaded and ready to be tested (with large systems, you might have to wait several + * seconds until they have finished loading) + * @return a started SUT, ready to be tested. + */ + @Override + protected SUT startSystem() throws SystemStartException{ + return super.startSystem(); + } + + /** + * This method is invoked each time the TESTAR starts the SUT to generate a new sequence. + * This can be used for example for bypassing a login screen by filling the username and password + * or bringing the system into a specific start state which is identical on each start (e.g. one has to delete or restore + * the SUT's configuration files etc.) + */ + @Override + protected void beginSequence(SUT system, State state){ + // Before executing the first SUT action, extract the initial coverage + codeCoverageManager.getActionCoverage("0"); + super.beginSequence(system, state); + } + + /** + * This method is called when the TESTAR requests the state of the SUT. + * Here you can add additional information to the SUT's state or write your + * own state fetching routine. The state should have attached an oracle + * (TagName: Tags.OracleVerdict) which describes whether the + * state is erroneous and if so why. + * + * super.getState(system) puts the state information also to the HTML sequence report + * + * @return the current state of the SUT with attached oracle. + */ + @Override + protected State getState(SUT system) throws StateBuildException{ + return super.getState(system); + } + + /** + * The getVerdict methods implements the online state oracles that + * examine the SUT's current state and returns an oracle verdict. + * @return oracle verdict, which determines whether the state is erroneous and why. + */ + @Override + protected Verdict getVerdict(State state){ + // The super methods implements the implicit online state oracles for: + // system crashes + // non-responsiveness + // suspicious tags + Verdict verdict = super.getVerdict(state); + + //-------------------------------------------------------- + // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE + //-------------------------------------------------------- + + return verdict; + } + + /** + * This method is used by TESTAR to determine the set of currently available actions. + * You can use the SUT's current state, analyze the widgets and their properties to create + * a set of sensible actions, such as: "Click every Button which is enabled" etc. + * The return value is supposed to be non-null. If the returned set is empty, TESTAR + * will stop generation of the current action and continue with the next one. + * @param system the SUT + * @param state the SUT's current state + * @return a set of actions + */ + @Override + protected Set deriveActions(SUT system, State state) throws ActionBuildException{ + //The super method returns a ONLY actions for killing unwanted processes if needed, or bringing the SUT to + //the foreground. You should add all other actions here yourself. + // These "special" actions are prioritized over the normal GUI actions in selectAction() / preSelectAction(). + Set actions = super.deriveActions(system,state); + + // Derive left-click actions, click and type actions, and scroll actions from + + // all widgets of the GUI: + //DerivedActions derived = deriveClickTypeScrollActionsFromAllWidgets(actions, state); + + // the top level widgets of the GUI such as menu items: + DerivedActions derived = deriveClickTypeScrollActionsFromTopLevelWidgets(actions, state); + + Set filteredActions = derived.getFilteredActions(); + actions = derived.getAvailableActions(); + + //Showing the grey dots for filtered actions if visualization is on: + if(visualizationOn || mode() == Modes.Spy) SutVisualization.visualizeFilteredActions(cv, state, filteredActions); + + //return the set of derived actions + return actions; + } + + /** + * Select one of the available actions using an action selection algorithm (for example random action selection) + * + * super.selectAction(state, actions) updates information to the HTML sequence report + * + * @param state the SUT's current state + * @param actions the set of derived actions + * @return the selected action (non-null!) + */ + @Override + protected Action selectAction(State state, Set actions){ + return super.selectAction(state, actions); + } + + /** + * Execute the selected action. + * + * super.executeAction(system, state, action) is updating the HTML sequence report with selected action + * + * @param system the SUT + * @param state the SUT's current state + * @param action the action to execute + * @return whether or not the execution succeeded + */ + @Override + protected boolean executeAction(SUT system, State state, Action action){ + boolean executed = super.executeAction(system, state, action); + // After executing the SUT action, extract the action coverage + codeCoverageManager.getActionCoverage(String.valueOf(actionCount)); + return executed; + } + + /** + * This method is invoked each time after TESTAR finished the generation of a sequence. + */ + @Override + protected void finishSequence() { + // Before finishing the sequence and closing the SUT, extract the sequence coverage + codeCoverageManager.getSequenceCoverage(); + super.finishSequence(); + } + +} diff --git a/testar/resources/settings/desktop_java_coverage/test.settings b/testar/resources/settings/desktop_java_coverage/test.settings new file mode 100644 index 000000000..4b9287809 --- /dev/null +++ b/testar/resources/settings/desktop_java_coverage/test.settings @@ -0,0 +1,223 @@ +################################################################# +# TESTAR mode +# +# Set the mode you want TESTAR to start in: Spy, Generate, Replay +################################################################# + +Mode = Spy + +################################################################# +# Connect to the System Under Test (SUT) +# +# Indicate how you want to connect to the SUT: +# +# SUTCONNECTOR = COMMAND_LINE, SUTConnectorValue property must be a command line that +# starts the SUT. +# It should work from a Command Prompt terminal window (e.g. java - jar SUTs/calc.jar ). +# For web applications, follow the next format: web_browser_path SUT_URL. +# +# SUTCONNECTOR = SUT_WINDOW_TITLE, then SUTConnectorValue property must be the title displayed +# in the SUT main window. The SUT must be manually started and closed. +# +# SUTCONNECTOR = SUT_PROCESS_NAME: SUTConnectorValue property must be the process name of the SUT. +# The SUT must be manually started and closed. +################################################################# +SUTConnector = COMMAND_LINE +SUTConnectorValue = java -jar -Dcom.sun.management.jmxremote.port=5000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:../lib/org.jacoco.agent-0.8.12-runtime.jar=output=none,jmx=true "suts/PrintLogFile.jar" + +################################################################# +# Sequences +# +# Number of sequences and the length of these sequences +################################################################# + +Sequences = 2 +SequenceLength = 4 + +################################################################# +# Oracles based on suspicious tag values +# +# Regular expression and Tags to apply them +################################################################# + +SuspiciousTags = +TagsForSuspiciousOracle = Title + +################################################################# +# Oracles based on Suspicious Outputs detected by Process Listeners +# +# Requires ProcessListenerEnabled +# (Only available for desktop applications through COMMAND_LINE) +# +# Regular expression SuspiciousProcessOutput contains the specification +# of what is considered to be suspicious output. +################################################################# + +ProcessListenerEnabled = false +SuspiciousProcessOutput = + +################################################################# +# Process Logs +# +# Requires ProcessListenerEnabled +# (Only available for desktop applications through COMMAND_LINE) +# +# Allow TESTAR to store execution logs coming from the processes. +# You can use the regular expression ProcessLogs below to filter +# the logs. Use .*.* if you want to store all the outputs of the +# process. +################################################################# + +ProcessLogs = .*.* + +################################################################# +# Oracles based on suspicious titles in SUT output files +################################################################# + +LogOracleEnabled = false +LogOracleRegex = .*([Ee]xception|[Ee]rror).* +LogOracleCommands = +LogOracleFiles = test_print_exception.log;../../testar/bin/test_print_error.log + +################################################################# +# Actionfilter +# +# Regular expression and Tags to apply them. +# More filters can be added in Spy mode, +# these will be added to the protocol_filter.xml file. +################################################################# + +ClickFilter = .*[sS]istema.*|.*[sS]ystem.*|.*[cC]errar.*|.*[cC]lose.*|.*[sS]alir.*|.*[eE]xit.*|.*[mM]inimizar.*|.*[mM]inimi[zs]e.*|.*[mM]aximizar.*|.*[mM]aximi[zs]e.* +TagsToFilter = Title + +################################################################# +# Processfilter +# +# Regular expression. Kill the processes that your SUT can start up +# but that you do not want to test. +################################################################# + +ProcessesToKillDuringTest = + +################################################################# +# SUT Protocol +# +# ProtocolClass: Indicate the location of the protocol class for your specific SUT +# AlwaysCompile: Compile the protocol before launching the selected TESTAR mode +################################################################# + +ProtocolClass = desktop_java_coverage/Protocol_desktop_java_coverage +AlwaysCompile = true + +################################################################# +# Time configurations +# +# ActionDuration: Sets the speed, in seconds, at which a GUI action is performed +# TimeToWaitAfterAction: Sets the delay, in seconds, between UI actions during a test +# StartupTime: Sets how many seconds to wait for the SUT to be ready for testing +# MaxTime: Sets a time, in seconds, after which the test run is finished (e.g. stop after an hour) +################################################################# + +ActionDuration = 0.0 +TimeToWaitAfterAction = 0.3 +StartupTime = 10.0 +MaxTime = 3.1536E7 + +################################################################# +# State model inference settings +# +# StateModelEnabled: Enable or disable the State Model feature +# DataStore: The graph database we use to store the State Model: OrientDB +# DataStoreType: The mode we use to connect to the database: remote or plocal +# DataStoreServer: IP address to connect if we desired to use remote mode +# DataStoreDirectory: Path of the directory on which local OrientDB exists, if we use plocal mode +# DataStoreDB: The name of the desired database on which we want to store the State Model. +# DataStoreUser: User credential to authenticate TESTAR in OrientDB +# DataStorePassword: Password credential to authenticate TESTAR in OrientDB +# DataStoreMode: Indicate how TESTAR should store the model objects in the database: instant, delayed, hybrid, none +# ApplicationName: Name to identify the SUT. Especially important to identify a State Model +# ApplicationVersion: Version to identify the SUT. Especially important to identify a State Model +# ActionSelectionAlgorithm: State Model Action Selection mechanism to explore the SUT: random or unvisited +# StateModelStoreWidgets: Save all widget tree information in the State Model every time TESTAR discovers a new Concrete State +# ResetDataStore: WARNING: Delete all existing State Models from the selected database before creating a new one +################################################################# + +StateModelEnabled = false +DataStore = +DataStoreType = remote +DataStoreServer = +DataStoreDirectory = +DataStoreDB = +DataStoreUser = +DataStorePassword = +DataStoreMode = none +ApplicationName = printerLogger +ApplicationVersion = +ActionSelectionAlgorithm = random +StateModelStoreWidgets = true +ResetDataStore = false + +################################################################# +# State identifier attributes +# +# Specify the widget attributes that you wish to use in constructing +# the widget and state hash strings. Use a comma separated list. +################################################################# +AbstractStateAttributes = WidgetControlType,WidgetTitle + +################################################################# +# Settings (string) that can be used for user specified protocols +################################################################# +ProtocolSpecificSetting_1 = +ProtocolSpecificSetting_2 = +ProtocolSpecificSetting_3 = +ProtocolSpecificSetting_4 = +ProtocolSpecificSetting_5 = + +################################################################# +# Sets whether to display TESTAR dialog. If false is used, then TESTAR will run in the mode of the Mode property +################################################################# + +ShowVisualSettingsDialogOnStartup = true + +################################################################# +# Sets whether to keep the SUTs GUI active in the screen (e.g. when its minimised or when a process is started and its UI is in front, etc.) +################################################################# + +ForceForeground = true + +################################################################# +# Sets whether to display overlay information, inside the SPY mode, for all the UI actions derived from the test set up +################################################################# + +VisualizeActions = false + +################################################################# +# Settings to configure code coverage features +################################################################# + +JacocoCoverage = true +JacocoCoverageIpAddress = localhost +JacocoCoveragePort = 5000 +JacocoCoverageClasses = suts/PrintLogFile +JacocoCoverageAccumulate = false + +################################################################# +# Other more advanced settings +################################################################# + +OutputDir = ./output +PathToReplaySequence = ./output/temp +TempDir = ./output/temp +UseRecordedActionDurationAndWaitTimeDuringReplay = false +LogLevel = 1 +FaultThreshold = 0.000000001 +MyClassPath = ./settings +OnlySaveFaultySequences = false +ReplayRetryTime = 30.0 +Delete = +TimeToFreeze = 30.0 +StopGenerationOnFault = true +CopyFromTo = +MaxReward = 9999999 +Discount = .95 diff --git a/testar/resources/settings/desktop_swingset2/Protocol_desktop_SwingSet2.java b/testar/resources/settings/desktop_swingset2/Protocol_desktop_SwingSet2.java index d38014e32..3ea6df5f7 100644 --- a/testar/resources/settings/desktop_swingset2/Protocol_desktop_SwingSet2.java +++ b/testar/resources/settings/desktop_swingset2/Protocol_desktop_SwingSet2.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2017 - 2021 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2019 - 2021 Open Universiteit - www.ou.nl + * Copyright (c) 2017 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2019 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -43,7 +43,7 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -//TODO Fernando: create a higher level SwingProtocol and document this one after that +//TODO: create a higher level SwingProtocol and document this one after that /** * Protocol specifically for testing Java Swing applications. */ diff --git a/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/Protocol_test_gradle_workflow_desktop_java_swing.java b/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/Protocol_test_gradle_workflow_desktop_java_swing.java index fde85c7c6..c149c6b82 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/Protocol_test_gradle_workflow_desktop_java_swing.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/Protocol_test_gradle_workflow_desktop_java_swing.java @@ -32,6 +32,8 @@ import static org.testar.monkey.alayer.Tags.Enabled; import java.io.File; +import java.io.FileNotFoundException; +import java.util.Scanner; import java.util.Set; import org.apache.commons.io.FileUtils; @@ -40,6 +42,7 @@ import org.testar.monkey.Main; import org.testar.monkey.Util; import org.testar.OutputStructure; +import org.testar.coverage.CodeCoverageManager; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; @@ -48,6 +51,7 @@ import org.testar.monkey.alayer.exceptions.StateBuildException; import org.testar.monkey.alayer.exceptions.SystemStartException; import org.testar.protocols.DesktopProtocol; +import org.testar.settings.Settings; /** * This protocol is used to test TESTAR by executing a gradle CI workflow. @@ -56,6 +60,15 @@ */ public class Protocol_test_gradle_workflow_desktop_java_swing extends DesktopProtocol { + private CodeCoverageManager codeCoverageManager; + + @Override + protected void initialize(Settings settings){ + super.initialize(settings); + // Initialize the code coverage extractor using the test settings values + codeCoverageManager = new CodeCoverageManager(settings); + } + @Override protected SUT startSystem() throws SystemStartException { SUT system = super.startSystem(); @@ -67,6 +80,9 @@ protected SUT startSystem() throws SystemStartException { @Override protected void beginSequence(SUT system, State state){ + // Before executing the first SUT action, extract the initial coverage + codeCoverageManager.getActionCoverage("0"); + super.beginSequence(system, state); // Verify that Themes Java Swing widget is found in the GUI @@ -163,8 +179,19 @@ public void widgetTree(Widget w, Set actions) { } } + @Override + protected boolean executeAction(SUT system, State state, Action action){ + boolean executed = super.executeAction(system, state, action); + // After executing the SUT action, extract the action coverage + codeCoverageManager.getActionCoverage(String.valueOf(actionCount)); + return executed; + } + @Override protected void finishSequence(){ + // Before finishing the sequence and closing the SUT, extract the sequence coverage + codeCoverageManager.getSequenceCoverage(); + // Don't use the call SystemProcessHandling.killTestLaunchedProcesses before stopSystem // Invoking a jar app considers the java.exe a test launched process instead the main one @@ -194,10 +221,53 @@ protected void stopSystem(SUT system) { @Override protected void closeTestSession() { super.closeTestSession(); + + // Verify the jacoco coverage folder has been created + String jacocoDirectory = OutputStructure.outerLoopOutputDir + + File.separator + "coverage" + + File.separator + "jacoco"; + Assert.isTrue(new File(jacocoDirectory).exists()); + + // Verify the jacoco coverage files have been created + String jacocoFiles = jacocoDirectory + File.separator + + OutputStructure.startOuterLoopDateString + "_" + + OutputStructure.executedSUTname; + + Assert.isTrue(new File(jacocoFiles + "_accumulative_ratio_coverage.txt").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1.exec").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1.csv").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_0.exec").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_0.csv").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_1.exec").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_1.csv").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_1_merged.exec").exists()); + Assert.isTrue(new File(jacocoFiles + "_sequence_1_action_1_merged.csv").exists()); + + Assert.isTrue(fileContains("FileChooserDemo", new File(jacocoFiles + "_sequence_1.csv"))); + Assert.isTrue(fileContains("SwingSet2", new File(jacocoFiles + "_sequence_1.csv"))); + Assert.isTrue(fileContains("ProgressBarDemo", new File(jacocoFiles + "_sequence_1.csv"))); + try { File originalFolder = new File(OutputStructure.outerLoopOutputDir).getCanonicalFile(); File artifactFolder = new File(Main.testarDir + settings.get(ConfigTags.ApplicationName,"")); FileUtils.copyDirectory(originalFolder, artifactFolder); } catch(Exception e) {System.out.println("ERROR: Creating Artifact Folder");} } + + private boolean fileContains(String searchText, File file) { + try (Scanner scanner = new Scanner(file)) { + // Read the content of the file line by line + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + + // Check if the line contains the specific text + if (line.contains(searchText)) { + return true; + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return false; + } } diff --git a/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/test.settings b/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/test.settings index 521a09c6c..f6aa70828 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/test.settings +++ b/testar/resources/workflow/settings/test_gradle_workflow_desktop_java_swing/test.settings @@ -23,7 +23,7 @@ Mode = Spy # The SUT must be manually started and closed. ################################################################# SUTConnector = COMMAND_LINE -SUTConnectorValue = java -jar "suts/SwingSet2.jar" +SUTConnectorValue = java -jar -Dcom.sun.management.jmxremote.port=5000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:../lib/org.jacoco.agent-0.8.12-runtime.jar=output=none,jmx=true "suts/SwingSet2.jar" ################################################################# # Java Swing applications & Access Bridge Enabled @@ -117,6 +117,16 @@ ProtocolClass = test_gradle_workflow_desktop_java_swing/Protocol_test_gradle_wor ################################################################# AbstractStateAttributes = WidgetControlType +################################################################# +# Settings to configure code coverage features +################################################################# + +JacocoCoverage = true +JacocoCoverageIpAddress = localhost +JacocoCoveragePort = 5000 +JacocoCoverageClasses = suts/SwingSet2 +JacocoCoverageAccumulate = true + ################################################################# # Other more advanced settings ################################################################# diff --git a/testar/src/org/testar/coverage/CodeCoverage.java b/testar/src/org/testar/coverage/CodeCoverage.java new file mode 100644 index 000000000..5cca5ddec --- /dev/null +++ b/testar/src/org/testar/coverage/CodeCoverage.java @@ -0,0 +1,38 @@ +/*************************************************************************************************** + * + * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.coverage; + +public interface CodeCoverage { + + public void getSequenceCoverage(); + public void getActionCoverage(String actionCount); + +} diff --git a/testar/src/org/testar/coverage/CodeCoverageManager.java b/testar/src/org/testar/coverage/CodeCoverageManager.java new file mode 100644 index 000000000..1ebfecc29 --- /dev/null +++ b/testar/src/org/testar/coverage/CodeCoverageManager.java @@ -0,0 +1,71 @@ +/*************************************************************************************************** + * + * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.coverage; + +import java.io.File; +import java.util.ArrayList; + +import org.testar.OutputStructure; +import org.testar.coverage.jacoco.JacocoCoverage; +import org.testar.monkey.ConfigTags; +import org.testar.settings.Settings; + +public class CodeCoverageManager implements CodeCoverage { + + private ArrayList coverageExtractors; + + public CodeCoverageManager(Settings settings) { + // Create a file directory to store the coverage file results + String outputCoveragePath = OutputStructure.outerLoopOutputDir + File.separator + "coverage"; + File outputCoverageDir = new File(outputCoveragePath); + if(!outputCoverageDir.exists()) + outputCoverageDir.mkdirs(); + + coverageExtractors = new ArrayList<>(); + + if(settings.get(ConfigTags.JacocoCoverage, false)) { + coverageExtractors.add(new JacocoCoverage(settings, outputCoveragePath)); + } + } + + @Override + public void getSequenceCoverage() { + for(CodeCoverage coverageExtractor : coverageExtractors) + coverageExtractor.getSequenceCoverage(); + } + + @Override + public void getActionCoverage(String actionCount) { + for(CodeCoverage coverageExtractor : coverageExtractors) + coverageExtractor.getActionCoverage(actionCount); + } + +} diff --git a/testar/src/org/testar/coverage/jacoco/JacocoCoverage.java b/testar/src/org/testar/coverage/jacoco/JacocoCoverage.java new file mode 100644 index 000000000..a5e183ef5 --- /dev/null +++ b/testar/src/org/testar/coverage/jacoco/JacocoCoverage.java @@ -0,0 +1,99 @@ +/*************************************************************************************************** + * + * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.coverage.jacoco; + +import java.io.File; + +import org.testar.OutputStructure; +import org.testar.coverage.CodeCoverage; +import org.testar.monkey.ConfigTags; +import org.testar.settings.Settings; + +public class JacocoCoverage implements CodeCoverage { + + private String outputJacocoCoveragePath; + private MBeanClient mbeanClient; + private JacocoReportCSV jacocoReportCSV; + private MergeJacocoFiles mergeJacocoFiles; + + public JacocoCoverage(Settings settings, String outputCoveragePath) { + // Create a file directory to store the jacoco coverage file results + outputJacocoCoveragePath = outputCoveragePath + File.separator + "jacoco"; + File outputJacocoCoverageDir = new File(outputJacocoCoveragePath); + if(!outputJacocoCoverageDir.exists()) + outputJacocoCoverageDir.mkdirs(); + + // Initialize the MBeanClient that connects to the Jacoco agent port to extract coverage + mbeanClient = new MBeanClient(settings.get(ConfigTags.JacocoCoverageIpAddress), settings.get(ConfigTags.JacocoCoveragePort)); + + // Initialize the CSV reporter + jacocoReportCSV = new JacocoReportCSV(settings); + + // Initialize MergeJacocoFiles if user wants to track accumulative Jacoco coverage + if(settings.get(ConfigTags.JacocoCoverageAccumulate, false)) { + mergeJacocoFiles = new MergeJacocoFiles(); + } + } + + @Override + public void getSequenceCoverage() { + // Create the default SUT + sequence name + sequence number to extract the jacoco coverage exec file + String destJacocoSequenceFileName = outputJacocoCoveragePath + File.separator + + OutputStructure.startOuterLoopDateString + "_" + OutputStructure.executedSUTname + + "_sequence_" + OutputStructure.sequenceInnerLoopCount; + + String jacocoExecSequenceCoverage = mbeanClient.dumpJacocoReport(destJacocoSequenceFileName + ".exec"); + + // Once the exec file is created, prepare the CSV results + jacocoReportCSV.generateCSVresults(jacocoExecSequenceCoverage, destJacocoSequenceFileName + ".csv"); + } + + @Override + public void getActionCoverage(String actionCount) { + // Create the default SUT + sequence name + sequence number + action number to extract the jacoco coverage exec file + String destJacocoActionFileName = outputJacocoCoveragePath + File.separator + + OutputStructure.startOuterLoopDateString + "_" + OutputStructure.executedSUTname + + "_sequence_" + OutputStructure.sequenceInnerLoopCount + + "_action_" + actionCount; + + String jacocoExecActionCoverage = mbeanClient.dumpJacocoReport(destJacocoActionFileName + ".exec"); + + // Once the exec file is created, prepare the CSV results + jacocoReportCSV.generateCSVresults(jacocoExecActionCoverage, destJacocoActionFileName + ".csv"); + + // Compute the accumulative Jacoco coverage if enabled by user + if(mergeJacocoFiles != null) { + mergeJacocoFiles.testarExecuteMojo(jacocoReportCSV, jacocoExecActionCoverage); + // Write an additional accumulative ratio coverage file + jacocoReportCSV.writeAccumulativeCoverage(outputJacocoCoveragePath, actionCount); + } + } +} diff --git a/testar/src/org/testar/coverage/jacoco/JacocoReportCSV.java b/testar/src/org/testar/coverage/jacoco/JacocoReportCSV.java new file mode 100644 index 000000000..665d819fb --- /dev/null +++ b/testar/src/org/testar/coverage/jacoco/JacocoReportCSV.java @@ -0,0 +1,244 @@ +/*************************************************************************************************** + * + * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.coverage.jacoco; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jacoco.core.analysis.Analyzer; +import org.jacoco.core.analysis.CoverageBuilder; +import org.jacoco.core.analysis.IClassCoverage; +import org.jacoco.core.tools.ExecFileLoader; +import org.testar.OutputStructure; +import org.testar.monkey.ConfigTags; +import org.testar.settings.Settings; + +public class JacocoReportCSV { + private static final Logger logger = LogManager.getLogger(); + + private String jacocoClassesPath; + private StringBuilder coverageRatioResults; + + public JacocoReportCSV(Settings settings) { + jacocoClassesPath = settings.get(ConfigTags.JacocoCoverageClasses); + coverageRatioResults = new StringBuilder(); + if(!new File(jacocoClassesPath).exists()) { + logger.error("JacocoCoverageClasses path does not exist: " + jacocoClassesPath); + System.err.println("JacocoCoverageClasses path does not exist: " + jacocoClassesPath); + } + } + + public void generateCSVresults(String jacocoExecFile, String outputCSV) { + try { + CoverageBuilder coverageBuilder = loadJacocoAnalysis(jacocoExecFile); + + // Initialize total counters + int totalInstructionMissed = 0; + int totalInstructionCovered = 0; + int totalBranchMissed = 0; + int totalBranchCovered = 0; + int totalLineMissed = 0; + int totalLineCovered = 0; + int totalComplexityMissed = 0; + int totalComplexityCovered = 0; + int totalMethodMissed = 0; + int totalMethodCovered = 0; + + // Write to CSV + try (CSVPrinter printer = new CSVPrinter(new FileWriter(outputCSV), CSVFormat.DEFAULT)) { + printer.printRecord("CLASS", + "INSTRUCTION_MISSED", "INSTRUCTION_COVERED", "INSTRUCTION_COVERED_RATIO", + "BRANCH_MISSED", "BRANCH_COVERED", "BRANCH_COVERED_RATIO", + "LINE_MISSED", "LINE_COVERED", "LINE_COVERED_RATIO", + "COMPLEXITY_MISSED", "COMPLEXITY_COVERED", "COMPLEXITY_COVERED_RATIO", + "METHOD_MISSED", "METHOD_COVERED", "METHOD_COVERED_RATIO"); + for (IClassCoverage cc : coverageBuilder.getClasses()) { + printer.printRecord( + cc.getName(), // Class name + + cc.getInstructionCounter().getMissedCount(), // INSTRUCTION_MISSED + cc.getInstructionCounter().getCoveredCount(), // INSTRUCTION_COVERED + formatPercentage(cc.getInstructionCounter().getCoveredRatio()), // INSTRUCTION_COVERED_RATIO + + cc.getBranchCounter().getMissedCount(), // BRANCH_MISSED + cc.getBranchCounter().getCoveredCount(), // BRANCH_COVERED + formatPercentage(cc.getBranchCounter().getCoveredRatio()), // BRANCH_COVERED_RATIO + + cc.getLineCounter().getMissedCount(), // LINE_MISSED + cc.getLineCounter().getCoveredCount(), // LINE_COVERED + formatPercentage(cc.getLineCounter().getCoveredRatio()), // LINE_COVERED_RATIO + + cc.getComplexityCounter().getMissedCount(), // COMPLEXITY_MISSED + cc.getComplexityCounter().getCoveredCount(), // COMPLEXITY_COVERED + formatPercentage(cc.getComplexityCounter().getCoveredRatio()), // COMPLEXITY_COVERED_RATIO + + cc.getMethodCounter().getMissedCount(), // METHOD_MISSED + cc.getMethodCounter().getCoveredCount(), // METHOD_COVERED + formatPercentage(cc.getMethodCounter().getCoveredRatio()) // METHOD_COVERED_RATIO + ); + + // Sum up totals + totalInstructionMissed += cc.getInstructionCounter().getMissedCount(); + totalInstructionCovered += cc.getInstructionCounter().getCoveredCount(); + totalBranchMissed += cc.getBranchCounter().getMissedCount(); + totalBranchCovered += cc.getBranchCounter().getCoveredCount(); + totalLineMissed += cc.getLineCounter().getMissedCount(); + totalLineCovered += cc.getLineCounter().getCoveredCount(); + totalComplexityMissed += cc.getComplexityCounter().getMissedCount(); + totalComplexityCovered += cc.getComplexityCounter().getCoveredCount(); + totalMethodMissed += cc.getMethodCounter().getMissedCount(); + totalMethodCovered += cc.getMethodCounter().getCoveredCount(); + } + + String totalInstructionRatio = totalRatio((double)totalInstructionCovered, (double)totalInstructionMissed); + String totalBranchRatio = totalRatio((double)totalBranchCovered, (double)totalBranchMissed); + String totalLineRatio = totalRatio((double)totalLineCovered, (double)totalLineMissed); + String totalComplexityRatio = totalRatio((double)totalComplexityCovered, (double)totalComplexityMissed); + String totalMethodRatio = totalRatio((double)totalMethodCovered, (double)totalMethodMissed); + + // Print CSV totals + printer.printRecord("TOTAL", + totalInstructionMissed, totalInstructionCovered, totalInstructionRatio, + totalBranchMissed, totalBranchCovered, totalBranchRatio, + totalLineMissed, totalLineCovered, totalLineRatio, + totalComplexityMissed, totalComplexityCovered, totalComplexityRatio, + totalMethodMissed, totalMethodCovered, totalMethodRatio); + + coverageRatioResults = new StringBuilder(); + + coverageRatioResults + .append("InstructionCoverage").append(" | ").append(totalInstructionRatio).append(" | ") + .append("BranchCoverage").append(" | ").append(totalBranchRatio).append(" | ") + .append("LineCoverage").append(" | ").append(totalLineRatio).append(" | ") + .append("ComplexityCoverage").append(" | ").append(totalComplexityRatio).append(" | ") + .append("MethodCoverage").append(" | ").append(totalMethodRatio); + } + + logger.trace("JacocoReportCSV generated the CSV report: " + outputCSV); + + } catch (IOException e) { + logger.error("JacocoReportCSV was not able to create the CSV report " + jacocoClassesPath + " with the exec file " + jacocoExecFile); + e.printStackTrace(); + } + } + + public void writeAccumulativeCoverage(String outputJacocoCoveragePath, String actionCount) { + String accumulativeCoverageFile = outputJacocoCoveragePath + File.separator + + OutputStructure.startOuterLoopDateString + "_" + OutputStructure.executedSUTname + + "_accumulative_ratio_coverage.txt"; + + try (FileWriter myWriter = new FileWriter(accumulativeCoverageFile, true)) { + myWriter.write("Sequence | " + OutputStructure.sequenceInnerLoopCount + + " | Action | " + actionCount + " | " + + coverageRatioResults.toString() + + System.lineSeparator()); + } catch (IOException e) { + logger.error("An error occurred while writing to the accumulative coverage file: " + e.getMessage()); + System.err.println("An error occurred while writing to the accumulative coverage file: " + e.getMessage()); + } + } + + CoverageBuilder loadJacocoAnalysis(String jacocoExecFile) throws IOException { + // Load the jacoco.exec file + ExecFileLoader loader = new ExecFileLoader(); + loader.load(new File(jacocoExecFile)); + + // Analyze the coverage data + CoverageBuilder coverageBuilder = new CoverageBuilder(); + Analyzer analyzer = new Analyzer(loader.getExecutionDataStore(), coverageBuilder); + + // Analyze all classes + analyzeAllClasses(jacocoClassesPath, analyzer); + + return coverageBuilder; + } + + private void analyzeAllClasses(String classesDirPath, Analyzer analyzer) { + File classesDir = new File(classesDirPath); + if (classesDir.isDirectory()) { + File[] files = classesDir.listFiles(); + if (files != null) { + for (File file : files) { + try { + if (file.isDirectory()) { + analyzeAllClasses(file.getPath(), analyzer); + } else if (file.getName().endsWith(".class") || file.getName().endsWith(".jar")) { + if (file.getName().endsWith(".jar")) { + try (ZipFile zipFile = new ZipFile(file)) { + zipFile.stream() + .filter(entry -> entry.getName().endsWith(".class")) + .forEach(entry -> analyzeZipEntry(zipFile, entry, analyzer)); + } + } else { + analyzer.analyzeAll(file); + } + } + } catch (Exception e) { + logger.error("Error analyzing file " + file.getPath() + ": " + e.getMessage()); + } + } + } + } + } + + private void analyzeZipEntry(ZipFile zipFile, ZipEntry entry, Analyzer analyzer) { + try { + analyzer.analyzeClass(zipFile.getInputStream(entry), entry.getName()); + } catch (IOException e) { + logger.error("Error analyzing zip entry " + entry.getName() + ": " + e.getMessage()); + } + } + + private String totalRatio(double covered, double missed) { + if (covered == 0.0 && missed == 0.0) { + return "0.00"; + } + return formatPercentage(covered / (covered + missed)); + } + + private String formatPercentage(double value) { + DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(Locale.getDefault()); + symbols.setDecimalSeparator('.'); + DecimalFormat df = new DecimalFormat("0.00", symbols); + return df.format(value * 100.0); + } + +} diff --git a/testar/src/org/testar/coverage/jacoco/MBeanClient.java b/testar/src/org/testar/coverage/jacoco/MBeanClient.java new file mode 100644 index 000000000..a8f304143 --- /dev/null +++ b/testar/src/org/testar/coverage/jacoco/MBeanClient.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * Copyright (c) 2009, 2020 Mountainminds GmbH & Co. KG and Contributors + * This program and the accompanying materials are made available under + * the terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Evgeny Mandrikov - initial API and implementation + * Fernando Pastor Ricos - adapt original code for TESTAR purposes + * + *******************************************************************************/ + +package org.testar.coverage.jacoco; + +import java.io.FileOutputStream; + +import javax.management.MBeanServerConnection; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * https://github.com/jacoco/jacoco/blob/master/org.jacoco.examples/src/org/jacoco/examples/MBeanClient.java + * + * This example connects to a JaCoCo agent that runs with the option + * jmx=yes and requests execution data. The collected data is + * dumped to a local file. + */ +public final class MBeanClient { + private static final Logger logger = LogManager.getLogger(); + + private String service_url; + + public MBeanClient(String service_ip_address, int service_port) { + // Initialize the JMX service to connect with the Jacoco Agent + service_url = "service:jmx:rmi:///jndi/rmi://" + service_ip_address + ":" + service_port + "/jmxrmi"; + } + + public interface IProxy { + String getVersion(); + + String getSessionId(); + + void setSessionId(String id); + + byte[] getExecutionData(boolean reset); + + void dump(boolean reset); + + void reset(); + } + + /** + * Use JMX service to connect with the JVM and extract one JaCoCo report file. + * IF success return the string that represents the path of this jacoco.exec file, + * ELSE return empty string. + * + * @param destFile + * @throws Exception + * @return string that contains extracted jacoco.exec file + */ + public String dumpJacocoReport(String destJacocoFileName) { + try { + // Open connection to the coverage agent: + final JMXServiceURL url = new JMXServiceURL(service_url); + final JMXConnector jmxc = JMXConnectorFactory.connect(url, null); + final MBeanServerConnection connection = jmxc.getMBeanServerConnection(); + + final IProxy proxy = (IProxy) MBeanServerInvocationHandler + .newProxyInstance(connection, new ObjectName("org.jacoco:type=Runtime"), IProxy.class, false); + + // Retrieve dump and write to file: + final byte[] data = proxy.getExecutionData(false); + final FileOutputStream localFile = new FileOutputStream(destJacocoFileName); + localFile.write(data); + localFile.close(); + + logger.trace("MBeanClient extracted a jacoco report exec file: " + destJacocoFileName); + + // Close connection: + jmxc.close(); + + } catch(Exception e) { + logger.error("MBeanClient was not able to dump the jacoco exec file " + destJacocoFileName); + // return an empty string to indicate we didn't create any jacoco.exec report + return ""; + } + + return destJacocoFileName; + } +} diff --git a/testar/src/org/testar/coverage/jacoco/MergeJacocoFiles.java b/testar/src/org/testar/coverage/jacoco/MergeJacocoFiles.java new file mode 100644 index 000000000..79ce47eac --- /dev/null +++ b/testar/src/org/testar/coverage/jacoco/MergeJacocoFiles.java @@ -0,0 +1,135 @@ +/******************************************************************************* + * Copyright (c) 2009, 2020 Mountainminds GmbH & Co. KG and Contributors + * This program and the accompanying materials are made available under + * the terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Mads Mohr Christensen - implementation of MergeMojo + * Fernando Pastor Ricos - adapt original code for TESTAR purposes + * + *******************************************************************************/ + +package org.testar.coverage.jacoco; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.jacoco.core.tools.ExecFileLoader; +import org.jacoco.maven.AbstractJacocoMojo; + +/** + * Mojo for merging a set of execution data files (*.exec) into a single file. + * Add TESTAR testarExecuteMojo method for tool coverage purposes. + * + * @since 0.6.4 + */ +public class MergeJacocoFiles extends AbstractJacocoMojo { + private static final Logger logger = LogManager.getLogger(); + + /** + * Path to the output file for execution data. + */ + private File destMergedFile; + + /** + * Deque to track exactly two exec jacocoFiles. + */ + private Deque jacocoFiles = new ArrayDeque<>(2); + + /** + * Indicate the list of existing jacoco.exec files and the desired output merged file + * + * @param jacocoFiles + * @param destFile + */ + public void testarExecuteMojo(JacocoReportCSV jacocoReportCSV, String jacocoFile) { + // Add the new file to the deque and manage size + addJacocoFile(jacocoFile); + + if (!canMergeReports()) { + return; + } + + // Merge the exec files and prepare the CSV results + String jacocoExecMerged = jacocoFile.replace(".exec", "_merged.exec"); + String jacocoCsvMerged = jacocoFile.replace(".exec", "_merged.csv"); + this.destMergedFile = new File(jacocoExecMerged); + executeMerge(); + jacocoReportCSV.generateCSVresults(jacocoExecMerged, jacocoCsvMerged); + + // Finally, add the merged file to the deque and manage size + // This is because the last merged file is the one that tracks the accumulative coverage + addJacocoFile(jacocoExecMerged); + } + + /** + * Adds a new jacoco file to the deque, replacing the oldest if it already contains two elements. + * @param newJacocoFile + */ + private void addJacocoFile(String newJacocoFile) { + // If deque already contains 2 elements, remove the oldest (from the front) + if (jacocoFiles.size() == 2) { + jacocoFiles.removeFirst(); + } + + // Add the new file to the back of the deque + jacocoFiles.addLast(newJacocoFile); + + logger.trace("Added Jacoco file: " + newJacocoFile + ", updated files: " + jacocoFiles); + } + + private boolean canMergeReports() { + if (jacocoFiles.size() != 2) { + logger.trace("Jacoco merge is not possible, we need two jacoco files"); + return false; + } + return true; + } + + private void executeMerge() { + final ExecFileLoader loader = new ExecFileLoader(); + + load(loader); + save(loader); + } + + private void load(final ExecFileLoader loader) { + for(final String file : jacocoFiles) { + final File inputFile = new File(file); + try { + logger.trace("Loading execution data file " + inputFile.getAbsolutePath()); + loader.load(inputFile); + } catch (final IOException e) { + logger.error("Unable to read " + inputFile.getAbsolutePath()); + } + } + } + + private void save(final ExecFileLoader loader) { + if (loader.getExecutionDataStore().getContents().isEmpty()) { + logger.error("MergeJacocoFiles save : getExecutionDataStore().getContents().isEmpty()"); + return; + } + logger.trace("Writing merged execution data to " + destMergedFile.getAbsolutePath()); + try { + loader.save(destMergedFile, false); + } catch (final IOException e) { + logger.error("Unable to write merged file " + destMergedFile.getAbsolutePath()); + } + } + + @Override + protected void executeMojo() throws MojoExecutionException, MojoFailureException { + // Nothing, customized behavior is in testarExecuteMojo() + } + +} \ No newline at end of file diff --git a/testar/src/org/testar/monkey/ConfigTags.java b/testar/src/org/testar/monkey/ConfigTags.java index ae7b0c8c4..eb3c4c041 100644 --- a/testar/src/org/testar/monkey/ConfigTags.java +++ b/testar/src/org/testar/monkey/ConfigTags.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -209,6 +209,25 @@ private ConfigTags() {} public static final Tag ProtocolSpecificSetting_5 = Tag.from("ProtocolSpecificSetting_5", String.class, "Settings (string) that can be used for user specified protocols"); + /** + * Coverage settings + */ + + public static final Tag JacocoCoverage = Tag.from("JacocoCoverage", Boolean.class, + "Sets whether to extract Jacoco Coverage"); + + public static final Tag JacocoCoverageIpAddress = Tag.from("JacocoCoverageIpAddress", String.class, + "The JMX IP Address on which the Jacoco Agent is running"); + + public static final Tag JacocoCoveragePort = Tag.from("JacocoCoveragePort", Integer.class, + "The JMX port on which the Jacoco Agent is running"); + + public static final Tag JacocoCoverageClasses = Tag.from("JacocoCoverageClasses", String.class, + "The SUT class files that Jacoco uses to create the CSV report"); + + public static final Tag JacocoCoverageAccumulate = Tag.from("JacocoCoverageAccumulate", Boolean.class, + "Sets whether Jacoco coverage will be accumulated across the run sequences"); + /** * Additional settings with descriptions */ diff --git a/testar/src/org/testar/monkey/Main.java b/testar/src/org/testar/monkey/Main.java index 472150d77..79f061cdb 100644 --- a/testar/src/org/testar/monkey/Main.java +++ b/testar/src/org/testar/monkey/Main.java @@ -58,7 +58,7 @@ public class Main { - public static final String TESTAR_VERSION = "2.6.23 (25-Oct-2024)"; + public static final String TESTAR_VERSION = "2.6.24 (29-Oct-2024)"; //public static final String TESTAR_DIR_PROPERTY = "DIRNAME"; //Use the OS environment to obtain TESTAR directory public static final String SETTINGS_FILE = "test.settings"; diff --git a/testar/src/org/testar/reporting/HtmlReporter.java b/testar/src/org/testar/reporting/HtmlReporter.java index b76d5ea35..b2fd8b36c 100644 --- a/testar/src/org/testar/reporting/HtmlReporter.java +++ b/testar/src/org/testar/reporting/HtmlReporter.java @@ -119,12 +119,9 @@ private String prepareScreenshotImagePath(String path) private String getActionString(Action action) { StringJoiner joiner = new StringJoiner(" || "); - - if(action.get(Tags.Desc) != null) - { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - joiner.add(""+ escaped +""); - } + + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc, "NoActionDescriptionAvailable")); + joiner.add(""+ escaped +""); joiner.add(StringEscapeUtils.escapeHtml(action.toString())); joiner.add("ConcreteID=" + action.get(Tags.ConcreteID, "NoConcreteIdAvailable")); joiner.add("AbstractID=" + action.get(Tags.AbstractID, "NoAbstractIdAvailable")); @@ -201,11 +198,8 @@ public void addSelectedAction(State state, Action action) htmlReportUtil.addHeading(2, "Selected Action "+innerLoopCounter+" leading to State "+innerLoopCounter); String stateString = "ConcreteID=" + actionConcreteID; - if(action.get(Tags.Desc) != null) - { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - stateString += " || " + escaped; - } + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc, "NoActionDescriptionAvailable")); + stateString += " || " + escaped; htmlReportUtil.addHeading(4, stateString); diff --git a/testar/src/org/testar/reporting/PlainTextReporter.java b/testar/src/org/testar/reporting/PlainTextReporter.java index 190929deb..5aca0f4d8 100644 --- a/testar/src/org/testar/reporting/PlainTextReporter.java +++ b/testar/src/org/testar/reporting/PlainTextReporter.java @@ -103,12 +103,9 @@ private String prepareScreenshotImagePath(String path) private String getActionString(Action action) { StringJoiner joiner = new StringJoiner(" || "); - - if(action.get(Tags.Desc) != null) - { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - joiner.add(escaped); - } + + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc, "NoActionDescriptionAvailable")); + joiner.add(escaped); joiner.add(StringEscapeUtils.escapeHtml(action.toString())); joiner.add("ConcreteID="+action.get(Tags.ConcreteID, "NoConcreteIdAvailable")); joiner.add("AbstractID="+action.get(Tags.AbstractID, "NoAbstractIdAvailable")); @@ -179,11 +176,8 @@ public void addSelectedAction(State state, Action action) plainTextReportUtil.addHeading(3, "Selected Action "+innerLoopCounter+" leading to State "+innerLoopCounter); String stateString = "ConcreteID=" + actionConcreteID; - if(action.get(Tags.Desc) != null) - { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - stateString += " || " + escaped; - } + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc, "NoActionDescriptionAvailable")); + stateString += " || " + escaped; plainTextReportUtil.addHeading(5, stateString); diff --git a/testar/src/org/testar/settings/SettingsDefaults.java b/testar/src/org/testar/settings/SettingsDefaults.java index 8994522a3..1c57e28b9 100644 --- a/testar/src/org/testar/settings/SettingsDefaults.java +++ b/testar/src/org/testar/settings/SettingsDefaults.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -186,6 +186,13 @@ private SettingsDefaults() {} defaults.add(Pair.from(LogOracleCommands, new ArrayList())); defaults.add(Pair.from(LogOracleFiles, new ArrayList())); + // Settings for Coverage + defaults.add(Pair.from(JacocoCoverage, false)); + defaults.add(Pair.from(JacocoCoverageIpAddress, "localhost")); + defaults.add(Pair.from(JacocoCoveragePort, 5000)); + defaults.add(Pair.from(JacocoCoverageClasses, "path/to/SUT/classes")); + defaults.add(Pair.from(JacocoCoverageAccumulate, false)); + return defaults; } diff --git a/testar/src/org/testar/settings/SettingsFileStructure.java b/testar/src/org/testar/settings/SettingsFileStructure.java index 8fe4a3868..11041f232 100644 --- a/testar/src/org/testar/settings/SettingsFileStructure.java +++ b/testar/src/org/testar/settings/SettingsFileStructure.java @@ -306,6 +306,16 @@ public static String getTestSettingsStructure() { , ConfigTags.ReportInHTML.name() + " = " , ConfigTags.ReportInPlainText.name() + " = " , "" + , "#################################################################" + , "# Settings to configure code coverage features" + , "#################################################################" + , "" + , ConfigTags.JacocoCoverage.name() + " = " + , ConfigTags.JacocoCoverageIpAddress.name() + " = " + , ConfigTags.JacocoCoveragePort.name() + " = " + , ConfigTags.JacocoCoverageClasses.name() + " = " + , ConfigTags.JacocoCoverageAccumulate.name() + " = " + , "" ); // Second, create a list of secondary configuration tags settings diff --git a/testar/src/org/testar/settings/SettingsVerification.java b/testar/src/org/testar/settings/SettingsVerification.java index 28dae92d6..5bd68ae95 100644 --- a/testar/src/org/testar/settings/SettingsVerification.java +++ b/testar/src/org/testar/settings/SettingsVerification.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2024 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,6 +30,7 @@ package org.testar.settings; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -57,6 +58,7 @@ public static void verifySettings(Settings settings) { verifyStateModelSettings(settings); verifyRegularExpressionSettings(settings); escapeSpecialCharactersInFileWritingSettings(settings); + verifyJacocoCoverageSettings(settings); } /** @@ -170,4 +172,64 @@ private static void escapeSpecialCharactersInFileWritingSettings(Settings settin } } + /** + * Verify the JaCoCo coverage settings are valid. + * + * @param settings + */ + private static void verifyJacocoCoverageSettings(Settings settings) { + if(!settings.get(ConfigTags.JacocoCoverage, false)) { + return; // If JaCoCo Coverage is disabled, TESTAR should not use these settings + } + + // Get the values from the settings + String ipAddress = settings.get(ConfigTags.JacocoCoverageIpAddress); + Integer port = settings.get(ConfigTags.JacocoCoveragePort); + String pathClasses = settings.get(ConfigTags.JacocoCoverageClasses); + + // Validate IP Address + if (!isValidIPAddress(ipAddress)) { + System.err.println("*** WARNING: Invalid JacocoCoverageIpAddress: " + ipAddress); + } + + // Validate Port + if (port == null || port < 1 || port > 65535) { + System.err.println("*** WARNING: Invalid JacocoCoveragePort: " + port); + } + + // Validate path classes + if (!new File(pathClasses).exists()) { + System.err.println("*** WARNING: Invalid JacocoCoverageClasses: " + pathClasses); + } + } + + private static boolean isValidIPAddress(String text) { + if (text.isEmpty()) { + System.err.println("Warning: Empty JacocoCoverageIpAddress"); + return true; + } + + if (text.equalsIgnoreCase("localhost")) { + return true; + } + + String[] parts = text.split("\\."); + + // Check if it's a valid partial IP or full IP address + if (parts.length > 4) return false; // Cannot have more than 4 octets + + for (String part : parts) { + if (part.isEmpty()) continue; // Allow partial input (e.g., "192.") + + try { + int value = Integer.parseInt(part); + if (value < 0 || value > 255) { + return false; // Each octet must be between 0 and 255 + } + } catch (NumberFormatException e) { + return false; // Not a valid integer + } + } + return true; + } } diff --git a/testar/src/org/testar/settings/dialog/AdvancedPanel.java b/testar/src/org/testar/settings/dialog/AdvancedPanel.java index 11bbda791..645a217a1 100644 --- a/testar/src/org/testar/settings/dialog/AdvancedPanel.java +++ b/testar/src/org/testar/settings/dialog/AdvancedPanel.java @@ -33,7 +33,20 @@ import org.testar.monkey.ConfigTags; import org.testar.settings.Settings; +import java.awt.Color; +import java.awt.event.ActionEvent; +import java.awt.event.ItemEvent; +import java.awt.event.ItemListener; +import java.io.File; +import java.util.HashSet; +import java.util.Set; + import javax.swing.*; +import javax.swing.border.TitledBorder; +import javax.swing.text.AbstractDocument; +import javax.swing.text.AttributeSet; +import javax.swing.text.BadLocationException; +import javax.swing.text.DocumentFilter; public class AdvancedPanel extends SettingsPanel { @@ -44,6 +57,18 @@ public class AdvancedPanel extends SettingsPanel { private JLabel labelSutProcesses = new JLabel("Desktop SUT Multi-Processes"); private JTextField sutProcessesField = new JTextField(); + private JPanel coveragePanel = new JPanel(); + private Set jacocoCoverageComponents; + private JCheckBox enableJacocoCoverage = new JCheckBox("Enable JaCoCo Coverage"); + private JLabel labelIpJacocoCoverage = new JLabel("IP address JaCoCo agent"); + private JTextField fieldIpJacocoCoverage = new JTextField("localhost"); + private JLabel labelPortJacocoCoverage = new JLabel("Port JaCoCo agent"); + private JTextField fieldPortJacocoCoverage = new JTextField("5000"); + private JLabel labelClassesJacocoCoverage = new JLabel("JaCoCo SUT classes"); + private JTextField fieldClassesJacocoCoverage = new JTextField(); + private JButton dirButton = new JButton(".."); + private JCheckBox enableAccumulativeJacocoCoverage = new JCheckBox("Accumulative Action Coverage"); + public AdvancedPanel() { setLayout(null); @@ -61,12 +86,96 @@ public AdvancedPanel() { sutProcessesField.setBounds(190, 70, 420, 27); sutProcessesField.setToolTipText(ToolTipTexts.sutProcessesTTT); add(sutProcessesField); + + coveragePanel.setLayout(null); + coveragePanel.setBounds(5, 100, 610, 170); + coveragePanel.setBorder(BorderFactory.createTitledBorder( + BorderFactory.createLineBorder(Color.GRAY, 1), + "JaCoCo Coverage Settings", + TitledBorder.LEFT, + TitledBorder.TOP)); + coveragePanel(); // Prepare coverage components + add(coveragePanel); + } + + private void coveragePanel() { + // The group of settings coverage components + jacocoCoverageComponents = new HashSet<>(); + jacocoCoverageComponents.add(labelIpJacocoCoverage); + jacocoCoverageComponents.add(fieldIpJacocoCoverage); + jacocoCoverageComponents.add(labelPortJacocoCoverage); + jacocoCoverageComponents.add(fieldPortJacocoCoverage); + jacocoCoverageComponents.add(labelClassesJacocoCoverage); + jacocoCoverageComponents.add(fieldClassesJacocoCoverage); + jacocoCoverageComponents.add(dirButton); + jacocoCoverageComponents.add(enableAccumulativeJacocoCoverage); + + enableJacocoCoverage.setBounds(10, 20, 180, 27); + enableJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverage.getDescription()); + enableJacocoCoverage.addItemListener(new ItemListener() { + @Override + public void itemStateChanged(ItemEvent e) { + jacocoCoverageComponents.forEach((component) -> component.setEnabled(enableJacocoCoverage.isSelected())); + } + }); + coveragePanel.add(enableJacocoCoverage); + + labelIpJacocoCoverage.setBounds(10, 50, 180, 27); + labelIpJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverageIpAddress.getDescription()); + coveragePanel.add(labelIpJacocoCoverage); + fieldIpJacocoCoverage.setBounds(160, 50, 180, 27); + fieldIpJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverageIpAddress.getDescription()); + coveragePanel.add(fieldIpJacocoCoverage); + + labelPortJacocoCoverage.setBounds(10, 80, 180, 27); + labelPortJacocoCoverage.setToolTipText(ConfigTags.JacocoCoveragePort.getDescription()); + coveragePanel.add(labelPortJacocoCoverage); + fieldPortJacocoCoverage.setBounds(160, 80, 180, 27); + fieldPortJacocoCoverage.setToolTipText(ConfigTags.JacocoCoveragePort.getDescription()); + ((AbstractDocument) fieldPortJacocoCoverage.getDocument()).setDocumentFilter(new PortNumberFilter()); + coveragePanel.add(fieldPortJacocoCoverage); + + labelClassesJacocoCoverage.setBounds(10, 110, 180, 27); + labelClassesJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverageClasses.getDescription()); + coveragePanel.add(labelClassesJacocoCoverage); + fieldClassesJacocoCoverage.setBounds(160, 110, 180, 27); + fieldClassesJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverageClasses.getDescription()); + fieldClassesJacocoCoverage.setEditable(false); + coveragePanel.add(fieldClassesJacocoCoverage); + dirButton.setBounds(350, 110, 20, 27); + dirButton.addActionListener(this::chooseFileActionPerformed); + coveragePanel.add(dirButton); + + enableAccumulativeJacocoCoverage.setBounds(10, 140, 200, 27); + enableAccumulativeJacocoCoverage.setToolTipText(ConfigTags.JacocoCoverageAccumulate.getDescription()); + coveragePanel.add(enableAccumulativeJacocoCoverage); + } + + // show a file dialog to choose the directory where the JaCoCo classes are located + private void chooseFileActionPerformed(ActionEvent evt) { + JFileChooser fd = new JFileChooser(); + fd.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + fd.setCurrentDirectory(new File(fieldClassesJacocoCoverage.getText()).getParentFile()); + + if (fd.showOpenDialog(this) == JFileChooser.APPROVE_OPTION) { + String file = fd.getSelectedFile().getAbsolutePath(); + + // Set the text from settings in txtSutPath + fieldClassesJacocoCoverage.setText(file); + } } public void populateFrom(final Settings settings) { keyBoardListenCheck.setSelected(settings.get(ConfigTags.KeyBoardListener)); accessBridgeEnabledCheck.setSelected(settings.get(ConfigTags.AccessBridgeEnabled)); sutProcessesField.setText(settings.get(ConfigTags.SUTProcesses)); + + enableJacocoCoverage.setSelected(settings.get(ConfigTags.JacocoCoverage)); + fieldIpJacocoCoverage.setText(settings.get(ConfigTags.JacocoCoverageIpAddress)); + fieldPortJacocoCoverage.setText(settings.get(ConfigTags.JacocoCoveragePort).toString()); + fieldClassesJacocoCoverage.setText(settings.get(ConfigTags.JacocoCoverageClasses)); + enableAccumulativeJacocoCoverage.setSelected(settings.get(ConfigTags.JacocoCoverageAccumulate)); + jacocoCoverageComponents.forEach((component) -> component.setEnabled(enableJacocoCoverage.isSelected())); } /** @@ -78,6 +187,55 @@ public void extractInformation(final Settings settings) { settings.set(ConfigTags.KeyBoardListener, keyBoardListenCheck.isSelected()); settings.set(ConfigTags.AccessBridgeEnabled, accessBridgeEnabledCheck.isSelected()); settings.set(ConfigTags.SUTProcesses, sutProcessesField.getText()); + + settings.set(ConfigTags.JacocoCoverage, enableJacocoCoverage.isSelected()); + settings.set(ConfigTags.JacocoCoverageIpAddress, fieldIpJacocoCoverage.getText()); + settings.set(ConfigTags.JacocoCoveragePort, Integer.valueOf(fieldPortJacocoCoverage.getText())); + settings.set(ConfigTags.JacocoCoverageClasses, fieldClassesJacocoCoverage.getText()); + settings.set(ConfigTags.JacocoCoverageAccumulate, enableAccumulativeJacocoCoverage.isSelected()); } + // DocumentFilter that allows only valid port numbers + private class PortNumberFilter extends DocumentFilter { + private static final int MIN_PORT = 1; + private static final int MAX_PORT = 65535; + + @Override + public void insertString(FilterBypass fb, int offset, String string, AttributeSet attr) throws BadLocationException { + String currentText = fb.getDocument().getText(0, fb.getDocument().getLength()); + String newText = new StringBuilder(currentText).insert(offset, string).toString(); + if (isValidPortNumber(newText)) { + super.insertString(fb, offset, string, attr); + } + } + + @Override + public void replace(FilterBypass fb, int offset, int length, String text, AttributeSet attrs) throws BadLocationException { + String currentText = fb.getDocument().getText(0, fb.getDocument().getLength()); + String newText = new StringBuilder(currentText).replace(offset, offset + length, text).toString(); + if (isValidPortNumber(newText)) { + super.replace(fb, offset, length, text, attrs); + } + } + + @Override + public void remove(FilterBypass fb, int offset, int length) throws BadLocationException { + String currentText = fb.getDocument().getText(0, fb.getDocument().getLength()); + String newText = new StringBuilder(currentText).delete(offset, offset + length).toString(); + if (isValidPortNumber(newText)) { + super.remove(fb, offset, length); + } + } + + // Helper method to check if the text is a valid port number within the range + private boolean isValidPortNumber(String text) { + if (text.isEmpty()) return true; // Allow empty string + try { + int port = Integer.parseInt(text); + return port >= MIN_PORT && port <= MAX_PORT; + } catch (NumberFormatException e) { + return false; // Not a valid integer + } + } + } } \ No newline at end of file diff --git a/testar/src/org/testar/settings/dialog/StateModelPanel.java b/testar/src/org/testar/settings/dialog/StateModelPanel.java index 30f0dfb58..86f7e43af 100644 --- a/testar/src/org/testar/settings/dialog/StateModelPanel.java +++ b/testar/src/org/testar/settings/dialog/StateModelPanel.java @@ -138,9 +138,9 @@ private void initialize() { // add the components to the panel setLayout(null); - label1.setBounds(10,14,150,27); + label1.setBounds(10, 14, 150, 27); add(label1); - stateModelEnabledChkBox.setBounds(160,14,50,27); + stateModelEnabledChkBox.setBounds(160, 14, 50, 27); stateModelEnabledChkBox.addItemListener(new ItemListener() { @Override public void itemStateChanged(ItemEvent e) { @@ -153,14 +153,14 @@ public void itemStateChanged(ItemEvent e) { }); add(stateModelEnabledChkBox); - label2.setBounds(10,52,150,27); + label2.setBounds(10, 52, 150, 27); add(label2); - dataStoreTextfield.setBounds(160,52,125,27); + dataStoreTextfield.setBounds(160, 52, 125, 27); add(dataStoreTextfield); - label3.setBounds(10,90,150,27); + label3.setBounds(10, 90, 150, 27); add(label3); - dataStoreTypeBox.setBounds(160,90,125,27); + dataStoreTypeBox.setBounds(160, 90, 125, 27); dataStoreTypeBox.addItemListener(new ItemListener() { @Override public void itemStateChanged(ItemEvent e) { @@ -169,14 +169,14 @@ public void itemStateChanged(ItemEvent e) { }); add(dataStoreTypeBox); - label4.setBounds(10,128,150,27); + label4.setBounds(10, 128, 150, 27); add(label4); - dataStoreServerTextfield.setBounds(160,128,125,27); + dataStoreServerTextfield.setBounds(160, 128, 125, 27); add(dataStoreServerTextfield); - label13.setBounds(10,166,150,27); + label13.setBounds(10, 166, 150, 27); add(label13); - dataStoreDirectoryField.setBounds(160,166,125,27); + dataStoreDirectoryField.setBounds(160, 166, 125, 27); dataStoreDirectoryField.setEditable(false); add(dataStoreDirectoryField); @@ -185,24 +185,24 @@ public void itemStateChanged(ItemEvent e) { dirButton.setToolTipText("Select the 'databases' folder in your orientdb installation. Make sure the OrientDB server is not running."); add(dirButton); - label5.setBounds(10,204,150,27); + label5.setBounds(10, 204, 150, 27); add(label5); - dataStoreDBTextfield.setBounds(160,204,125,27); + dataStoreDBTextfield.setBounds(160, 204, 125, 27); add(dataStoreDBTextfield); - label6.setBounds(10,242,150,27); + label6.setBounds(10, 242, 150, 27); add(label6); - dataStoreUserTextfield.setBounds(160,242,125,27); + dataStoreUserTextfield.setBounds(160, 242, 125, 27); add(dataStoreUserTextfield); - label7.setBounds(10,280,150,27); + label7.setBounds(10, 280, 150, 27); add(label7); - dataStorePasswordfield.setBounds(160,280,125,27); + dataStorePasswordfield.setBounds(160, 280, 125, 27); add(dataStorePasswordfield); - label8.setBounds(10,318,150,27); + label8.setBounds(10, 318, 150, 27); add(label8); - dataStoreModeBox.setBounds(160,318,125,27); + dataStoreModeBox.setBounds(160, 318, 125, 27); add(dataStoreModeBox); // NEW COLUMN @@ -211,13 +211,13 @@ public void itemStateChanged(ItemEvent e) { stateModelWidgetStoreChkBox.setBounds(480, 52, 50, 27); add(stateModelWidgetStoreChkBox); - label9.setBounds(330,128,150,27); + label9.setBounds(330,90,150,27); add(label9); - resetDatabaseCheckbox.setBounds(480, 128, 50, 27); + resetDatabaseCheckbox.setBounds(480, 90, 50, 27); resetDatabaseCheckbox.setToolTipText("This will reset the database. All stored information will be lost."); add(resetDatabaseCheckbox); - analysisButton.setBounds(330, 166, 150, 27); + analysisButton.setBounds(330, 128, 150, 27); analysisButton.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { @@ -227,7 +227,7 @@ public void actionPerformed(ActionEvent e) { }); add(analysisButton); - stateTagsButton.setBounds(330, 204, 150, 27); + stateTagsButton.setBounds(330, 166, 150, 27); stateTagsButton.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { @@ -236,12 +236,12 @@ public void actionPerformed(ActionEvent e) { }); add(stateTagsButton); - label15.setBounds(330,242,100,27); + label15.setBounds(330, 204, 100, 27); add(label15); - actionSelectionBox.setBounds(430, 242,175,27); + actionSelectionBox.setBounds(430, 204, 175, 27); add(actionSelectionBox); - label14.setBounds(330, 280, 300, 27); + label14.setBounds(330, 242, 300, 27); add(label14); } diff --git a/testar/test/org/testar/coverage/TestCodeCoverageManager.java b/testar/test/org/testar/coverage/TestCodeCoverageManager.java new file mode 100644 index 000000000..89bd2c2e9 --- /dev/null +++ b/testar/test/org/testar/coverage/TestCodeCoverageManager.java @@ -0,0 +1,40 @@ +package org.testar.coverage; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testar.OutputStructure; +import org.testar.monkey.ConfigTags; +import org.testar.monkey.Pair; +import org.testar.settings.Settings; + +public class TestCodeCoverageManager { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testCoverageConstructorCreatesDirectory() { + // Assign the outerLoopOutputDir to the tempFolder path + OutputStructure.outerLoopOutputDir = tempFolder.getRoot().getPath(); + + // Prepare the settings required by the CodeCoverageManager constructor + List> tags = new ArrayList>(); + tags.add(Pair.from(ConfigTags.JacocoCoverage, false)); + Settings settings = new Settings(tags, new Properties()); + + String coverageDirectory = OutputStructure.outerLoopOutputDir + File.separator + "coverage"; + Assert.assertFalse(new File(coverageDirectory).exists()); + + new CodeCoverageManager(settings); + + Assert.assertTrue(new File(coverageDirectory).exists()); + } + +} diff --git a/testar/test/org/testar/coverage/jacoco/TestJacocoCoverage.java b/testar/test/org/testar/coverage/jacoco/TestJacocoCoverage.java new file mode 100644 index 000000000..d82e8ca71 --- /dev/null +++ b/testar/test/org/testar/coverage/jacoco/TestJacocoCoverage.java @@ -0,0 +1,71 @@ +package org.testar.coverage.jacoco; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testar.OutputStructure; +import org.testar.coverage.CodeCoverageManager; +import org.testar.monkey.ConfigTags; +import org.testar.monkey.Pair; +import org.testar.settings.Settings; + +public class TestJacocoCoverage { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testJacocoCoverageDirectory() { + // Assign the outerLoopOutputDir to the tempFolder path + OutputStructure.outerLoopOutputDir = tempFolder.getRoot().getPath(); + // Fake jacoco classes path + String classesPath = OutputStructure.outerLoopOutputDir + File.separator + "classes_path"; + new File(classesPath).mkdir(); + + // Prepare the settings required by the JacocoCoverage constructor + List> tags = new ArrayList>(); + tags.add(Pair.from(ConfigTags.JacocoCoverage, true)); + tags.add(Pair.from(ConfigTags.JacocoCoverageIpAddress, "localhost")); + tags.add(Pair.from(ConfigTags.JacocoCoveragePort, 5000)); + tags.add(Pair.from(ConfigTags.JacocoCoverageClasses, classesPath)); + Settings settings = new Settings(tags, new Properties()); + + String jacocoDirectory = OutputStructure.outerLoopOutputDir + + File.separator + "coverage" + + File.separator + "jacoco"; + + Assert.assertFalse(new File(jacocoDirectory).exists()); + + new CodeCoverageManager(settings); + + Assert.assertTrue(new File(jacocoDirectory).exists()); + } + + @Test + public void testWarningInformationJacocoSettings() { + // Redirect the console System-output to a variable + ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + System.setErr(new PrintStream(errContent)); + + // Prepare the settings required by the JacocoCoverage constructor + List> tags = new ArrayList>(); + tags.add(Pair.from(ConfigTags.JacocoCoverage, true)); + tags.add(Pair.from(ConfigTags.JacocoCoverageIpAddress, "my_ip")); + tags.add(Pair.from(ConfigTags.JacocoCoveragePort, 9999999)); + tags.add(Pair.from(ConfigTags.JacocoCoverageClasses, "path/suts")); + new Settings(tags, new Properties()); + + Assert.assertTrue(errContent.toString().contains("Invalid JacocoCoverageIpAddress: my_ip")); + Assert.assertTrue(errContent.toString().contains("Invalid JacocoCoveragePort: 9999999")); + Assert.assertTrue(errContent.toString().contains("Invalid JacocoCoverageClasses: path/suts")); + } + +} diff --git a/testar/test/org/testar/coverage/jacoco/TestJacocoReportCSV.java b/testar/test/org/testar/coverage/jacoco/TestJacocoReportCSV.java new file mode 100644 index 000000000..4b8fb5fd0 --- /dev/null +++ b/testar/test/org/testar/coverage/jacoco/TestJacocoReportCSV.java @@ -0,0 +1,123 @@ +package org.testar.coverage.jacoco; + +import static org.mockito.Mockito.*; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; +import org.testar.OutputStructure; +import org.testar.monkey.ConfigTags; +import org.testar.monkey.Pair; +import org.testar.settings.Settings; +import org.junit.Assert; +import org.junit.Rule; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Properties; +import java.util.Scanner; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.jacoco.core.analysis.CoverageBuilder; +import org.jacoco.core.analysis.IClassCoverage; +import org.jacoco.core.analysis.ICounter; + +public class TestJacocoReportCSV { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testGenerateJacocoReportCSV() throws Exception { + // Mocking the CoverageBuilder behavior + IClassCoverage mockClassCoverage = Mockito.mock(IClassCoverage.class); + Collection classCoverages = new ArrayList<>(); + classCoverages.add(mockClassCoverage); + + CoverageBuilder mockCoverageBuilder = Mockito.mock(CoverageBuilder.class); + Mockito.when(mockCoverageBuilder.getClasses()).thenReturn(classCoverages); + + // Mock InstructionCounter + ICounter mockInstructionCounter = Mockito.mock(ICounter.class); + Mockito.when(mockInstructionCounter.getMissedCount()).thenReturn(10); + Mockito.when(mockInstructionCounter.getCoveredCount()).thenReturn(90); + Mockito.when(mockInstructionCounter.getCoveredRatio()).thenReturn(0.9); + + // Mock BranchCounter + ICounter mockBranchCounter = Mockito.mock(ICounter.class); + Mockito.when(mockBranchCounter.getMissedCount()).thenReturn(5); + Mockito.when(mockBranchCounter.getCoveredCount()).thenReturn(50); + Mockito.when(mockBranchCounter.getCoveredRatio()).thenReturn(0.91); + + // Mock LineCounter + ICounter mockLineCounter = Mockito.mock(ICounter.class); + Mockito.when(mockLineCounter.getMissedCount()).thenReturn(7); + Mockito.when(mockLineCounter.getCoveredCount()).thenReturn(93); + Mockito.when(mockLineCounter.getCoveredRatio()).thenReturn(0.93); + + // Mock ComplexityCounter + ICounter mockComplexityCounter = Mockito.mock(ICounter.class); + Mockito.when(mockComplexityCounter.getMissedCount()).thenReturn(2); + Mockito.when(mockComplexityCounter.getCoveredCount()).thenReturn(98); + Mockito.when(mockComplexityCounter.getCoveredRatio()).thenReturn(0.98); + + // Mock MethodCounter + ICounter mockMethodCounter = Mockito.mock(ICounter.class); + Mockito.when(mockMethodCounter.getMissedCount()).thenReturn(1); + Mockito.when(mockMethodCounter.getCoveredCount()).thenReturn(99); + Mockito.when(mockMethodCounter.getCoveredRatio()).thenReturn(0.99); + + // Mock the class coverage to return the mocked counters + Mockito.when(mockClassCoverage.getInstructionCounter()).thenReturn(mockInstructionCounter); + Mockito.when(mockClassCoverage.getBranchCounter()).thenReturn(mockBranchCounter); + Mockito.when(mockClassCoverage.getLineCounter()).thenReturn(mockLineCounter); + Mockito.when(mockClassCoverage.getComplexityCounter()).thenReturn(mockComplexityCounter); + Mockito.when(mockClassCoverage.getMethodCounter()).thenReturn(mockMethodCounter); + + // Mock class name + Mockito.when(mockClassCoverage.getName()).thenReturn("MockClass"); + + // Assign the outerLoopOutputDir to the tempFolder path + OutputStructure.outerLoopOutputDir = tempFolder.getRoot().getPath(); + // Fake Jacoco classes path + String classesPath = OutputStructure.outerLoopOutputDir + File.separator + "classes_path"; + new File(classesPath).mkdir(); + + // Prepare the settings required by the JacocoCoverage constructor + List> tags = new ArrayList<>(); + tags.add(Pair.from(ConfigTags.JacocoCoverageClasses, classesPath)); + Settings settings = new Settings(tags, new Properties()); + + // Spy on the JacocoReportCSV class to mock loadJacocoAnalysis + JacocoReportCSV mockJacocoReportCSV = Mockito.spy(new JacocoReportCSV(settings)); + Mockito.doReturn(mockCoverageBuilder).when(mockJacocoReportCSV).loadJacocoAnalysis(any(String.class)); + + // Generate the CSV report with mocked coverage data + String outputCSVpath = tempFolder.getRoot().getPath() + File.separator + "output.csv"; + mockJacocoReportCSV.generateCSVresults("mockJacoco.exec", outputCSVpath); + + File outputCVSFile = new File(outputCSVpath); + Assert.assertTrue(outputCVSFile.exists()); + Assert.assertTrue(fileContains("CLASS,INSTRUCTION_MISSED,INSTRUCTION_COVERED,INSTRUCTION_COVERED_RATIO,BRANCH_MISSED,BRANCH_COVERED,BRANCH_COVERED_RATIO,LINE_MISSED,LINE_COVERED,LINE_COVERED_RATIO,COMPLEXITY_MISSED,COMPLEXITY_COVERED,COMPLEXITY_COVERED_RATIO,METHOD_MISSED,METHOD_COVERED,METHOD_COVERED_RATIO", outputCVSFile)); + Assert.assertTrue(fileContains("MockClass,10,90,90.00,5,50,91.00,7,93,93.00,2,98,98.00,1,99,99.00", outputCVSFile)); + Assert.assertTrue(fileContains("TOTAL,10,90,90.00,5,50,90.91,7,93,93.00,2,98,98.00,1,99,99.00", outputCVSFile)); + } + + private boolean fileContains(String searchText, File file) { + try (Scanner scanner = new Scanner(file)) { + // Read the content of the file line by line + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + + // Check if the line contains the specific text + if (line.contains(searchText)) { + return true; + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return false; + } + +} diff --git a/testar/test/org/testar/monkey/BuildActionsIdentifiers.java b/testar/test/org/testar/monkey/BuildActionsIdentifiers.java index 3e63871c3..307cc9a72 100644 --- a/testar/test/org/testar/monkey/BuildActionsIdentifiers.java +++ b/testar/test/org/testar/monkey/BuildActionsIdentifiers.java @@ -220,4 +220,21 @@ public void buildPasteReplaceAction() { Assert.notNull(pasteReplaceAction.get(Tags.AbstractID)); Assert.notNull(pasteReplaceAction.get(Tags.ConcreteID)); } + + @Test + public void buildNoOperationalStateAction() { + // Create the NOP action and Assert that the state is set as the OriginWidget Tag + Action NOPAction = ac.noOperationalState(state); + // Verify Action <-> Widget mapping + Assert.notNull(NOPAction.get(Tags.OriginWidget)); + Assert.notNull(state.get(Tags.ActionSet)); + Assert.isTrue(state.get(Tags.ActionSet).size() == 1); + Assert.notNull(state.get(Tags.ActionSet).iterator().next().get(Tags.Desc)); + Assert.isTrue(state.get(Tags.ActionSet).iterator().next().get(Tags.Desc, "").contains("No Operation")); + // Then build the action identifiers + defaultProtocol.buildStateActionsIdentifiers(state, new HashSet<>(Collections.singletonList(NOPAction))); + // To check that Action identifiers were built + Assert.notNull(NOPAction.get(Tags.AbstractID)); + Assert.notNull(NOPAction.get(Tags.ConcreteID)); + } } diff --git a/testar/test/org/testar/reporting/TestReportManager.java b/testar/test/org/testar/reporting/TestReportManager.java index a47693627..373b0f5b9 100644 --- a/testar/test/org/testar/reporting/TestReportManager.java +++ b/testar/test/org/testar/reporting/TestReportManager.java @@ -57,6 +57,9 @@ public void setUp() throws IOException { pasteAction.set(Tags.ConcreteID, "pasteActionConcreteID"); pasteAction.set(Tags.Desc, "pasteActionDescription"); derivedActions.add(pasteAction); + + Action emptyTagsAction = new Type("emptyTagsAction"); + derivedActions.add(emptyTagsAction); } @Test @@ -106,6 +109,7 @@ public void testHtmlReport() { Assert.assertTrue(fileContains("

Set of actions:

", htmlReportFile)); Assert.assertTrue(fileContains("typeActionDescription", htmlReportFile)); Assert.assertTrue(fileContains("pasteActionDescription", htmlReportFile)); + Assert.assertTrue(fileContains("NoActionDescriptionAvailable", htmlReportFile)); // Verify selected action information Assert.assertTrue(fileContains("

ConcreteID=typeActionConcreteID || typeActionDescription

", htmlReportFile)); // Verify verdict information @@ -151,6 +155,7 @@ public void testPlainReport() { Assert.assertTrue(fileContains("Set of actions:", txtReportFile)); Assert.assertTrue(fileContains("typeActionDescription", txtReportFile)); Assert.assertTrue(fileContains("pasteActionDescription", txtReportFile)); + Assert.assertTrue(fileContains("NoActionDescriptionAvailable", txtReportFile)); // Verify selected action information Assert.assertTrue(fileContains("ConcreteID=typeActionConcreteID || typeActionDescription", txtReportFile)); // Verify verdict information diff --git a/testar/workflow.gradle b/testar/workflow.gradle index c3dcd69e9..452c58bbd 100644 --- a/testar/workflow.gradle +++ b/testar/workflow.gradle @@ -175,7 +175,16 @@ task runTestDesktopGenericTitleNameSuspiciousTag(type: Exec, dependsOn:['install } // Verify TESTAR JavaAccessBridge works with SwingSet2 Java Swing app -task runTestDesktopJavaSwing(type: Exec, dependsOn:'installDist') { +task unzipSwingSet2Jar(type: Copy) { + dependsOn installDist + + def jarFile = file('target/install/testar/bin/suts/SwingSet2.jar') + def outputDir = file('target/install/testar/bin/suts/SwingSet2') + + from zipTree(jarFile) + into outputDir +} +task runTestDesktopJavaSwing(type: Exec, dependsOn: ['installDist', 'unzipSwingSet2Jar']) { // Read command line output standardOutput = new ByteArrayOutputStream() group = 'test_testar_workflow'