Skip to content

Commit

Permalink
Merge pull request #69 from sanctuuary/refactoring
Browse files Browse the repository at this point in the history 
Refactor, add logger and improve CWL output
  • Loading branch information
@vedran-kasalica
vedran-kasalica authored Jul 31, 2023
2 parents 8507030 + 4673f5e commit 0003126
Show file tree
Hide file tree
Showing 40 changed files with 1,283 additions and 1,041 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/maven.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ jobs:
pip3 install cwlref-runner
- name: Test CWL generation
run: |
git clone https://github.com/sanctuuary/APE_UseCases
cd APE_UseCases
java -jar ../target/APE-*-executable.jar ./GeoGMT/E0/config.json
cd ./GeoGMT/E0/CWL
git clone https://github.com/Workflomics/domain-annotations
cd domain-annotations
java -jar ../target/APE-*-executable.jar ./WombatP_tools/config_local.json
cd ./WombatP_tools/CWL
cwltool --enable-dev --validate workflowSolution_0.cwl
# Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
Expand Down
8 changes: 4 additions & 4 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ jobs:
# Build JAR
- mvn package -DskipTests=true
# Run use case
- git clone https://github.com/sanctuuary/APE_UseCases
- cd APE_UseCases
- java -jar ../target/APE-*-executable.jar ./GeoGMT/E0/config.json
- git clone https://github.com/Workflomics/domain-annotations
- cd domain-annotations
- java -jar ../target/APE-*-executable.jar WombatP_tools/config_local.json
# Validate with CWL runner
- cd ./GeoGMT/E0/CWL
- cd ./WombatP_tools/CWL
- cwltool --enable-dev --validate workflowSolution_0.cwl
20 changes: 16 additions & 4 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>io.github.sanctuuary</groupId>
<artifactId>APE</artifactId>
<version>2.1.1</version>
<version>2.1.4</version>
<packaging>jar</packaging>
<name>io.github.sanctuuary:APE</name>
<description>APE is a command line tool and an API for the automated exploration of possible computational pipelines (workflows) from large collections of computational tools.</description>
Expand All @@ -19,9 +19,9 @@
<developers>
<developer>
<name>Vedran Kasalica</name>
<email>v.kasalica@uu.nl</email>
<organization>Utrecht University</organization>
<organizationUrl>https://www.uu.nl/</organizationUrl>
<email>v.kasalica@esciencecenter.nl</email>
<organization>Netherlands eScience Center</organization>
<organizationUrl>https://www.esciencecenter.nl/</organizationUrl>
</developer>
</developers>
<scm>
Expand Down Expand Up @@ -128,6 +128,11 @@
<version>5.8.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.7</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
Expand Down Expand Up @@ -155,6 +160,13 @@
<artifactId>okhttp</artifactId>
<version>5.0.0-alpha.11</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.26</version>
<scope>provided</scope>
</dependency>

</dependencies>
</project>
101 changes: 52 additions & 49 deletions src/main/java/nl/uu/cs/ape/APE.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
/**
*
*/
package nl.uu.cs.ape;

import guru.nidi.graphviz.attribute.Rank.RankDir;
Expand All @@ -10,12 +7,12 @@
import nl.uu.cs.ape.constraints.ConstraintTemplate;
import nl.uu.cs.ape.core.SynthesisEngine;
import nl.uu.cs.ape.core.implSAT.SATSynthesisEngine;
import nl.uu.cs.ape.core.solutionStructure.AbstractCWLCreator;
import nl.uu.cs.ape.core.solutionStructure.DefaultCWLCreator;
import nl.uu.cs.ape.core.solutionStructure.ExecutableCWLCreator;
import nl.uu.cs.ape.core.solutionStructure.SolutionWorkflow;
import nl.uu.cs.ape.core.solutionStructure.SolutionsList;
import nl.uu.cs.ape.io.APEFiles;
import nl.uu.cs.ape.models.MappingsException;
import nl.uu.cs.ape.models.enums.SynthesisFlag;
import nl.uu.cs.ape.models.logic.constructs.TaxonomyPredicate;
import nl.uu.cs.ape.configuration.APECoreConfig;
Expand All @@ -38,12 +35,15 @@
import java.nio.file.Path;
import java.util.*;

import lombok.extern.slf4j.Slf4j;

/**
* The {@code APE} class is the main class of the library and is supposed to be
* the main interface for working with the library.
*
* @author Vedran Kasalica
*/
@Slf4j
public class APE implements APEInterface {

/** Core configuration object defined from the configuration file. */
Expand Down Expand Up @@ -129,7 +129,7 @@ private boolean setupDomain() throws APEDimensionsException, IOException, OWLOnt
boolean ontologyRead = owlReader.readOntology();

if (!ontologyRead) {
System.out.println("Error occurred while reading the provided ontology.");
log.warn("Error occurred while reading the provided ontology.");
return false;
}

Expand All @@ -146,7 +146,7 @@ private boolean setupDomain() throws APEDimensionsException, IOException, OWLOnt
succRun &= apeDomainSetup
.updateCWLAnnotationsFromYaml(cwlAnnotations);
} catch (FileNotFoundException e) {
System.err.println("Could not find CWL yaml configuration file!");
log.error("Could not find CWL yaml configuration file!");
e.printStackTrace();
}
}
Expand Down Expand Up @@ -326,7 +326,7 @@ private SolutionsList executeSynthesis(APERunConfig runConfig) throws IOExceptio

/* Encoding of the synthesis problem */
if (!implSynthesis.synthesisEncoding()) {
System.err.println("Internal error in problem encoding.");
log.error("Internal error in problem encoding.");
return null;
}
/* Execution of the synthesis - updates the object allSolutions */
Expand All @@ -350,7 +350,7 @@ private SolutionsList executeSynthesis(APERunConfig runConfig) throws IOExceptio
allSolutions.setFlag(SynthesisFlag.UNKNOWN);
}

System.out.println(allSolutions.getFlag().getMessage());
log.info(allSolutions.getFlag().getMessage());
long runTimeMS = APEUtils.timerPrintSolutions(globalTimerID, allSolutions.getNumberOfSolutions());

allSolutions.setSolvingTime(runTimeMS);
Expand Down Expand Up @@ -420,26 +420,25 @@ public static boolean writeExecutableWorkflows(SolutionsList allSolutions) {
final File executeDir = executionsFolder.toFile();
if (executeDir.isDirectory()) {
// If the directory already exists, empty it first
deleteExistingFiles(executeDir, "workflowSolution_");
deleteExistingFiles(executeDir, SolutionWorkflow.getFileNamePrefix());
} else {
executeDir.mkdir();
}
System.out.print("Loading");
log.debug("Generating executable scripts.");

/* Creating the requested scripts in parallel. */
allSolutions.getParallelStream().filter(solution -> solution.getIndex() < noExecutions).forEach(solution -> {
try {
String title = "workflowSolution_" + solution.getIndex() + ".sh";
File script = executionsFolder.resolve(title).toFile();
File script = executionsFolder.resolve(solution.getFileName() + ".sh").toFile();
APEFiles.write2file(solution.getScriptExecution(), script, false);
System.out.print(".");

} catch (IOException e) {
System.err.println("Error occurred while writing a graph to the file system.");
log.error("Error occurred while writing an executable (workflow) script to the file system.");
e.printStackTrace();
}
});

APEUtils.timerPrintText("executingWorkflows", "\nWorkflows have been executed.");
APEUtils.timerPrintText("executingWorkflows", "Workflow executables have been generated.");
return true;
}

Expand Down Expand Up @@ -473,31 +472,31 @@ public static boolean writeDataFlowGraphs(SolutionsList allSolutions, RankDir or
}
APEUtils.printHeader(null, "Generating graphical representation", "of the first " + noGraphs + " workflows");
APEUtils.timerStart("drawingGraphs", true);
System.out.println();

/* Removing the existing files from the file system. */
File graphDir = graphsFolder.toFile();
if (graphDir.isDirectory()) {
// If the directory already exists, empty it first
deleteExistingFiles(graphDir, "SolutionNo");
deleteExistingFiles(graphDir, SolutionWorkflow.getFileNamePrefix());
} else {
graphDir.mkdir();
}
System.out.print("Loading");
log.debug("Generating data flow graphs (png).");
/* Creating the requested graphs in parallel. */
allSolutions.getParallelStream().filter(solution -> solution.getIndex() < noGraphs).forEach(solution -> {
try {
String title = "SolutionNo_" + solution.getIndex() + "_length_" + solution.getSolutionLength();
String title = solution.getFileName();
Path path = graphsFolder.resolve(title);
solution.getDataflowGraph(title, orientation).write2File(path.toFile(),
allSolutions.getRunConfiguration().getDebugMode());
System.out.print(".");

} catch (IOException e) {
System.err.println("Error occurred while writing a graph to the file system.");
log.error("Error occurred while data flow graphs (png) to the file system.");
e.printStackTrace();
}
});

APEUtils.timerPrintText("drawingGraphs", "\nGraphical files have been generated.");
APEUtils.timerPrintText("drawingGraphs", "Graphical files have been generated.");

return true;
}
Expand All @@ -524,39 +523,38 @@ public static boolean writeControlFlowGraphs(SolutionsList allSolutions) {
* @param orientation Orientation in which the graph will be presented.
* @return true if the generating was successfully performed, false otherwise.
*/
public static boolean writeControlFlowGraphs(SolutionsList allSolutions, RankDir orientation)
{
public static boolean writeControlFlowGraphs(SolutionsList allSolutions, RankDir orientation) {
Path graphsFolder = allSolutions.getRunConfiguration().getSolutionDirPath2Figures();
Integer noGraphs = allSolutions.getRunConfiguration().getNoGraphs();
if (graphsFolder == null || noGraphs == null || noGraphs == 0 || allSolutions.isEmpty()) {
return false;
}
APEUtils.printHeader(null, "Generating graphical representation", "of the first " + noGraphs + " workflows");
APEUtils.timerStart("drawingGraphs", true);
System.out.println();

/* Removing the existing files from the file system. */
File graphDir = graphsFolder.toFile();
if (graphDir.isDirectory()) {
// If the directory already exists, empty it first
deleteExistingFiles(graphDir, "SolutionNo");
deleteExistingFiles(graphDir, SolutionWorkflow.getFileNamePrefix());
} else {
graphDir.mkdir();
}
System.out.print("Loading");
log.debug("Generating control flow graphs (png).");
/* Creating the requested graphs in parallel. */
allSolutions.getParallelStream().filter(solution -> solution.getIndex() < noGraphs).forEach(solution -> {
try {
String title = "SolutionNo_" + solution.getIndex() + "_length_" + solution.getSolutionLength();
String title = solution.getFileName();
Path path = graphsFolder.resolve(title);
solution.getControlflowGraph(title, orientation).write2File(path.toFile(),
allSolutions.getRunConfiguration().getDebugMode());
System.out.print(".");

} catch (IOException e) {
System.err.println("Error occurred while writing a graph to the file system.");
log.error("Error occurred while writing control flow graphs (png) to the file system.");
e.printStackTrace();
}
});
APEUtils.timerPrintText("drawingGraphs", "\nGraphical files have been generated.");
APEUtils.timerPrintText("drawingGraphs", "Graphical files have been generated.");

return true;
}
Expand All @@ -580,32 +578,37 @@ public static boolean writeCWLWorkflows(SolutionsList allSolutions) {
APEUtils.printHeader(null, String.format("Writing the first %o solution(s) to CWL files", noCWLFiles));
APEUtils.timerStart(timerID, true);

final String filePrefix = "workflowSolution_";
final File cwlDir = cwlFolder.toFile();
if (cwlDir.isDirectory()) {
// If the directory already exists, empty it first
deleteExistingFiles(cwlDir, filePrefix);
deleteExistingFiles(cwlDir, SolutionWorkflow.getFileNamePrefix());
} else {
// Create the CWL directory if it does not already exist
cwlDir.mkdir();
}
System.out.print("Loading");
log.debug("Generating CWL files.");

// Write the CWL files
allSolutions.getParallelStream().filter(solution -> solution.getIndex() < noCWLFiles).forEach(solution -> {
try {
String title = String.format("%s%o.cwl", filePrefix, solution.getIndex());
File script = cwlFolder.resolve(title).toFile();
// Write the cwl file to the file system
String titleCWL = solution.getFileName() + ".cwl";
File script = cwlFolder.resolve(titleCWL).toFile();
DefaultCWLCreator cwlCreator = new DefaultCWLCreator(solution);
APEFiles.write2file(cwlCreator.generate(), script, false);
System.out.print(".");

// Write the cwl input file (in YML) to the file system
String titleInputs = solution.getFileName() + "_inp.yml";
File inputScipt = cwlFolder.resolve(titleInputs).toFile();
APEFiles.write2file(cwlCreator.generateCWLWorkflowInputs(), inputScipt, false);

} catch (IOException e) {
System.err.println("Error occurred while writing a CWL file to the file system.");
log.error("Error occurred while writing a CWL file to the file system.");
e.printStackTrace();
}
});

APEUtils.timerPrintText(timerID, "\nCWL files have been generated.");
APEUtils.timerPrintText(timerID, "CWL files have been generated.");
return true;
}

Expand All @@ -616,13 +619,14 @@ public static boolean writeCWLWorkflows(SolutionsList allSolutions) {
* @param filePrefix The prefix of the files to delete.
*/
private static void deleteExistingFiles(File dirName, String filePrefix) {
File[] oldFiles = dirName.listFiles((dir, fileName) -> fileName.toLowerCase().startsWith(filePrefix.toLowerCase()));
File[] oldFiles = dirName
.listFiles((dir, fileName) -> fileName.toLowerCase().startsWith(filePrefix.toLowerCase()));
if (oldFiles != null) {
Arrays.stream(oldFiles).forEach(f -> {
try {
Files.delete(f.toPath());
} catch (IOException e) {
System.err.printf("Failed to delete file %s%n", f.getName());
log.warn("Failed to delete file {}.", f.getName());
}
});
}
Expand All @@ -640,7 +644,7 @@ private static void deleteExistingFiles(File dirName, String filePrefix) {
public static boolean writeExecutableCWLWorkflows(SolutionsList allSolutions, APECoreConfig coreConfig) {
// Check if the CWL annotations file is configured.
if (!coreConfig.getCwlAnnotationsFile().isPresent()) {
System.out.println("CWL annotations file not configured. No executable CWL files are generated.");
log.warn("CWL annotations file not configured. No executable CWL files are generated.");
return false;
}

Expand All @@ -654,32 +658,31 @@ public static boolean writeExecutableCWLWorkflows(SolutionsList allSolutions, AP
APEUtils.printHeader(null, String.format("Writing the first %o solution(s) to executable CWL files", noFiles));
APEUtils.timerStart(timerID, true);

final String filePrefix = "workflowSolution_";
final File cwlDir = executableCWLFolder.toFile();
if (cwlDir.isDirectory()) {
// If the directory already exists, empty it first
deleteExistingFiles(cwlDir, filePrefix);
deleteExistingFiles(cwlDir, SolutionWorkflow.getFileNamePrefix());
} else {
// Create the CWL directory if it does not already exist
cwlDir.mkdir();
}
System.out.print("Loading");
log.trace("Loading");

// Write the CWL files
allSolutions.getParallelStream().filter(solution -> solution.getIndex() < noFiles).forEach(solution -> {
try {
String title = String.format("%s%o.cwl", filePrefix, solution.getIndex());
String title = solution.getFileName() + ".cwl";
File script = executableCWLFolder.resolve(title).toFile();
ExecutableCWLCreator cwlCreator = new ExecutableCWLCreator(solution);
APEFiles.write2file(cwlCreator.generate(), script, false);
System.out.print(".");

} catch (IOException e) {
System.err.println("Error occurred while writing an executable CWL file to the file system.");
log.error("Error occurred while writing an executable CWL file to the file system.");
e.printStackTrace();
}
});

APEUtils.timerPrintText(timerID, "\nExecutable CWL files have been generated.");
APEUtils.timerPrintText(timerID, "Executable CWL files have been generated.");
return true;
}
}
Loading

0 comments on commit 0003126

Please sign in to comment.