From 47b02cee8fa69e603b194c14b631518c01c41b79 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 13 Aug 2024 20:38:20 -0400 Subject: [PATCH] refactored download function --- pom.xml | 8 + .../com/gw/database/HistoryRepository.java | 12 +- .../com/gw/database/ProcessRepository.java | 9 + src/main/java/com/gw/tools/ProcessTool.java | 12 +- src/main/java/com/gw/tools/WorkflowTool.java | 227 +++++------------- 5 files changed, 93 insertions(+), 175 deletions(-) diff --git a/pom.xml b/pom.xml index 952c471d0..36f186ec9 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,14 @@ + + + com.google.code.gson + gson + 2.10.1 + + + org.springframework.boot spring-boot-starter-data-jpa diff --git a/src/main/java/com/gw/database/HistoryRepository.java b/src/main/java/com/gw/database/HistoryRepository.java index c35d59fc1..cf86e6a4b 100644 --- a/src/main/java/com/gw/database/HistoryRepository.java +++ b/src/main/java/com/gw/database/HistoryRepository.java @@ -1,12 +1,13 @@ package com.gw.database; import com.gw.jpa.History; -import com.gw.jpa.HistoryDTO; -import java.util.Collection; import java.util.List; +import java.util.Set; + import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import javax.transaction.Transactional; @@ -163,4 +164,11 @@ public interface HistoryRepository extends JpaRepository { @Query(value = "SELECT * FROM history, gwprocess WHERE history.history_id = ?1 AND history.history_process = gwprocess.id", nativeQuery = true) List findOneHistoryofProcess(String history_id); + + @Query( + value = "SELECT * FROM history WHERE history_process IN (:processIds)", + nativeQuery = true) + List findByProcessIds(@Param("processIds") List processIds); + + } diff --git a/src/main/java/com/gw/database/ProcessRepository.java b/src/main/java/com/gw/database/ProcessRepository.java index 8e7451395..26d6539be 100644 --- a/src/main/java/com/gw/database/ProcessRepository.java +++ b/src/main/java/com/gw/database/ProcessRepository.java @@ -2,6 +2,9 @@ import com.gw.jpa.GWProcess; import java.util.Collection; +import java.util.List; +import java.util.Set; + import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; @@ -77,4 +80,10 @@ public interface ProcessRepository extends CrudRepository { */ @Query(value = "select * from gwprocess where lang = 'jupyter'", nativeQuery = true) Collection findNotebookProcess(); + + + @Query( + value = "SELECT * FROM gwprocess WHERE id IN :ids", + nativeQuery = true) + List findProcessesByIds(@Param("ids") List ids); } diff --git a/src/main/java/com/gw/tools/ProcessTool.java b/src/main/java/com/gw/tools/ProcessTool.java index 8a50762ac..748ef9c05 100644 --- a/src/main/java/com/gw/tools/ProcessTool.java +++ b/src/main/java/com/gw/tools/ProcessTool.java @@ -2,10 +2,7 @@ import java.io.File; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; +import java.util.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -744,6 +741,9 @@ public String all_history(String pid, String mode) { return all_history(pid, false, mode); } - - + + + public List getProcessesByIds(List processIds) { + return processrepository.findProcessesByIds(processIds); + } } diff --git a/src/main/java/com/gw/tools/WorkflowTool.java b/src/main/java/com/gw/tools/WorkflowTool.java index 6b5c928a6..19a5e5ad5 100644 --- a/src/main/java/com/gw/tools/WorkflowTool.java +++ b/src/main/java/com/gw/tools/WorkflowTool.java @@ -1,8 +1,10 @@ package com.gw.tools; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; import com.gw.database.CheckpointRepository; import com.gw.database.HistoryRepository; +import com.gw.database.ProcessRepository; import com.gw.database.WorkflowRepository; import com.gw.jpa.ExecutionStatus; import com.gw.jpa.GWProcess; @@ -14,13 +16,9 @@ import com.gw.utils.RandomString; import java.io.File; import java.nio.file.FileSystems; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; + import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -43,6 +41,9 @@ public class WorkflowTool { @Autowired WorkflowRepository workflowrepository; + @Autowired + ProcessRepository processRepository; + @Autowired HistoryRepository historyrepository; @Autowired CheckpointRepository checkpointrepository; @@ -433,7 +434,7 @@ public String all_active_process() { /** * show the history of every execution of the workflow * - * @param string + * @param workflow_id * @return */ public String all_history(String workflow_id) { @@ -587,198 +588,90 @@ public String getExportModeById(int mode_no) { * @return * @throws ParseException */ - public String download(String wid, String option) throws ParseException { + public String download(String wid, String option) throws ParseException { Workflow wf = this.getById(wid); String fileurl = "download/temp/" + wf.getId() + ".zip"; - - String savefilepath = - bt.getFileTransferFolder() + wf.getId() + FileSystems.getDefault().getSeparator(); + String savefilepath = bt.getFileTransferFolder() + wf.getId() + FileSystems.getDefault().getSeparator(); File tf = new File(savefilepath); + if (tf.exists()) { + bt.deleteDirectory(tf); + } + tf.mkdirs(); - bt.deleteDirectory(tf); - - if (!tf.exists()) tf.mkdirs(); - - String workflowstring = bt.toJSON(wf); + Gson gson = new Gson(); - bt.writeString2File(workflowstring, savefilepath + "workflow.json"); + // Write workflow to JSON + String workflowJson = gson.toJson(wf); + bt.writeString2File(workflowJson, savefilepath + "workflow.json"); if (option.contains("processcode")) { - JSONParser jsonParser = new JSONParser(); - JSONArray arrayobj = (JSONArray) jsonParser.parse(wf.getNodes()); String codesavefile = savefilepath + "code" + FileSystems.getDefault().getSeparator(); - File codef = new File(codesavefile); + if (!codef.exists()) { + codef.mkdirs(); + } - if (!codef.exists()) codef.mkdirs(); - - StringBuffer processjson = new StringBuffer("["); - - String prefix = ""; - - for (int i = 0; i < arrayobj.size(); i++) { - - try { - - JSONObject jsonObj = (JSONObject) arrayobj.get(i); - - String process_workflow_id = (String) jsonObj.get("id"); - - String process_id = process_workflow_id.split("-")[0]; - - String targetsourcefile = codesavefile + pt.getProcessFileName(process_id); - - if (new File(targetsourcefile).exists()) continue; - - GWProcess p = pt.getProcessById(process_id); - - bt.writeString2File(p.getCode(), targetsourcefile); - - processjson.append(prefix); - - prefix = ","; - - processjson.append(pt.toJSON(p)); - - } catch (Exception e) { - - e.printStackTrace(); - } + List processIds = new ArrayList<>(); + for (Object obj : arrayobj) { + JSONObject jsonObj = (JSONObject) obj; + String process_workflow_id = (String) jsonObj.get("id"); + String process_id = process_workflow_id.split("-")[0]; + processIds.add(process_id); } - processjson.append("]"); + // Fetch all processes at once + List processes = pt.getProcessesByIds(processIds); + String processJson = gson.toJson(processes); + + bt.writeString2File(processJson, codesavefile + "process.json"); - bt.writeString2File(processjson.toString(), codesavefile + "process.json"); + // Write individual process code files + processes.parallelStream().forEach(process -> { + String targetSourceFile = codesavefile + pt.getProcessFileName(process.getId()); + bt.writeString2File(process.getCode(), targetSourceFile); + }); } if (option.contains("history")) { + String wfhistorysavefile = savefilepath + "history" + FileSystems.getDefault().getSeparator() + wid + ".json"; - String wfhistorysavefile = - savefilepath + "history" + FileSystems.getDefault().getSeparator() + wid + ".json"; - - // first save all history of the workflow - + // Fetch workflow history List histlist = historyrepository.findByWorkflowId(wid); + String workflowHistoryJson = gson.toJson(histlist); + bt.writeString2File(workflowHistoryJson, wfhistorysavefile); - StringBuffer workflowhistory = new StringBuffer("["); - - String prefix = ""; - - for (History h : histlist) { - - if ("workflowwithprocesscodegoodhistory".equals(option) - && !ExecutionStatus.DONE.equals(h.getIndicator())) { - - continue; - } - - String historystr = bt.toJSON(h); - - workflowhistory.append(prefix); - - prefix = ","; - - workflowhistory.append(historystr); - } - ; - - workflowhistory.append("]"); - - bt.writeString2File(workflowhistory.toString(), wfhistorysavefile); - - // second, save process history of one workflow execution into a file - HashSet process_id_set = new HashSet<>(); - + // Collect process IDs and fetch process histories + HashSet processIdSet = new HashSet<>(); for (History h : histlist) { - - if ("workflowwithprocesscodegoodhistory".equals(option) - && !ExecutionStatus.DONE.equals(h.getIndicator())) { - + if ("workflowwithprocesscodegoodhistory".equals(option) && !ExecutionStatus.DONE.equals(h.getIndicator())) { continue; } - String[] processhistorylist = h.getHistory_output().split(";"); - - prefix = ""; - - String processhistorysavefile = - savefilepath - + "history" - + FileSystems.getDefault().getSeparator() - + h.getHistory_id() - + ".json"; // all the process history of one workflow run - - StringBuffer processhistorybuffer = new StringBuffer("["); - - for (String processhitoryid : processhistorylist) { - - Optional hisop = historyrepository.findById(processhitoryid); - - if (hisop.isPresent()) { - - History hist = hisop.get(); - - if ("workflowwithprocesscodegoodhistory".equals(option) - && !ExecutionStatus.DONE.equals(hist.getIndicator())) { - - continue; - } - - processhistorybuffer.append(prefix); - - prefix = ","; - - processhistorybuffer.append(bt.toJSON(hist)); - - if (!process_id_set.contains(hist.getHistory_process())) - process_id_set.add(hist.getHistory_process()); - } - } - - processhistorybuffer.append("]"); - - bt.writeString2File(processhistorybuffer.toString(), processhistorysavefile); + String[] processHistoryList = h.getHistory_output().split(";"); + Collections.addAll(processIdSet, processHistoryList); } - // if need all the history of the involved processes, go into this if - if (option.contains("allhistory") || "workflowwithprocesscodegoodhistory".equals(option)) { - - for (String history_process_id : process_id_set) { - - histlist = historyrepository.findByProcessIdFull(history_process_id); + if (!processIdSet.isEmpty()) { + // Fetch all process histories at once + List processIdList = new ArrayList<>(processIdSet); + List allProcessHistories = historyrepository.findByProcessIds(processIdList); + Map> processHistoriesMap = allProcessHistories.stream() + .collect(Collectors.groupingBy(History::getHistory_process)); - StringBuffer allprocesshistorybuffer = new StringBuffer("["); + // Write each process history to its respective file + processIdSet.parallelStream().forEach(processId -> { + List processHistories = processHistoriesMap.get(processId); + String processHistoryJson = gson.toJson(processHistories); - // every process has a history file - String allprocesshistorysavefile = - savefilepath - + "history" - + FileSystems.getDefault().getSeparator() - + "process_" - + history_process_id - + ".json"; - - for (History hist : histlist) { - - if ("workflowwithprocesscodegoodhistory".equals(option) - && !ExecutionStatus.DONE.equals(hist.getIndicator())) { - - continue; - } - - allprocesshistorybuffer.append(bt.toJSON(hist)).append(","); - } - - allprocesshistorybuffer.append("]"); - - bt.writeString2File(allprocesshistorybuffer.toString(), allprocesshistorysavefile); - } + String processHistorySaveFile = savefilepath + "history" + FileSystems.getDefault().getSeparator() + "process_" + processId + ".json"; + bt.writeString2File(processHistoryJson, processHistorySaveFile); + }); } }