From f9f802e0fef94466e9ce111a9e8bc9d4e6582db9 Mon Sep 17 00:00:00 2001 From: Kubilay Eren Date: Wed, 14 Feb 2024 20:37:28 +0100 Subject: [PATCH 1/4] Add Jupyter kernel scripts --- scripts/jupyterkernel/README.md | 1 + .../Untitled-checkpoint.ipynb | 6 + .../Untitled1-checkpoint.ipynb | 6 + .../Untitled10-checkpoint.ipynb | 6 + .../Untitled11-checkpoint.ipynb | 6 + .../Untitled12-checkpoint.ipynb | 6 + .../Untitled13-checkpoint.ipynb | 6 + .../Untitled14-checkpoint.ipynb | 6 + .../Untitled15-checkpoint.ipynb | 6 + .../Untitled16-checkpoint.ipynb | 6 + .../Untitled17-checkpoint.ipynb | 6 + .../Untitled18-checkpoint.ipynb | 6 + .../Untitled2-checkpoint.ipynb | 6 + .../Untitled3-checkpoint.ipynb | 6 + .../Untitled4-checkpoint.ipynb | 6 + .../Untitled5-checkpoint.ipynb | 6 + .../Untitled6-checkpoint.ipynb | 6 + .../Untitled7-checkpoint.ipynb | 6 + .../Untitled8-checkpoint.ipynb | 6 + .../Untitled9-checkpoint.ipynb | 6 + scripts/jupyterkernel/kernelsds/pom.xml | 96 +++++++ .../src/main/java/com/kernelsds/app/App.java | 42 ++++ .../com/kernelsds/app/ISystemDsKernel.java | 235 ++++++++++++++++++ .../test/java/com/kernelsds/app/AppTest.java | 38 +++ 24 files changed, 526 insertions(+) create mode 100644 scripts/jupyterkernel/README.md create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb create mode 100644 scripts/jupyterkernel/kernelsds/pom.xml create mode 100644 scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/App.java create mode 100644 scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java create mode 100644 scripts/jupyterkernel/kernelsds/src/test/java/com/kernelsds/app/AppTest.java diff --git a/scripts/jupyterkernel/README.md b/scripts/jupyterkernel/README.md new file mode 100644 index 00000000000..1d402749754 --- /dev/null +++ b/scripts/jupyterkernel/README.md @@ -0,0 +1 @@ +# SystemDSKernel \ No newline at end of file diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb new file mode 100644 index 00000000000..363fcab7ed6 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/jupyterkernel/kernelsds/pom.xml b/scripts/jupyterkernel/kernelsds/pom.xml new file mode 100644 index 00000000000..c7209f14aee --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/pom.xml @@ -0,0 +1,96 @@ + + 4.0.0 + com.kernelsds.app + kernelsds + jar + 1.0-SNAPSHOT + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 11 + 11 + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + + + true + com.kernelsds.app.App + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package + + shade + + + true + + + com.kernelsds.app.App + + + + + + + + + kernelsds + http://maven.apache.org + + + oss-sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + + + junit + junit + 3.8.1 + test + + + io.github.spencerpark + jupyter-jvm-basekernel + 2.3.0 + + + + org.apache.sds + sds + 3.2.0 + + + diff --git a/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/App.java b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/App.java new file mode 100644 index 00000000000..06f13ee271e --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/App.java @@ -0,0 +1,42 @@ +package com.kernelsds.app; + + +import io.github.spencerpark.jupyter.channels.JupyterConnection; + +import io.github.spencerpark.jupyter.kernel.KernelConnectionProperties; +import io.github.spencerpark.jupyter.channels.JupyterSocket; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.logging.Level; + +public class App +{ + public static void main( String[] args ) throws Exception { + if (args.length < 1) + throw new IllegalArgumentException("Missing connection file argument"); + + Path connectionFile = Paths.get(args[0]); + + if (!Files.isRegularFile(connectionFile)) + throw new IllegalArgumentException("Connection file '" + connectionFile + "' isn't a file."); + + String contents = new String(Files.readAllBytes(connectionFile)); + + //JupyterSocket.JUPYTER_LOGGER.setLevel(Level.WARNING); + JupyterSocket.JUPYTER_LOGGER.setLevel(Level.WARNING); + + KernelConnectionProperties connProps = KernelConnectionProperties.parse(contents); + JupyterConnection connection = new JupyterConnection(connProps); + + ISystemDsKernel kernel = new ISystemDsKernel(); + //ISystemDsKernel k = new ISystemDsKernel(); + kernel.becomeHandlerForConnection(connection); + + connection.connect(); + connection.waitUntilClose(); + } + +} diff --git a/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java new file mode 100644 index 00000000000..6134364b63a --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java @@ -0,0 +1,235 @@ +package com.kernelsds.app; +import org.apache.sysds.api.jmlc.*; +import org.apache.sysds.common.Types; +import org.apache.sysds.hops.DataOp; +import org.apache.sysds.hops.Hop; +import org.apache.sysds.lops.Lop; +import org.apache.sysds.parser.StatementBlock; +import org.apache.sysds.parser.dml.DMLParserWrapper; +import org.apache.sysds.parser.DMLTranslator; +import org.apache.sysds.parser.DMLProgram; +import org.apache.sysds.parser.ParserFactory; + +import io.github.spencerpark.jupyter.kernel.BaseKernel; +import io.github.spencerpark.jupyter.kernel.LanguageInfo; +import io.github.spencerpark.jupyter.kernel.ReplacementOptions; +import io.github.spencerpark.jupyter.kernel.display.DisplayData; +import io.github.spencerpark.jupyter.kernel.util.CharPredicate; +import io.github.spencerpark.jupyter.kernel.util.SimpleAutoCompleter; +import io.github.spencerpark.jupyter.kernel.util.StringSearch; + +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; + +public class ISystemDsKernel extends BaseKernel{ + //private static final NashornScriptEngineFactory NASHORN_ENGINE_FACTORY = new NashornScriptEngineFactory(); + + private static final SimpleAutoCompleter autoCompleter = SimpleAutoCompleter.builder() + .preferLong() + //Keywords from a great poem at https://stackoverflow.com/a/12114140 + .withKeywords("let", "this", "long", "package", "float") + .withKeywords("goto", "private", "class", "if", "short") + .withKeywords("while", "protected", "with", "debugger", "case") + .withKeywords("continue", "volatile", "interface") + .withKeywords("instanceof", "super", "synchronized", "throw") + .withKeywords("extends", "final", "export", "throws") + .withKeywords("try", "import", "double", "enum") + .withKeywords("false", "boolean", "abstract", "function") + .withKeywords("implements", "typeof", "transient", "break") + .withKeywords("void", "static", "default", "do") + .withKeywords("switch", "int", "native", "new") + .withKeywords("else", "delete", "null", "public", "var") + .withKeywords("in", "return", "for", "const", "true", "char") + .withKeywords("finally", "catch", "byte") + .build(); + + private static final CharPredicate idChar = CharPredicate.builder() + .inRange('a', 'z') + .inRange('A', 'Z') + .match('_') + .build(); + + + private LanguageInfo languageInfo; + private final Connection connection = new Connection(); + private int sca; + private HashMap hashMap = new HashMap<>(); + + public ISystemDsKernel() { + + } + + + + @Override + public LanguageInfo getLanguageInfo() { + return languageInfo; + } + + + /*String script = + " X = rand(rows=10, cols=10);" + + "R = matrix(0, rows=10, cols=1)" + + "parfor(i in 1:nrow(X))" + + " R[i,] = sum(X[i,])" + + "print(sum(R))";*/ + + + void analyzeVariables(Hop hop, HashSet inVars, HashSet outVars) { + // Check if the hop is a DataOp + if (hop instanceof DataOp) { + DataOp dataOp = (DataOp) hop; + Types.OpOpData opType = dataOp.getOp(); + + // If the hop is a TRANSIENTREAD (input variable), add to inVars + if (opType == Types.OpOpData.TRANSIENTREAD) { + inVars.add(hop.getName()); + } + // If the hop is a TRANSIENTWRITE (output variable), add to outVars + else if (opType == Types.OpOpData.TRANSIENTWRITE) { + outVars.add(hop.getName()); + } + } + + // Recursively analyze the input hops + for (Hop inputHop : hop.getInput()) { + analyzeVariables(inputHop, inVars, outVars); + } + } + void analyzeStatementBlock(StatementBlock sb, HashSet outVars) { + // Extract variable names from 'updated' and 'kill' sets and add them to outVars + if (sb.variablesUpdated() != null) { + + outVars.addAll(sb.variablesUpdated().getVariableNames()); + + } + + } + + @Override + public DisplayData eval(String expr) throws Exception { + + HashSet inputVaraibles = new HashSet<>(); + HashSet outputVaraibles = new HashSet<>(); + + if (this.hashMap.size() > 0){ + int i = 0; + for (String in: this.hashMap.keySet() + ) { + i++; + expr = in + "= read(\"./tmp/doesntexist" + i+ "\", data_type=\"scalar\", value_type=\"string\");\n"+ expr ; + } + i = 0; + + } + DMLParserWrapper dmlParser = new DMLParserWrapper(); + DMLProgram dml_program = dmlParser.parse(null,expr,null); + DMLTranslator translator = new DMLTranslator(dml_program); + translator.validateParseTree(dml_program); + translator.liveVariableAnalysis(dml_program); + translator.constructHops(dml_program); + + + + + for (StatementBlock sb : dml_program.getStatementBlocks()) { + //System.out.println(sb.toString()); + // Get HOPs for each statement block + //ArrayList hops = sb.getHops(); + //ArrayList lops = sb.getLops(); + analyzeStatementBlock(sb, outputVaraibles); + } + + + String[] outputArray; + //System.out.println("size: " + this.hashMap.size()); + //System.out.println("hash: " + this.hashMap.entrySet()); + if (this.hashMap.size() > 0){ + outputArray = outputVaraibles.toArray(new String[0]); + + for (String out: outputArray + ) { + //System.out.println("out: "+out); + expr = expr + ";\nwrite("+out+", './tmp/"+out+"');" ; + } + //System.out.println("expr: "+expr); + //PreparedScript preparedScript = this.connection.prepareScript(expr,new String[]{"b"}, outputArray); + PreparedScript preparedScript = this.connection.prepareScript(expr,this.hashMap.keySet().toArray(new String[0]), outputArray); + //preparedScript.setScalar("b",7,true); + //inScalar1 = read("./tmp/doesntexist1", data_type="scalar"); + //\nwrite(outString, './tmp/outString'); + for (String key: this.hashMap.keySet().toArray(new String[0]) + ) { + //System.out.println("key: "+key); + preparedScript.setScalar(key, Integer.parseInt(this.hashMap.get(key).toString())); + + } + + ResultVariables res = preparedScript.executeScript(); + for (String output: outputArray) { + // Retrieve and store the output variable values + Object value = res.getScalarObject(output); + + this.hashMap.put(output, value); + } + } + else { + outputArray = outputVaraibles.toArray(new String[0]); + //new String[]{"Z"} + for (String out: outputArray + ) { + //System.out.println("out: "+out); + expr = expr + ";\nwrite("+out+", './tmp/"+out+"');" ; + } + //System.out.println("expr: "+expr); + PreparedScript preparedScript = this.connection.prepareScript(expr,new String[]{}, outputArray); + + + //preparedScript.setScalar("X",10.0,true); + //preparedScript.setScalar("X",7,true); + ResultVariables res = preparedScript.executeScript(); + for (String output: outputArray) { + // Retrieve and store the output variable values + + Object value = res.getScalarObject(output); + + this.hashMap.put(output, value); + } + } + + + + //return new DisplayData(res.getString("Z")); + //PreparedScript preparedScript = this.connection.prepareScript(expr,new String[]{}, new String[] {}); + //ResultVariables res = preparedScript.executeScript(); + return null; + } + + /*@Override + public DisplayData inspect(String code, int at, boolean extraDetail) throws Exception { + StringSearch.Range match = StringSearch.findLongestMatchingAt(code, at, idChar); + String id = ""; + Object val = null; + if (match != null) { + id = match.extractSubString(code); + val = this.engine.getContext().getAttribute(id); + } + + return new DisplayData(val == null ? "No memory value for '" + id + "'" : val.toString()); + }*/ + + @Override + public ReplacementOptions complete(String code, int at) throws Exception { + StringSearch.Range match = StringSearch.findLongestMatchingAt(code, at, idChar); + if (match == null) + return null; + String prefix = match.extractSubString(code); + return new ReplacementOptions(autoCompleter.autocomplete(prefix), match.getLow(), match.getHigh()); + } +} diff --git a/scripts/jupyterkernel/kernelsds/src/test/java/com/kernelsds/app/AppTest.java b/scripts/jupyterkernel/kernelsds/src/test/java/com/kernelsds/app/AppTest.java new file mode 100644 index 00000000000..4f0a5388cd8 --- /dev/null +++ b/scripts/jupyterkernel/kernelsds/src/test/java/com/kernelsds/app/AppTest.java @@ -0,0 +1,38 @@ +package com.kernelsds.app; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} From 5ca3de66aab9387d65e5956516e03519e8215f6c Mon Sep 17 00:00:00 2001 From: Kubilay Eren Date: Wed, 14 Feb 2024 20:38:59 +0100 Subject: [PATCH 2/4] Update README.md --- scripts/jupyterkernel/README.md | 89 ++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/scripts/jupyterkernel/README.md b/scripts/jupyterkernel/README.md index 1d402749754..dff97799b7e 100644 --- a/scripts/jupyterkernel/README.md +++ b/scripts/jupyterkernel/README.md @@ -1 +1,88 @@ -# SystemDSKernel \ No newline at end of file + +# SystemDS Kernel Setup Guide + +This README outlines the steps to set up a Jupyter kernel for SystemDS, focusing on building SystemDS from source, preparing a custom repository with specific dependencies, and setting up a Jupyter kernel. + +## Prerequisites + +- Java JDK 11 or later +- Maven +- Git +- Jupyter Notebook or JupyterLab + +## Step 1: Clone and Build SystemDS + +Clone the Apache SystemDS repository and build it. This step ensures that the SystemDS JAR is locally available. + +[How to install SystemDS?](https://apache.github.io/systemds/site/install.html) + +After the build, save the JAR to your local Maven Repo. + +```bash +mvn install:install-file -Dfile=path/to/your/jar-file.jar -DgroupId=org.apache.sds -DartifactId=sds -Dversion=3.2.0 -Dpackaging=jar +``` +Note: Ensure that any modifications to groupId, artifactId, and version are carefully mirrored in the pom.xml dependencies section of the Kernel. Inconsistencies between these identifiers in your project setup and the pom.xml file can lead to build failures or dependency resolution issues. + +## Step 2: Set Up Your Repository + +Clone SystemDS kernel. + +```bash +git clone https://github.com/kubieren/SystemDSKernel.git +cd SystemDSKernel/kernelsds +``` + +Build your project, which is configured with Maven. This step compiles your code and packages it, taking into account the dependencies specified in your `pom.xml`. + +```bash +mvn clean package +``` + +## Step 3: Create the Kernel Specification + +Navigate to or create a directory where you wish to store your kernel's configuration. For example, create `my_systemds_kernel` in your home directory: + +```bash +mkdir -p ~/my_systemds_kernel +``` + +Within this directory, create a `kernel.json` file with the following content, adjusting the path to your JAR file as necessary: + +```json +{ + "argv": ["java", "-jar", "/path/to/your/kernelJarFile/kernelsds-1.0-SNAPSHOT.jar", "{connection_file}"], + "display_name": "SystemDS Kernel", + "language": "systemds", + "interrupt_mode": "message" +} +``` + +## Step 4: Install the Kernel Specification + +Install your kernel specification with Jupyter by running: + +```bash +jupyter kernelspec install ~/path_to_my_systemds_kernel --user +``` + +This command makes the SystemDS kernel available to Jupyter. + +## Step 5: Launch Jupyter Notebook + +Start Jupyter Notebook or JupyterLab: + +```bash +jupyter notebook +``` + +or + +```bash +jupyter lab +``` + +You should now be able to create new notebooks with the "SystemDS Kernel" option. + +## Conclusion + +Follow these steps to integrate SystemDS with Jupyter Notebook, allowing you to execute SystemDS operations directly from Jupyter notebooks. Ensure all paths and URLs are correct based on your environment and where you've placed the SystemDS JAR file. From 0e27754200e0513df04a3bd52804e2b991d3358c Mon Sep 17 00:00:00 2001 From: Kubilay Eren Date: Wed, 14 Feb 2024 20:40:03 +0100 Subject: [PATCH 3/4] Delete scripts/jupyterkernel/kernelsds/.ipynb_checkpoints directory --- .../kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled10-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled11-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled12-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled13-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled14-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled15-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled16-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled17-checkpoint.ipynb | 6 ------ .../.ipynb_checkpoints/Untitled18-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb | 6 ------ .../kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb | 6 ------ 19 files changed, 114 deletions(-) delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb delete mode 100644 scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled1-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled10-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled11-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled12-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled13-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled14-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled15-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled16-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled17-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled18-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled2-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled3-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled4-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled5-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled6-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled7-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled8-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb b/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb deleted file mode 100644 index 363fcab7ed6..00000000000 --- a/scripts/jupyterkernel/kernelsds/.ipynb_checkpoints/Untitled9-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} From 6a23b5817b2f60feea51fbb975199269024e0758 Mon Sep 17 00:00:00 2001 From: Kubilay Eren Date: Thu, 22 Feb 2024 18:32:14 +0100 Subject: [PATCH 4/4] matrix typed objects can be given as input --- .../com/kernelsds/app/ISystemDsKernel.java | 244 +++++++++++++++--- 1 file changed, 204 insertions(+), 40 deletions(-) diff --git a/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java index 6134364b63a..590802fb57f 100644 --- a/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java +++ b/scripts/jupyterkernel/kernelsds/src/main/java/com/kernelsds/app/ISystemDsKernel.java @@ -17,6 +17,7 @@ import io.github.spencerpark.jupyter.kernel.util.CharPredicate; import io.github.spencerpark.jupyter.kernel.util.SimpleAutoCompleter; import io.github.spencerpark.jupyter.kernel.util.StringSearch; +import org.apache.sysds.runtime.instructions.cp.ScalarObject; import javax.script.ScriptContext; import javax.script.ScriptEngine; @@ -26,6 +27,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class ISystemDsKernel extends BaseKernel{ //private static final NashornScriptEngineFactory NASHORN_ENGINE_FACTORY = new NashornScriptEngineFactory(); @@ -60,6 +63,7 @@ public class ISystemDsKernel extends BaseKernel{ private final Connection connection = new Connection(); private int sca; private HashMap hashMap = new HashMap<>(); + private HashMap hashMapForMatrix = new HashMap<>(); public ISystemDsKernel() { @@ -80,8 +84,111 @@ public LanguageInfo getLanguageInfo() { + " R[i,] = sum(X[i,])" + "print(sum(R))";*/ + public double[][] stringToDoubleArray(String matrixString) { + // Split the input string into rows. + String[] rows = matrixString.split(";"); + + // Prepare the resulting double array with the correct number of rows. + double[][] matrix = new double[rows.length][]; + + for (int i = 0; i < rows.length; i++) { + // Split each row into its individual values. + String[] values = rows[i].split(","); + + // Prepare the row array. + matrix[i] = new double[values.length]; + + for (int j = 0; j < values.length; j++) { + // Convert each value to double and store in the matrix. + try { + matrix[i][j] = Double.parseDouble(values[j].trim()); // Trim to remove any leading or trailing spaces. + } catch (NumberFormatException e) { + // Handle the case where the string cannot be parsed as a double. + System.err.println("Error parsing '" + values[j] + "' as double."); + // Optionally, initialize to a default value or throw an exception. + } + } + } + + return matrix; + } + + + public static Set extractOutputVariables(String script) { + Set outputVariables = new HashSet<>(); // Use a set to avoid duplicates + String[] lines = script.split("\\n"); // Split the script into lines + + // Pattern to extract variable names (considering words before '=' and ignoring whitespace) + Pattern pattern = Pattern.compile("\\s*(\\w+)\\s*(?:\\[.*\\])?\\s*=.*"); + + for (String line : lines) { + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + outputVariables.add(matcher.group(1)); // Add the variable name to the set + } + } + + return outputVariables; + } + + public static Set findInputVariables(String script) { + // Split the script into lines and then into individual statements + String[] statements = script.replace("\n", ";").split(";"); + + Set definedVariables = new HashSet<>(); + Set usedVariables = new HashSet<>(); - void analyzeVariables(Hop hop, HashSet inVars, HashSet outVars) { + // Pattern to match identifiers that are not purely numeric and not followed by '(' (to exclude function calls) + // and not enclosed in quotes. + Pattern variablePattern = Pattern.compile("\\b[a-zA-Z]\\w*\\b(?!\\()"); + Pattern stringPattern = Pattern.compile("\"[^\"]*\"|'[^']*'"); + + for (String statement : statements) { + statement = statement.trim(); // Trim whitespace from each statement + + // Remove string literals from the statement before analyzing + Matcher stringMatcher = stringPattern.matcher(statement); + String statementWithoutStrings = stringMatcher.replaceAll(""); + + // Check for variable definitions and uses in the modified statement + Matcher varMatcher = variablePattern.matcher(statementWithoutStrings); + while (varMatcher.find()) { + usedVariables.add(varMatcher.group()); + } + + // Check for variable definitions (assuming simple assignment statements) + String[] parts = statementWithoutStrings.split("="); + if (parts.length == 2) { + String potentialVariable = parts[0].trim(); + if (variablePattern.matcher(potentialVariable).matches()) { + definedVariables.add(potentialVariable); + } + } + } + + // Input variables are those that are used but not defined within the script + usedVariables.removeAll(definedVariables); + + return usedVariables; + } + + public static Object createObject(String type, ScalarObject value) { + System.out.println("type: "+type); + switch (type) { + case "class org.apache.sysds.runtime.instructions.cp.StringObject": + return value.getStringValue(); + case "class org.apache.sysds.runtime.instructions.cp.BooleanObject": + return value.getBooleanValue(); + case "class org.apache.sysds.runtime.instructions.cp.DoubleObject": + return value.getDoubleValue(); + case "class org.apache.sysds.runtime.instructions.cp.IntObject": + return Integer.parseInt(value.getStringValue()); + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + /*void analyzeVariables(Hop hop, HashSet inVars, HashSet outVars) { // Check if the hop is a DataOp if (hop instanceof DataOp) { DataOp dataOp = (DataOp) hop; @@ -101,34 +208,27 @@ else if (opType == Types.OpOpData.TRANSIENTWRITE) { for (Hop inputHop : hop.getInput()) { analyzeVariables(inputHop, inVars, outVars); } - } + }*/ void analyzeStatementBlock(StatementBlock sb, HashSet outVars) { // Extract variable names from 'updated' and 'kill' sets and add them to outVars if (sb.variablesUpdated() != null) { - outVars.addAll(sb.variablesUpdated().getVariableNames()); - } - } @Override public DisplayData eval(String expr) throws Exception { - HashSet inputVaraibles = new HashSet<>(); - HashSet outputVaraibles = new HashSet<>(); + //HashSet inputVariables = new HashSet<>(); + //HashSet outputVariables = new HashSet<>(); + String[] outputArray; + Set set1 = findInputVariables(expr); + Set setoutputs = findInputVariables(expr); - if (this.hashMap.size() > 0){ - int i = 0; - for (String in: this.hashMap.keySet() - ) { - i++; - expr = in + "= read(\"./tmp/doesntexist" + i+ "\", data_type=\"scalar\", value_type=\"string\");\n"+ expr ; - } - i = 0; - } - DMLParserWrapper dmlParser = new DMLParserWrapper(); + + + /*DMLParserWrapper dmlParser = new DMLParserWrapper(); DMLProgram dml_program = dmlParser.parse(null,expr,null); DMLTranslator translator = new DMLTranslator(dml_program); translator.validateParseTree(dml_program); @@ -139,55 +239,102 @@ public DisplayData eval(String expr) throws Exception { for (StatementBlock sb : dml_program.getStatementBlocks()) { - //System.out.println(sb.toString()); - // Get HOPs for each statement block - //ArrayList hops = sb.getHops(); - //ArrayList lops = sb.getLops(); - analyzeStatementBlock(sb, outputVaraibles); + analyzeStatementBlock(sb, outputVariables); + }*/ + int i = 0; + for (String str: set1 + ) { + i++; + if(this.hashMapForMatrix.get(str) != null){ + expr = str + "= read(\"./tmp/doesntexist" + str+ "\", data_type=\"matrix\");\n"+ expr ; + } + else{ + expr = str + "= read(\"./tmp/doesntexist" + i+ "\", data_type=\"scalar\", value_type=\"string\");\n"+ expr ; + } } + //i = 0; - - String[] outputArray; - //System.out.println("size: " + this.hashMap.size()); - //System.out.println("hash: " + this.hashMap.entrySet()); - if (this.hashMap.size() > 0){ - outputArray = outputVaraibles.toArray(new String[0]); + if (this.hashMap.size() > 0 || this.hashMapForMatrix.size()>0){ + setoutputs.addAll(extractOutputVariables(expr)); + setoutputs.addAll(set1); + outputArray = setoutputs.toArray(new String[0]); for (String out: outputArray ) { //System.out.println("out: "+out); expr = expr + ";\nwrite("+out+", './tmp/"+out+"');" ; } - //System.out.println("expr: "+expr); + System.out.println("expr: "+expr); //PreparedScript preparedScript = this.connection.prepareScript(expr,new String[]{"b"}, outputArray); - PreparedScript preparedScript = this.connection.prepareScript(expr,this.hashMap.keySet().toArray(new String[0]), outputArray); + PreparedScript preparedScript = this.connection.prepareScript(expr,set1.toArray(new String[0]), outputArray); //preparedScript.setScalar("b",7,true); //inScalar1 = read("./tmp/doesntexist1", data_type="scalar"); //\nwrite(outString, './tmp/outString'); - for (String key: this.hashMap.keySet().toArray(new String[0]) + for (String key: set1 ) { - //System.out.println("key: "+key); - preparedScript.setScalar(key, Integer.parseInt(this.hashMap.get(key).toString())); + System.out.println("hello: " + key); + if(this.hashMapForMatrix.get(key) != null){ + + double [][] matrix = this.hashMapForMatrix.get(key); + + preparedScript.setMatrix(key, matrix); + + } + else { + switch (this.hashMap.get(key).getClass().getSimpleName()) { + case "String": + preparedScript.setScalar(key, this.hashMap.get(key).toString()); + break; // Prevent fall-through + case "Boolean": + preparedScript.setScalar(key, Boolean.parseBoolean(this.hashMap.get(key).toString())); + break; // Prevent fall-through + case "Double": + preparedScript.setScalar(key, Double.parseDouble(this.hashMap.get(key).toString())); + break; // Prevent fall-through + case "Integer": + preparedScript.setScalar(key, Integer.parseInt(this.hashMap.get(key).toString())); + break; // Prevent fall-through + + default: + throw new IllegalArgumentException("Unsupported type " + this.hashMap.get(key).getClass().getSimpleName()); + } + } + } + ResultVariables res = preparedScript.executeScript(); + //System.out.println(res.); for (String output: outputArray) { // Retrieve and store the output variable values - Object value = res.getScalarObject(output); + Types.DataType dt = res.getDataType(output); + System.out.println("output is: " + output); + System.out.println("datatype is: " + dt); + if(dt.isMatrix()){ + this.hashMapForMatrix.put(output, res.getMatrix(output)); + } + + else{ + ScalarObject tmp = res.getScalarObject(output); + + Object value = createObject(tmp.getClass().toString(), tmp); + this.hashMap.put(output, value); + } - this.hashMap.put(output, value); } } else { - outputArray = outputVaraibles.toArray(new String[0]); + setoutputs.addAll(extractOutputVariables(expr)); + setoutputs.addAll(set1); + outputArray = setoutputs.toArray(new String[0]); //new String[]{"Z"} for (String out: outputArray ) { - //System.out.println("out: "+out); + System.out.println("out: "+out); expr = expr + ";\nwrite("+out+", './tmp/"+out+"');" ; } - //System.out.println("expr: "+expr); + System.out.println("expr: "+expr); PreparedScript preparedScript = this.connection.prepareScript(expr,new String[]{}, outputArray); @@ -196,10 +343,27 @@ public DisplayData eval(String expr) throws Exception { ResultVariables res = preparedScript.executeScript(); for (String output: outputArray) { // Retrieve and store the output variable values + Types.DataType dt = res.getDataType(output); + System.out.println("output is: " + output); + System.out.println("datatype is: " + dt); + if(dt.isMatrix()){ + + //Object value = res.getMatrix(output); + //System.out.println(res.getMatrix(output).length); + + this.hashMapForMatrix.put(output, res.getMatrix(output)); + + + } + else{ + ScalarObject tmp = res.getScalarObject(output); + Object value = createObject(tmp.getClass().toString(), tmp); + System.out.println("value: "+value); + this.hashMap.put(output, value); + } + - Object value = res.getScalarObject(output); - this.hashMap.put(output, value); } }