Skip to content

Commit 689ad62

Browse files
ipatiniipatini
committed
Extended EMS-Nebulous plugin to (a) cache optimiser and solver solutions (at platform side), (b) parse and cache composite constant bindings, (c) use the cached solutions at application side (and evaluate composite constants)
1 parent 9a5b5a5 commit 689ad62

19 files changed

+521
-41
lines changed

‎nebulous/ems-core/broker-cep/src/main/java/gr/iccs/imu/ems/brokercep/cep/MathUtil.java

+55
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,22 @@ public static void clearFunctionDefinitions() {
5050

5151
// ------------------------------------------------------------------------
5252

53+
public static Double getConstant(String constantName) {
54+
return constants.get(constantName).getConstantValue();
55+
}
56+
5357
public static void setConstant(String constantName, double constantValue) {
5458
log.debug("MathUtil: Set constant: name={}, value={}", constantName, constantValue);
5559
Constant con = new Constant(constantName, constantValue);
5660
constants.put(constantName, con);
5761
}
5862

63+
public static Map<String, Double> getConstants() {
64+
Map<String, Double> map = new HashMap<>();
65+
constants.forEach((name,constant) -> map.put(name, constant.getConstantValue()));
66+
return map;
67+
}
68+
5969
public static void setConstants(Map<String, Double> constantsMap) {
6070
log.debug("MathUtil: Add constants using map: {}", constantsMap);
6171
//constantsMap.entrySet().stream().forEach(c -> setConstant(c.getKey(), c.getValue()));
@@ -139,6 +149,51 @@ public static void clearConstants() {
139149
return initTokens;
140150
}
141151

152+
public static @NonNull String renameFormulaArguments(String formula, Map<String,String> argRenames) {
153+
log.debug("MathUtil: renameFormulaArguments: formula={}, renames={}", formula, argRenames);
154+
if (StringUtils.isBlank(formula)) {
155+
log.debug("MathUtil: renameFormulaArguments: Formula is null or empty");
156+
return formula;
157+
}
158+
if (argRenames==null || argRenames.isEmpty()) {
159+
log.debug("MathUtil: renameFormulaArguments: Formula arg. renames map is null or empty");
160+
return formula;
161+
}
162+
163+
// Create MathParser expression
164+
Expression e = new Expression(formula);
165+
//e.setVerboseMode();
166+
log.trace("MathUtil: renameFormulaArguments: expression={}", e.getExpressionString());
167+
168+
// Tokenize expression
169+
@NonNull List<Token> initTokens = extractFormulaTokens(e);
170+
171+
// Search for variables and rename them
172+
String newFormula = initTokens.stream()
173+
.map(t -> {
174+
String s = t.tokenStr;
175+
if (t.tokenTypeId == Token.NOT_MATCHED) {
176+
if ("argument".equals(t.looksLike)) {
177+
s = argRenames.getOrDefault(s.trim(), s);
178+
}
179+
}
180+
return s;
181+
})
182+
.collect(Collectors.joining());
183+
log.trace("MathUtil: renameFormulaArguments: New expression={}", e.getExpressionString());
184+
185+
// Check new formula validity
186+
e = new Expression(newFormula);
187+
boolean lexSyntax = e.checkLexSyntax();
188+
boolean genSyntax = e.checkSyntax();
189+
if (log.isTraceEnabled()) {
190+
log.trace("MathUtil: renameFormulaArguments: New expression: lexSyntax={}, genSyntax: {}", lexSyntax, genSyntax);
191+
log.trace("MathUtil: renameFormulaArguments: New expression: syntax-status={}, error={}", e.getSyntaxStatus(), e.getErrorMessage());
192+
}
193+
194+
return newFormula;
195+
}
196+
142197
// ------------------------------------------------------------------------
143198

144199
public static boolean containsAggregator(String formula) {

‎nebulous/ems-nebulous/pom.xml

+5-5
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@
3030
<ems.version>7.0.0-SNAPSHOT</ems.version>
3131

3232
<!-- Spring Boot versions -->
33-
<spring.version>6.1.13</spring.version>
34-
<spring-boot.version>3.2.10</spring-boot.version>
35-
<snakeyaml.version>2.2</snakeyaml.version>
36-
<lombok.version>1.18.34</lombok.version>
33+
<spring.version>6.2.2</spring.version>
34+
<spring-boot.version>3.4.2</spring-boot.version>
35+
<jackson.version>2.18.2</jackson.version>
36+
<snakeyaml.version>2.3</snakeyaml.version>
37+
<lombok.version>1.18.36</lombok.version>
3738

3839
<!-- Nebulous-EMS extension dependency versions -->
39-
<jackson.version>2.17.2</jackson.version>
4040
<json-path.version>2.9.0</json-path.version>
4141
<thymeleaf.version>3.1.2.RELEASE</thymeleaf.version>
4242
<schematron.version>8.0.1</schematron.version>

‎nebulous/ems-nebulous/src/main/java/eu/nebulous/ems/boot/BootService.java

+20-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import eu.nebulouscloud.exn.core.Publisher;
1313
import lombok.RequiredArgsConstructor;
1414
import lombok.extern.slf4j.Slf4j;
15+
import org.apache.commons.lang3.StringUtils;
16+
import org.springframework.beans.factory.InitializingBean;
1517
import org.springframework.stereotype.Service;
1618

1719
import java.io.IOException;
@@ -23,11 +25,16 @@
2325
@Slf4j
2426
@Service
2527
@RequiredArgsConstructor
26-
public class BootService {
28+
public class BootService implements InitializingBean {
2729
private final EmsBootProperties properties;
2830
private final IndexService indexService;
2931
private final ObjectMapper objectMapper;
3032

33+
@Override
34+
public void afterPropertiesSet() throws Exception {
35+
log.info("EMS Boot Service: {}", properties.isEnabled() ? "enabled" : "disabled");
36+
}
37+
3138
void processEmsBootMessage(Command command, String appId, Publisher emsBootResponsePublisher) throws IOException {
3239
// Process EMS Boot message
3340
log.info("Received a new EMS Boot message from external broker: {}", command.body());
@@ -41,20 +48,30 @@ void processEmsBootMessage(Command command, String appId, Publisher emsBootRespo
4148
}
4249
String modelFile = entry.get(ModelsService.MODEL_FILE_KEY);
4350
String bindingsFile = entry.get(ModelsService.BINDINGS_FILE_KEY);
51+
String solutionFile = entry.get(ModelsService.SOLUTIONS_FILE_KEY);
4452
String optimiserMetricsFile = entry.get(ModelsService.OPTIMISER_METRICS_FILE_KEY);
4553
log.info("""
4654
Received EMS Boot request:
4755
App-Id: {}
4856
Model File: {}
4957
Bindings File: {}
58+
Solution File: {}
5059
Opt. Metrics File: {}
51-
""", appId, modelFile, bindingsFile, optimiserMetricsFile);
60+
""", appId, modelFile, bindingsFile, solutionFile, optimiserMetricsFile);
61+
62+
if (StringUtils.isAnyBlank(appId, modelFile, bindingsFile, optimiserMetricsFile)) {
63+
log.warn("Missing info in EMS Boot entry for app-id: {}", appId);
64+
return;
65+
}
5266

5367
String modelStr = Files.readString(Paths.get(properties.getModelsDir(), modelFile));
5468
log.debug("Model file contents:\n{}", modelStr);
5569
String bindingsStr = Files.readString(Paths.get(properties.getModelsDir(), bindingsFile));
5670
Map bindingsMap = objectMapper.readValue(bindingsStr, Map.class);
5771
log.debug("Bindings file contents:\n{}", bindingsMap);
72+
String solutionStr = Files.readString(Paths.get(properties.getModelsDir(), solutionFile));
73+
Map solutionMap = objectMapper.readValue(solutionStr, Map.class);
74+
log.debug("Solution file contents:\n{}", solutionMap);
5875
String metricsStr = Files.readString(Paths.get(properties.getModelsDir(), optimiserMetricsFile));
5976
List metricsList = objectMapper.readValue(metricsStr, List.class);
6077
log.debug("Optimiser Metrics file contents:\n{}", metricsList);
@@ -64,6 +81,7 @@ void processEmsBootMessage(Command command, String appId, Publisher emsBootRespo
6481
"application", appId,
6582
"metric-model", modelStr,
6683
"bindings", bindingsMap,
84+
"solution", solutionMap,
6785
"optimiser-metrics", metricsList
6886
);
6987
emsBootResponsePublisher.send(message, appId);

‎nebulous/ems-nebulous/src/main/java/eu/nebulous/ems/boot/EmsBootProperties.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
import org.springframework.boot.context.properties.ConfigurationProperties;
1616
import org.springframework.context.annotation.Configuration;
1717

18+
import static gr.iccs.imu.ems.util.EmsConstant.EMS_PROPERTIES_PREFIX;
19+
1820
@Slf4j
1921
@Data
2022
@Configuration
21-
@ConfigurationProperties(prefix = "boot")
23+
@ConfigurationProperties(prefix = EMS_PROPERTIES_PREFIX + "boot")
2224
public class EmsBootProperties implements InitializingBean {
2325
public final static String NEBULOUS_TOPIC_PREFIX = "eu.nebulouscloud.";
2426
public static final String COMPONENT_NAME = "monitoring";
@@ -37,6 +39,7 @@ public void afterPropertiesSet() throws Exception {
3739

3840
private String dslTopic = NEBULOUS_TOPIC_PREFIX + "ui.dsl.generic";
3941
private String optimiserMetricsTopic = NEBULOUS_TOPIC_PREFIX + "optimiser.controller.metric_list";
42+
private String solutionsTopic = NEBULOUS_TOPIC_PREFIX + "optimiser.solver.solution";
4043
private String modelsTopic = NEBULOUS_TOPIC_PREFIX + "ui.dsl.metric_model";
4144
private String modelsResponseTopic = NEBULOUS_TOPIC_PREFIX + "ui.dsl.metric_model.reply";
4245
private String emsBootTopic = NEBULOUS_TOPIC_PREFIX + "ems.boot";

‎nebulous/ems-nebulous/src/main/java/eu/nebulous/ems/boot/IndexService.java

+6
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,12 @@ public Map<String,String> getAppBindings(@NonNull String appId) throws IOExcepti
154154
return objectMapper.readValue(bindingsStr, Map.class);
155155
}
156156

157+
public Map<String,Double> getAppSolution(@NonNull String appId) throws IOException {
158+
String fileName = getAppData(appId).get(ModelsService.SOLUTIONS_FILE_KEY);
159+
String solutionStr = applicationContext.getBean(ModelsService.class).readFromFile(fileName);
160+
return objectMapper.readValue(solutionStr, Map.class);
161+
}
162+
157163
public synchronized boolean deleteAppData(@NonNull String appId) throws IOException {
158164
@NonNull Map<String, Object> map = loadIndexContents();
159165
boolean removed = map.remove(appId) != null;

‎nebulous/ems-nebulous/src/main/java/eu/nebulous/ems/boot/ModelsService.java

+114-8
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
import com.fasterxml.jackson.databind.ObjectMapper;
1212
import eu.nebulous.ems.translate.TranslationService;
13+
import gr.iccs.imu.ems.brokercep.cep.MathUtil;
14+
import lombok.NonNull;
1315
import lombok.RequiredArgsConstructor;
1416
import lombok.extern.slf4j.Slf4j;
1517
import org.apache.commons.lang3.StringUtils;
@@ -21,8 +23,10 @@
2123
import java.nio.file.Files;
2224
import java.nio.file.Path;
2325
import java.nio.file.Paths;
26+
import java.util.HashMap;
2427
import java.util.List;
2528
import java.util.Map;
29+
import java.util.Set;
2630
import java.util.stream.Collectors;
2731

2832
@Slf4j
@@ -31,13 +35,19 @@
3135
public class ModelsService implements InitializingBean {
3236
final static String MODEL_FILE_KEY = "model-file";
3337
final static String BINDINGS_FILE_KEY = "bindings-file";
38+
final static String SOLUTIONS_FILE_KEY = "solutions-file";
3439
final static String OPTIMISER_METRICS_FILE_KEY = "optimiser-metrics-file";
3540

41+
public final static String SIMPLE_BINDING_KEY = "simple-bindings";
42+
public final static String COMPOSITE_BINDING_KEY = "composite-bindings";
43+
3644
private final TranslationService translationService;
3745
private final EmsBootProperties properties;
3846
private final ObjectMapper objectMapper;
3947
private final IndexService indexService;
4048

49+
private final Map<String,Map<String,Double>> allVariableValues = new HashMap<>();
50+
4151
@Override
4252
public void afterPropertiesSet() throws Exception {
4353
if (! properties.isEnabled()) {
@@ -65,19 +75,62 @@ String extractBindings(Command command, String appId) throws IOException {
6575
log.debug("Received a new DSL Generic message from external broker: {}", command.body());
6676

6777
// Extract EMS constants-to-Optimizer variables bindings
68-
Map<String, String> bindingsMap = null;
78+
Map<String,Map<String, String>> bindingsMap = null;
6979
try {
7080
List<Map<String,Object>> list = (List) command.body().get("utilityFunctions");
7181
if (list==null || list.isEmpty()) {
7282
log.warn("No utilityFunctions found in DSL generic message: {}", command.body());
7383
} else {
74-
bindingsMap = list.stream()
75-
.filter(uf -> "constant".equalsIgnoreCase( uf.getOrDefault("type", "").toString() ))
84+
Map<String, String> simpleBindingsMap = list.stream()
85+
.filter(uf -> "constant".equalsIgnoreCase(uf.getOrDefault("type", "").toString()))
86+
.filter(uf -> ((List) ((Map) uf.get("expression")).get("variables")).size() == 1
87+
&& StringUtils.equals(
88+
((Map) uf.get("expression")).getOrDefault("formula", "").toString().trim(),
89+
((Map) ((List) ((Map) uf.get("expression")).get("variables")).get(0)).getOrDefault("name", "").toString().trim()
90+
)
91+
)
92+
.collect(Collectors.toMap(
93+
uf -> ((Map) ((List) ((Map) uf.get("expression")).get("variables")).get(0)).getOrDefault("value", "").toString().trim(),
94+
uf -> uf.getOrDefault("name", "").toString().trim()
95+
));
96+
Map<String, String> compositeBindingsMap = list.stream()
97+
.filter(uf -> "constant".equalsIgnoreCase(uf.getOrDefault("type", "").toString()))
98+
.filter(uf -> ((List) ((Map) uf.get("expression")).get("variables")).size() > 1
99+
|| ! StringUtils.equals(
100+
((Map) uf.get("expression")).getOrDefault("formula", "").toString().trim(),
101+
((Map) ((List) ((Map) uf.get("expression")).get("variables")).get(0)).getOrDefault("name", "").toString().trim()
102+
)
103+
)
76104
.collect(Collectors.toMap(
77-
uf -> ((Map) ((List) ((Map) uf.get("expression")).get("variables")).get(0)).getOrDefault("value", "").toString(),
78-
uf -> uf.getOrDefault("name", "").toString()
105+
uf -> {
106+
log.trace("Got composite constant entry: {}", uf);
107+
108+
// Get UF variable names-to-vela variables mapping
109+
final Map<String,String> varBindings = new HashMap<>();
110+
((List) ((Map) uf.get("expression")).get("variables")).forEach(o -> {
111+
if (o instanceof Map map) {
112+
String varName = map.getOrDefault("name", "").toString();
113+
String varValue = map.getOrDefault("value", "").toString();
114+
varBindings.put(varName, varValue);
115+
}
116+
});
117+
log.trace("Composite constant bindings: {}", varBindings);
118+
119+
// Rename formula variables
120+
String formula = ((Map) uf.get("expression")).getOrDefault("formula", "").toString().trim();
121+
log.trace("Composite constant original formula: {}", formula);
122+
@NonNull String newFormula = MathUtil.renameFormulaArguments(formula, varBindings);
123+
log.trace("Composite constant modified formula: {}", newFormula);
124+
125+
return newFormula;
126+
},
127+
uf -> uf.getOrDefault("name", "").toString().trim()
79128
));
80-
if (bindingsMap.isEmpty())
129+
bindingsMap = Map.of(
130+
SIMPLE_BINDING_KEY, simpleBindingsMap,
131+
COMPOSITE_BINDING_KEY, compositeBindingsMap
132+
);
133+
if (simpleBindingsMap.isEmpty())
81134
log.warn("No bindings found in DSL generic message: {}", command.body());
82135
}
83136
} catch (Exception e) {
@@ -91,7 +144,7 @@ String extractBindings(Command command, String appId) throws IOException {
91144
storeToFile(bindingsFile, objectMapper.writeValueAsString(bindingsMap));
92145
log.info("Stored bindings in file: app-id={}, file={}", appId, bindingsFile);
93146

94-
// Add appId-modelFile entry in the stored Index
147+
// Add appId-bindingsMap entry in the stored Index
95148
indexService.storeToIndex(appId, Map.of(BINDINGS_FILE_KEY, bindingsFile));
96149

97150
return "OK";
@@ -125,12 +178,65 @@ String extractOptimiserMetrics(Command command, String appId) throws IOException
125178
storeToFile(metricsFile, objectMapper.writeValueAsString(metricsList));
126179
log.info("Stored metrics in file: app-id={}, file={}", appId, metricsFile);
127180

128-
// Add appId-modelFile entry in the stored Index
181+
// Add appId-metricsList entry in the stored Index
129182
indexService.storeToIndex(appId, Map.of(OPTIMISER_METRICS_FILE_KEY, metricsFile));
130183

131184
return "OK";
132185
}
133186

187+
String extractSolution(Command command, String appId) throws IOException {
188+
// Process Optimiser Metrics message
189+
log.debug("Received a new Solution message from external broker: {}", command.body());
190+
191+
// Extract Optimizer metrics
192+
boolean deployFlag;
193+
Map<String,Double> varValues;
194+
try {
195+
deployFlag = Boolean.parseBoolean(command.body().getOrDefault("DeploySolution", "false").toString());
196+
Map<String,Object> map = (Map) command.body().get("VariableValues");
197+
if (map==null || map.isEmpty()) {
198+
log.warn("No VariableValues found in Solution message: {}", command.body());
199+
return "ERROR: No VariableValues found in Solution message: "+command.body();
200+
} else {
201+
Map<String, Object> varValuesObj = map.entrySet().stream()
202+
.filter(e -> StringUtils.isNotBlank(e.getKey()))
203+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
204+
if (varValuesObj.isEmpty()) {
205+
log.warn("Blank VariableValues found in Solution message: {}", command.body());
206+
return "ERROR: Blank VariableValues found in Solution message: "+command.body();
207+
}
208+
209+
// Update variable values
210+
varValues = varValuesObj.entrySet().stream()
211+
.filter(e -> StringUtils.isNotBlank(e.getKey()))
212+
.filter(e -> e.getValue()!=null)
213+
.filter(e -> e.getValue() instanceof Number)
214+
.collect(Collectors.toMap(
215+
Map.Entry::getKey,
216+
e->((Number)e.getValue()).doubleValue()
217+
));
218+
}
219+
} catch (Exception e) {
220+
log.warn("Error while extracting VariableValues from Solution message: ", e);
221+
return "ERROR: Error while extracting VariableValues from Solution message: "+e.getMessage();
222+
}
223+
224+
// Get previous application (UF) variable values (solution)
225+
Map<String, Double> appVariableValues = allVariableValues.computeIfAbsent(appId, app_id -> new HashMap<>());
226+
if (appVariableValues.isEmpty() || deployFlag)
227+
appVariableValues.putAll(varValues);
228+
229+
// Store app solution in models store
230+
String solutionFile = getFileName("sol", appId, "json");
231+
storeToFile(solutionFile, objectMapper.writeValueAsString(appVariableValues));
232+
log.info("Stored solution in file: app-id={}, file={}", appId, solutionFile);
233+
234+
// Add appId-solutionFile entry in the stored Index
235+
indexService.storeToIndex(appId, Map.of(SOLUTIONS_FILE_KEY, solutionFile));
236+
237+
return "OK";
238+
}
239+
134240
String processMetricModelMessage(Command command, String appId) throws IOException {
135241
// Process metric model message
136242
log.debug("Received a new Metric Model message from external broker: {}", command.body());

0 commit comments

Comments
 (0)