From 2ab28fb5e380bfc33465efb5dccba8038148b18d Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 6 Oct 2023 22:59:38 +0300 Subject: [PATCH 001/132] RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. --- .../DeviceManagementController.java | 41 +- .../freebees_webdesign_6/device-view.html | 470 ++++++++++++++++++ .../static/freebees_webdesign_6/devices.html | 38 +- .../freebees_webdesign_6/request-edit.html | 2 +- 4 files changed, 532 insertions(+), 19 deletions(-) create mode 100644 management/src/main/resources/static/freebees_webdesign_6/device-view.html diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index de33edb..34caf74 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -2,14 +2,14 @@ import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; -import eu.nebulous.resource.discovery.monitor.service.DeviceConversionService; import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; -import eu.nebulous.resource.discovery.registration.IRegistrationRequestProcessor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -21,10 +21,29 @@ @PreAuthorize("hasAuthority('ROLE_ADMIN')") public class DeviceManagementController { private final DeviceManagementService deviceService; - private final DeviceConversionService deviceConversionService; - private final IRegistrationRequestProcessor deviceRequestProcessor; - @GetMapping(value = { "/device", "/device/all" }, produces = MediaType.APPLICATION_JSON_VALUE) + private boolean isAuthenticated(Authentication authentication) { + return authentication!=null && StringUtils.isNotBlank(authentication.getName()); + } + + private boolean isAdmin(Authentication authentication) { + if (isAuthenticated(authentication)) { + return authentication.getAuthorities().stream() + .map(GrantedAuthority::getAuthority) + .anyMatch("ROLE_ADMIN"::equals); + } + return false; + } + + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @GetMapping(value = "/device", produces = MediaType.APPLICATION_JSON_VALUE) + public List listDevicesUser(Authentication authentication) { + return isAuthenticated(authentication) + ? deviceService.getByOwner(authentication.getName().trim()) + : listDevicesAll(); + } + + @GetMapping(value = "/device/all", produces = MediaType.APPLICATION_JSON_VALUE) public List listDevicesAll() { return deviceService.getAll(); } @@ -34,10 +53,16 @@ public List listDevicesForOwner(@PathVariable String owner) { return deviceService.getByOwner(owner); } + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/device/{id}", produces = MediaType.APPLICATION_JSON_VALUE) - public Device getDevice(@PathVariable String id) { - return deviceService.getById(id) - .orElseThrow(() -> new DeviceException("Not found device with id: "+id)); + public Device getDevice(@PathVariable String id, Authentication authentication) { + Device device = deviceService.getById(id) + .orElseThrow(() -> new DeviceException("Not found device with id: " + id)); + if (isAuthenticated(authentication) + && ! authentication.getName().trim().equals(device.getOwner()) + && ! isAdmin(authentication)) + throw new DeviceException("Cannot retrieve device with id: " + id); + return device; } @GetMapping(value = "/device/ipaddress/{ipAddress}", produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/management/src/main/resources/static/freebees_webdesign_6/device-view.html b/management/src/main/resources/static/freebees_webdesign_6/device-view.html new file mode 100644 index 0000000..a72adee --- /dev/null +++ b/management/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -0,0 +1,470 @@ + + + + + + + + NebulOuS Resource Discovery - Management page + + + + + + + + + + + + + + + + +
+
+ +
+
+ + +       + + + +       + +
+
+
+ +
+ +
+ +
+

* * * CAUTION: YOU'RE VIEWING A DEVICE YOU DON'T OWN * * *

+ +

Device ---

+ + + +       + +       + +       + +

 

+ +
+
+
Device details
+
+ + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+ +
+
Device metrics
+
+ + ++++ TODO ++++ + + + +
+ +
+ +
+
+
+
+
+
+

Links

+ About Us + Portfolio + Blog + Sing In +
+
+

Get In Touch

+
+
+ + +
+ +
+
+
+

© 2022 YOUR-DOMAIN | Created by Firmbee.com

+ +
+
+
+
+
Fbee 2022 copyright
+ + \ No newline at end of file diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index 4b413c0..4fc8cf2 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -19,12 +19,23 @@ @@ -188,6 +202,12 @@
+
+ + + + +             From 2c6839edfad23ecd6b6b6065a7449dad7d36e3a2 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 12 Oct 2023 15:56:51 +0300 Subject: [PATCH 022/132] RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] --- .../resource/discovery/REQUEST_TYPE.java | 5 + .../DeviceManagementController.java | 16 ++ .../service/DeviceManagementService.java | 164 ++++++++++++++++++ .../RegistrationRequestProcessor.java | 12 +- .../static/freebees_webdesign_6/devices.html | 22 +++ 5 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java b/management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java new file mode 100644 index 0000000..d44f01f --- /dev/null +++ b/management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java @@ -0,0 +1,5 @@ +package eu.nebulous.resource.discovery; + +public enum REQUEST_TYPE { + INSTALL, REINSTALL, UNINSTALL, INFO, DIAGNOSTICS, OTHER +} diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 34caf74..d185dfb 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -89,9 +89,25 @@ public void deleteDevice(@PathVariable String id) { deviceService.deleteById(id); } + @GetMapping(value = "/device/{id}/onboard") + public void onboardDevice(@PathVariable String id) { + deviceService.reinstallRequest(id); + } + + @GetMapping(value = "/device/{id}/offboard") + public void offboardDevice(@PathVariable String id) { + deviceService.uninstallRequest(id); + } + @GetMapping(value = "/device/{id}/archive", produces = MediaType.APPLICATION_JSON_VALUE) public String archiveDevice(@PathVariable String id) { deviceService.archiveDevice(id); return "ARCHIVED"; } + + @GetMapping(value = "/request-update") + public String requestUpdate() { + deviceService.requestInfoUpdate(); + return "REQUESTED-UPDATE"; + } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index acf4faa..5916665 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -1,15 +1,28 @@ package eu.nebulous.resource.discovery.monitor.service; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.nebulous.resource.discovery.REQUEST_TYPE; +import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.ArchivedDevice; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; import eu.nebulous.resource.discovery.monitor.repository.ArchivedDeviceRepository; import eu.nebulous.resource.discovery.monitor.repository.DeviceRepository; +import jakarta.jms.JMSException; +import jakarta.jms.MessageNotWriteableException; +import jakarta.jms.MessageProducer; +import jakarta.jms.Session; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.activemq.ActiveMQConnection; +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.command.ActiveMQMessage; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.apache.activemq.command.ActiveMQTopic; import org.apache.commons.lang3.StringUtils; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import java.time.Instant; @@ -19,9 +32,11 @@ @Service @RequiredArgsConstructor public class DeviceManagementService { + private final ResourceDiscoveryProperties properties; private final DeviceRepository deviceRepository; private final ArchivedDeviceRepository archivedDeviceRepository; private final DeviceConversionService deviceConversionService; + private final ObjectMapper objectMapper; // ------------------------------------------------------------------------ @@ -163,4 +178,153 @@ public void unarchiveDevice(String id) { deviceRepository.save(deviceConversionService.toDevice(result.get())); archivedDeviceRepository.deleteById(result.get().getId()); } + + // ------------------------------------------------------------------------ + + public void reinstallRequest(String id) { + log.trace("reinstallRequest: BEGIN: device-id {}", id); + Optional result = getById(id); + if (result.isEmpty()) + throw new DeviceException( + "Device with the Id does not exists in repository: " + id); + Device device = result.get(); + + try { + // Prepare request + log.debug("reinstallRequest: Requesting device re-onboarding with Id: {}", device.getId()); + Map onboardingRequest = prepareRequestPayload(REQUEST_TYPE.REINSTALL, device); + String jsonMessage = objectMapper.writer().writeValueAsString(onboardingRequest); + + // Connect to Message broker + Pair connAndProducer = connectToBroker(); + + // Send request + connAndProducer.getSecond().send(createMessage(jsonMessage)); + device.setStatus(DeviceStatus.ONBOARDING); + + // Close connection to Message broker + connAndProducer.getFirst().close(); + + log.debug("reinstallRequest: Save updated device: id={}, device={}", device.getId(), device); + update(device); + log.debug("reinstallRequest: Onboarding request sent for device with Id: {}", device.getId()); + } catch (Exception e) { + log.warn("reinstallRequest: EXCEPTION while sending onboarding request for device with Id: {}\n", device.getId(), e); + device.setStatus(DeviceStatus.ONBOARD_ERROR); + device.getMessages().add("EXCEPTION "+e.getMessage()); + update(device); + } + + log.trace("reinstallRequest: END"); + } + + public void uninstallRequest(String id) { + log.trace("uninstallRequest: BEGIN: device-id {}", id); + Optional result = getById(id); + if (result.isEmpty()) + throw new DeviceException( + "Device with the Id does not exists in repository: " + id); + Device device = result.get(); + + try { + // Prepare request + log.debug("uninstallRequest: Requesting device off-onboarding with Id: {}", device.getId()); + Map offboardingRequest = prepareRequestPayload(REQUEST_TYPE.UNINSTALL, device); + String jsonMessage = objectMapper.writer().writeValueAsString(offboardingRequest); + + // Connect to Message broker + Pair connAndProducer = connectToBroker(); + + // Send request + connAndProducer.getSecond().send(createMessage(jsonMessage)); + device.setStatus(DeviceStatus.OFFBOARDING); + + // Close connection to Message broker + connAndProducer.getFirst().close(); + + log.debug("uninstallRequest: Save updated device: id={}, device={}", device.getId(), device); + update(device); + log.debug("uninstallRequest: Off-boarding request sent for device with Id: {}", device.getId()); + } catch (Exception e) { + log.warn("uninstallRequest: EXCEPTION while sending off-boarding request for device with Id: {}\n", device.getId(), e); + device.setStatus(DeviceStatus.OFFBOARD_ERROR); + device.getMessages().add("EXCEPTION "+e.getMessage()); + update(device); + } + + log.trace("uninstallRequest: END"); + } + + public void requestInfoUpdate() { + try { + // Prepare request + log.debug("requestInfoUpdate: Requesting device info and metrics update"); + Map updateRequest = prepareRequestPayload(REQUEST_TYPE.INFO, null); + String jsonMessage = objectMapper.writer().writeValueAsString(updateRequest); + + // Connect to Message broker + Pair connAndProducer = connectToBroker(); + + // Send request + connAndProducer.getSecond().send(createMessage(jsonMessage)); + + // Close connection to Message broker + connAndProducer.getFirst().close(); + + log.debug("requestInfoUpdate: Update request sent"); + } catch (Exception e) { + log.warn("requestInfoUpdate: EXCEPTION while sending update request:\n", e); + } + log.trace("requestInfoUpdate: END"); + } + + // ------------------------------------------------------------------------ + + private static Map prepareRequestPayload(@NonNull REQUEST_TYPE requestType, Device device) { + try { + Map payload; + if (device==null) { + payload = new LinkedHashMap<>(Map.of( + "requestType", requestType.name() + )); + } else { + payload = new LinkedHashMap<>(Map.of( + "requestId", device.getRequestId(), + "requestType", requestType.name(), + "deviceId", device.getId(), + "deviceOs", device.getOs(), + "deviceName", device.getName(), + "deviceIpAddress", device.getIpAddress(), + "deviceUsername", device.getUsername(), + "devicePassword", new String(device.getPassword()), + "devicePublicKey", new String(device.getPublicKey()) + )); + } + payload.put("timestamp", Long.toString(Instant.now().toEpochMilli())); + payload.put("priority", Double.toString(1.0)); + payload.put("retry", Integer.toString(1)); + return payload; + } catch (Exception e) { + log.error("prepareRequestPayload: EXCEPTION: request-type={}, device={}\nException: ", + requestType, device, e); + throw e; + } + } + + protected Pair connectToBroker() throws JMSException { + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( + properties.getBrokerUsername(), properties.getBrokerPassword(), + properties.getBrokerURL()); + ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); + Session session = conn.createSession(); + MessageProducer producer = session.createProducer( + new ActiveMQTopic(properties.getDataCollectionRequestTopic())); + return Pair.of(conn, producer); + } + + protected ActiveMQMessage createMessage(String message) throws MessageNotWriteableException { + ActiveMQTextMessage textMessage = new ActiveMQTextMessage(); + textMessage.setText(message); + return textMessage; + } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index a77c5f6..0d3535b 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.nebulous.resource.discovery.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; @@ -42,9 +43,6 @@ @EnableScheduling @RequiredArgsConstructor public class RegistrationRequestProcessor implements IRegistrationRequestProcessor, InitializingBean, MessageListener { - private static final String REQUEST_TYPE_DATA_COLLECTION = "DIAGNOSTICS"; // EMS task type for collecting node info - private static final String REQUEST_TYPE_ONBOARDING = "VM"; // EMS task type for installing EMS client - private final static List STATUSES_TO_ARCHIVE = List.of( RegistrationRequestStatus.PRE_AUTHORIZATION_REJECT, RegistrationRequestStatus.PRE_AUTHORIZATION_ERROR, @@ -135,7 +133,7 @@ private void processNewRequests(MessageProducer producer) { for (RegistrationRequest registrationRequest : newRequests) { try { log.debug("processNewRequests: Requesting collection of device data for request with Id: {}", registrationRequest.getId()); - Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE_DATA_COLLECTION, registrationRequest); + Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE.DIAGNOSTICS, registrationRequest); String jsonMessage = objectMapper.writer().writeValueAsString(dataCollectionRequest); producer.send(createMessage(jsonMessage)); registrationRequest.setStatus(RegistrationRequestStatus.DATA_COLLECTION_REQUESTED); @@ -170,7 +168,7 @@ private void processOnboardingRequests(MessageProducer producer) { deviceManagementService.checkDevice(deviceForMonitoring, true); log.debug("processOnboardingRequests: Requesting device onboarding for request with Id: {}", registrationRequest.getId()); - Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE_ONBOARDING, registrationRequest); + Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE.INSTALL, registrationRequest); String jsonMessage = objectMapper.writer().writeValueAsString(dataCollectionRequest); producer.send(createMessage(jsonMessage)); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_REQUESTED); @@ -189,11 +187,11 @@ private void processOnboardingRequests(MessageProducer producer) { log.trace("processOnboardingRequests: END"); } - private static Map prepareRequestPayload(@NonNull String requestType, RegistrationRequest registrationRequest) { + private static Map prepareRequestPayload(@NonNull REQUEST_TYPE requestType, RegistrationRequest registrationRequest) { try { Map payload = new LinkedHashMap<>(Map.of( "requestId", registrationRequest.getId(), - "requestType", requestType, + "requestType", requestType.name(), "deviceId", registrationRequest.getDevice().getId(), "deviceOs", registrationRequest.getDevice().getOs(), "deviceName", registrationRequest.getDevice().getName(), diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index b24add1..2389ab2 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -139,6 +139,25 @@ }) ; } + +function requestUpdate() { + $.ajax({ + url: '/monitor/request-update', + method: 'GET', + async: 'true' + + }) + .done(function(data, status) { + console.log('requestUpdate: OK'); + }) + .fail(function(xhr, status, error) { + console.error('requestUpdate: ERROR: ', status, error); + }) + .always(function(data, status) { + updateDevicesList(); + }) + ; +} @@ -183,6 +202,9 @@

Device Management

      +      
-
+ +

Archived Registration Requests

@@ -178,6 +254,50 @@

Archived Registration Requests

+ +
+
+
+

Archived Devices

+ + + +       +       + +       + + +
+ + +
+ + + + + + + + + + + + + + +
#OwnerDevice nameIP AddressReg. DateStatusActions
+
+ +
+
+
diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index 2389ab2..f0f1136 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -59,15 +59,24 @@ var status = `${ ((item.statusUpdate && item.statusUpdate.state) ? item.statusUpdate.state : 'na') }
(${item.status}) `; var color = getStatusColor(status); + var isOffboarded = item.status==='OFFBOARDED' || item.status==='OFFBOARD_ERROR'; - var adminActions = (isAdmin) ? ` + var adminActions = ''; + if (isAdmin) { + adminActions = (! isOffboarded) + ? ` - ` : ''; + ` : ` + + `; + } ii++; tbody.append( $(` @@ -78,10 +87,10 @@ ${load} ${status} - ${adminActions} + ${adminActions} ` ) ); @@ -158,6 +167,20 @@ }) ; } + +function archiveDevice(devId) { + $.ajax({ + url: `/monitor/device/${devId}/archive`, + dataType: 'text' + }) + .done(function(data, status) { + console.log('archiveDevice: OK: ', devId, data); + updateDevicesList(true); + }) + .fail(function(xhr, status, error) { + console.error('archiveDevice: ERROR: ', status, error); + }); +} From 546fa0d9df342486ec14f04c2536e1746a7040d2 Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 13 Oct 2023 16:25:03 +0300 Subject: [PATCH 025/132] RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. --- .../ResourceDiscoveryProperties.java | 2 + .../discovery/monitor/DeviceProcessor.java | 96 ++++++++++++++++++- .../DeviceManagementController.java | 45 +++++---- .../DeviceLifeCycleResponseService.java | 2 +- .../RegistrationRequestProcessor.java | 10 +- .../static/freebees_webdesign_6/devices.html | 18 ++++ 6 files changed, 150 insertions(+), 23 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index d8b2345..35247b1 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -51,6 +51,8 @@ public class ResourceDiscoveryProperties { // Archiving settings private boolean automaticArchivingEnabled; private long archivingThreshold; // in minutes + private boolean immediatelyArchiveSuccessRequests = true; + private boolean immediatelyArchiveOffboardedDevices = true; // Users private List users; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 3a97f7d..21cfac1 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -1,14 +1,104 @@ package eu.nebulous.resource.discovery.monitor; +import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.monitor.model.Device; +import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; +import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.stereotype.Service; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + @Slf4j @Service -//@EnableAsync -//@EnableScheduling +@EnableAsync +@EnableScheduling @RequiredArgsConstructor -public class DeviceProcessor { +public class DeviceProcessor implements InitializingBean { + private final static List STATUSES_TO_ARCHIVE = List.of( + DeviceStatus.OFFBOARDED, + DeviceStatus.OFFBOARD_ERROR + ); + + private final ResourceDiscoveryProperties processorProperties; + private final DeviceManagementService deviceManagementService; + private final TaskScheduler taskScheduler; + private final AtomicBoolean isRunning = new AtomicBoolean(false); + + @Override + public void afterPropertiesSet() throws Exception { + // Initialize periodic device processing + if (processorProperties.isEnablePeriodicProcessing()) { + Instant firstRun; + taskScheduler.scheduleAtFixedRate(this::processRequests, + firstRun = Instant.now().plusSeconds(processorProperties.getProcessingStartupDelay()), + Duration.ofSeconds(processorProperties.getProcessingPeriod())); + log.info("DeviceProcessor: Started periodic device processing: period={}s, first-run-at={}", + processorProperties.getProcessingPeriod(), firstRun.atZone(ZoneId.systemDefault())); + } else { + log.info("DeviceProcessor: Periodic device processing is disabled. You can still invoke it through GUI"); + } + } + + public Future processRequests() { + try { + // Check and set if already running + if (!isRunning.compareAndSet(false, true)) { + log.warn("processRequests: Already running"); + return CompletableFuture.completedFuture("ALREADY RUNNING"); + } + log.debug("processRequests: Processing devices"); + + // Process requests + try { + if (processorProperties.isAutomaticArchivingEnabled()) + archiveDevices(); + } catch (Throwable t) { + log.error("processRequests: ERROR while processing devices: ", t); + } + + log.debug("processRequests: Processing completed"); + + return CompletableFuture.completedFuture("DONE"); + } catch (Throwable e) { + log.error("processRequests: EXCEPTION: ", e); + return CompletableFuture.completedFuture("ERROR: "+e.getMessage()); + } finally { + // Clear running flag + isRunning.set(false); + } + } + + private void archiveDevices() { + Instant archiveThreshold = Instant.now().minus(processorProperties.getArchivingThreshold(), ChronoUnit.MINUTES); + log.trace("archiveDevices: BEGIN: archive-threshold: {}", archiveThreshold); + List devicesForArchiving = deviceManagementService.getAll().stream() + .filter(r -> STATUSES_TO_ARCHIVE.contains(r.getStatus())) + .filter(r -> r.getLastUpdateDate().isBefore(archiveThreshold)) + .toList(); + + log.debug("archiveDevices: Found {} devices for archiving: {}", + devicesForArchiving.size(), devicesForArchiving.stream().map(Device::getId).toList()); + + for (Device device : devicesForArchiving) { + log.debug("archiveDevices: Archiving request with Id: {}", device.getId()); + deviceManagementService.archiveRequestBySystem(device.getId()); + log.info("archiveDevices: Archived request with Id: {}", device.getId()); + } + + log.trace("archiveDevices: END"); + } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 6aa615b..b0c41d7 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.monitor.controller; +import eu.nebulous.resource.discovery.monitor.DeviceProcessor; import eu.nebulous.resource.discovery.monitor.model.ArchivedDevice; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; @@ -18,6 +19,8 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; @Slf4j @RestController @@ -25,6 +28,7 @@ @RequestMapping("/monitor") @PreAuthorize("hasAuthority('ROLE_ADMIN')") public class DeviceManagementController { + private final DeviceProcessor deviceProcessor; private final DeviceManagementService deviceService; private final DeviceLifeCycleRequestService deviceLifeCycleRequestService; @@ -97,6 +101,29 @@ public void deleteDevice(@PathVariable String id) { // ------------------------------------------------------------------------ + @GetMapping(value = "/device/{id}/onboard") + public void onboardDevice(@PathVariable String id) { + deviceLifeCycleRequestService.reinstallRequest(id); + } + + @GetMapping(value = "/device/{id}/offboard") + public void offboardDevice(@PathVariable String id) { + deviceLifeCycleRequestService.uninstallRequest(id); + } + + @GetMapping(value = "/request-update") + public String requestUpdate() { + deviceLifeCycleRequestService.requestInfoUpdate(); + return "REQUESTED-UPDATE"; + } + + @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @GetMapping(value = "/device/process") + public Map processDevices() throws ExecutionException, InterruptedException { + Future future = deviceProcessor.processRequests(); + return Map.of("result", future.isDone() ? future.get() : "STARTED"); + } + @PreAuthorize("hasAuthority('ROLE_ADMIN')") @GetMapping(value = "/device/{id}/archive", produces = MediaType.APPLICATION_JSON_VALUE) public String archiveDevice(@PathVariable String id) { @@ -132,24 +159,6 @@ public ArchivedDevice getArchivedRequest(@PathVariable String id, Authentication // ------------------------------------------------------------------------ - @GetMapping(value = "/device/{id}/onboard") - public void onboardDevice(@PathVariable String id) { - deviceLifeCycleRequestService.reinstallRequest(id); - } - - @GetMapping(value = "/device/{id}/offboard") - public void offboardDevice(@PathVariable String id) { - deviceLifeCycleRequestService.uninstallRequest(id); - } - - @GetMapping(value = "/request-update") - public String requestUpdate() { - deviceLifeCycleRequestService.requestInfoUpdate(); - return "REQUESTED-UPDATE"; - } - - // ------------------------------------------------------------------------ - @ResponseStatus(value = HttpStatus.BAD_REQUEST) @ExceptionHandler(DeviceException.class) public Map handleRegistrationRequestException(DeviceException exception) { diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java index 8d2d20a..7b76f13 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java @@ -124,7 +124,7 @@ private void processUninstallMessage(Map responseMap, Device dev newStatus, requestType, status, requestId, deviceId, ipAddress, reference, device.getNodeReference()); // Archive device, if successfully off-boarded - if (newStatus==DeviceStatus.OFFBOARDED) { + if (newStatus==DeviceStatus.OFFBOARDED && monitorProperties.isImmediatelyArchiveOffboardedDevices()) { deviceManagementService.archiveDevice(device.getId()); log.debug("DeviceLifeCycleResponseService: processUninstallMessage: Device ARCHIVED: id={}, ip-address={}, reference={}", device.getId(), device.getIpAddress(), device.getNodeReference()); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 0d3535b..de2138a 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -331,6 +331,7 @@ private void processResponse(@NonNull Map response) { return; } + boolean doArchive = false; Object obj = response.get("nodeInfo"); if (obj instanceof Map devInfo) { // Update request info @@ -365,8 +366,10 @@ private void processResponse(@NonNull Map response) { // Set new status if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); - if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) + if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); + doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); + } log.debug("processResponse: Done processing response for request: id={}, timestamp={}", requestId, timestamp); } else { @@ -387,6 +390,11 @@ private void processResponse(@NonNull Map response) { // Store changes log.debug("processResponse: Save updated request: id={}, request={}", requestId, registrationRequest); registrationRequestService.update(registrationRequest, false, true); + + // Archive success requests + if (doArchive) { + registrationRequestService.archiveRequestBySystem(registrationRequest.getId()); + } } else { log.warn("processResponse: Request not found: id={}", requestId); } diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index f0f1136..610a170 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -168,6 +168,20 @@ ; } +function processDevices() { + $.ajax({ url: '/monitor/device/process' }) + .done(function(data, status) { + //console.log('processDevices: OK: ', data); + }) + .fail(function(xhr, status, error) { + console.error('processDevices: ERROR: ', status, error); + }) + .always(function(data, status) { + setTimeout(updateDevicesList, 500); + }) + ; +} + function archiveDevice(devId) { $.ajax({ url: `/monitor/device/${devId}/archive`, @@ -228,6 +242,10 @@

Device Management

+       +       - ` : ` + ` : ''; + adminActions += + ` From 23f5830f44b5fb85fce42748289d03a8b4c6174b Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 13 Oct 2023 17:09:45 +0300 Subject: [PATCH 027/132] RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html --- .../archived-device-view.html | 414 ++++++++++++++++++ ...d-view.html => archived-request-view.html} | 0 .../static/freebees_webdesign_6/archived.html | 8 +- 3 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html rename management/src/main/resources/static/freebees_webdesign_6/{archived-view.html => archived-request-view.html} (100%) diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html new file mode 100644 index 0000000..3cd9680 --- /dev/null +++ b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html @@ -0,0 +1,414 @@ + + + + + + + + NebulOuS Resource Discovery - Management page + + + + + + + + + + + + + + + + +
+
+ +
+
+ + +       + + + +       + +
+
+
+ +
+ +
+ +
+

* * * CAUTION: YOU'RE VIEWING A DEVICE YOU DON'T OWN * * *

+ +

Device ---

+ + + +       + +       + +

 

+ +
+
+ +
+
+
Device details
+
+ + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+ + + +
+ +
+ +
+
+
Device metrics
+
+ +
+ +
+
+ +
+ +
+
+
+
+
+
+

Links

+ About Us + Portfolio + Blog + Sing In +
+
+

Get In Touch

+
+
+ + +
+ +
+
+
+

© 2022 YOUR-DOMAIN | Created by Firmbee.com

+ +
+
+
+
+
Fbee 2022 copyright
+ + \ No newline at end of file diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/archived-view.html rename to management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived.html b/management/src/main/resources/static/freebees_webdesign_6/archived.html index 3a96b47..4c3498c 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived.html @@ -64,13 +64,13 @@ ${ii} ${requester} - ${devName} + ${devName} ${ipAddress} ${dateStr} ${status} - ${adminActions} @@ -143,13 +143,13 @@ ${ii} ${owner} - ${devName} + ${devName} ${ipAddress} ${dateStr} ${status} - ${adminActions} From 287bf41fe96bfa7d67ba15d816fa7d50b827b038 Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 13 Oct 2023 20:20:15 +0300 Subject: [PATCH 028/132] RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. --- .../DeviceManagementController.java | 9 +++-- .../service/DeviceMetricsMonitorService.java | 2 + .../static/freebees_webdesign_6/archived.html | 16 ++++++-- .../static/freebees_webdesign_6/devices.html | 38 +++++++++---------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index b0c41d7..6a1ebaf 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -101,37 +101,37 @@ public void deleteDevice(@PathVariable String id) { // ------------------------------------------------------------------------ + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/device/{id}/onboard") public void onboardDevice(@PathVariable String id) { deviceLifeCycleRequestService.reinstallRequest(id); } + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/device/{id}/offboard") public void offboardDevice(@PathVariable String id) { deviceLifeCycleRequestService.uninstallRequest(id); } + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/request-update") public String requestUpdate() { deviceLifeCycleRequestService.requestInfoUpdate(); return "REQUESTED-UPDATE"; } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") @GetMapping(value = "/device/process") public Map processDevices() throws ExecutionException, InterruptedException { Future future = deviceProcessor.processRequests(); return Map.of("result", future.isDone() ? future.get() : "STARTED"); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") @GetMapping(value = "/device/{id}/archive", produces = MediaType.APPLICATION_JSON_VALUE) public String archiveDevice(@PathVariable String id) { deviceService.archiveDevice(id); return "ARCHIVED"; } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") @GetMapping(value = "/device/{id}/unarchive", produces = MediaType.APPLICATION_JSON_VALUE) public String unarchiveDevice(@PathVariable String id) { deviceService.unarchiveDevice(id); @@ -140,17 +140,18 @@ public String unarchiveDevice(@PathVariable String id) { // ------------------------------------------------------------------------ + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/device/archived", produces = MediaType.APPLICATION_JSON_VALUE) public List listArchivedRequests(Authentication authentication) { return deviceService.getArchivedByOwner(authentication); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") @GetMapping(value = "/device/archived/all", produces = MediaType.APPLICATION_JSON_VALUE) public List listArchivedRequestsAdmin() { return deviceService.getArchivedAll(); } + @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") @GetMapping(value = "/device/archived/{id}", produces = MediaType.APPLICATION_JSON_VALUE) public ArchivedDevice getArchivedRequest(@PathVariable String id, Authentication authentication) { return deviceService.getArchivedById(id, authentication) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 643dd38..615d8aa 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -5,6 +5,7 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceMetrics; +import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -106,6 +107,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Update device data device.setMetrics(metrics); + device.setStatus(DeviceStatus.ONBOARDED); deviceManagementService.update(device); log.debug("DeviceMetricsMonitorService: Device metrics updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), metrics); diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived.html b/management/src/main/resources/static/freebees_webdesign_6/archived.html index 4c3498c..425d0aa 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived.html @@ -52,7 +52,7 @@ var dateStr = date.toLocaleDateString('en-GB') + ' ' + date.toLocaleTimeString('en-GB') + '
' + Intl.DateTimeFormat().resolvedOptions().timeZone; var status = item.status; - var color = getStatusColor(status); + var color = getRequestStatusColor(status); var adminActions = (isAdmin) ? ` + + `; + var adminActions = (isAdmin) ? ` - - - ` : ''; - adminActions += - ` - `; - } + ` : ''; ii++; tbody.append( $(` @@ -92,6 +90,7 @@ + ${userActions} ${adminActions} ` @@ -124,11 +123,10 @@ function getStatusColor(status) { if (status.indexOf('ERROR')>0) return 'table-danger'; - if (status.indexOf('REJECT')>0) return 'bg-danger'; - if (status.indexOf('PENDING')>=0) return 'table-warning'; + if (status.indexOf('BOARDING')>0) return 'table-warning'; if (status=='NEW_DEVICE') return ''; - if (status=='BUSY') return 'table-success'; - if (status=='IDLE') return 'table-warning'; + if (status=='ONBOARDED') return 'table-success'; + if (status=='OFFBOARDED') return 'table-secondary'; return 'table-info'; } @@ -241,7 +239,7 @@

Device Management

      -       From 550da0afa35873420b535d2bfb9f7ecdf2c27f87 Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 13 Oct 2023 21:28:38 +0300 Subject: [PATCH 029/132] RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. --- .../ResourceDiscoveryProperties.java | 5 ++ .../discovery/monitor/DeviceProcessor.java | 77 ++++++++++++++++--- .../DeviceManagementController.java | 2 +- .../discovery/monitor/model/Device.java | 8 +- .../discovery/monitor/model/DeviceStatus.java | 3 +- .../service/AbstractMonitorService.java | 10 ++- .../service/DeviceManagementService.java | 12 +-- .../service/DeviceMetricsMonitorService.java | 2 +- .../service/DeviceStatusMonitorService.java | 1 + .../static/freebees_webdesign_6/archived.html | 10 ++- .../static/freebees_webdesign_6/devices.html | 7 +- 11 files changed, 109 insertions(+), 28 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 35247b1..0f5f84f 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -48,6 +48,11 @@ public class ResourceDiscoveryProperties { private String deviceLifeCycleRequestsTopic = "ems.client.installation.requests"; private String deviceLifeCycleResponsesTopic = "ems.client.installation.reports"; + // Failed devices detection + private boolean automaticFailedDetection = true; + private long suspectDeviceThreshold = 5; // in minutes + private long failedDeviceThreshold = 10; // in minutes + // Archiving settings private boolean automaticArchivingEnabled; private long archivingThreshold; // in minutes diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 21cfac1..054135e 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -28,7 +28,12 @@ @EnableScheduling @RequiredArgsConstructor public class DeviceProcessor implements InitializingBean { + private final static List STATUSES_TO_EXCLUDE_FROM_SUSPECT_CHECK = List.of( + DeviceStatus.ON_HOLD, DeviceStatus.ONBOARDING, DeviceStatus.FAILED, + DeviceStatus.OFFBOARDING, DeviceStatus.OFFBOARDED, DeviceStatus.OFFBOARD_ERROR + ); private final static List STATUSES_TO_ARCHIVE = List.of( + DeviceStatus.FAILED, DeviceStatus.OFFBOARDED, DeviceStatus.OFFBOARD_ERROR ); @@ -40,10 +45,17 @@ public class DeviceProcessor implements InitializingBean { @Override public void afterPropertiesSet() throws Exception { + // Check configuration + Instant suspectDeviceThreshold = Instant.now().minus(processorProperties.getSuspectDeviceThreshold(), ChronoUnit.MINUTES); + Instant failedDeviceThreshold = Instant.now().minus(processorProperties.getFailedDeviceThreshold(), ChronoUnit.MINUTES); + if (suspectDeviceThreshold.isBefore(failedDeviceThreshold)) + throw new IllegalArgumentException("DeviceProcessor: Configuration error: suspectDeviceThreshold is before failedDeviceThreshold: " + + processorProperties.getSuspectDeviceThreshold() + " < " + processorProperties.getFailedDeviceThreshold()); + // Initialize periodic device processing if (processorProperties.isEnablePeriodicProcessing()) { Instant firstRun; - taskScheduler.scheduleAtFixedRate(this::processRequests, + taskScheduler.scheduleAtFixedRate(this::processDevices, firstRun = Instant.now().plusSeconds(processorProperties.getProcessingStartupDelay()), Duration.ofSeconds(processorProperties.getProcessingPeriod())); log.info("DeviceProcessor: Started periodic device processing: period={}s, first-run-at={}", @@ -53,28 +65,30 @@ public void afterPropertiesSet() throws Exception { } } - public Future processRequests() { + public Future processDevices() { try { // Check and set if already running if (!isRunning.compareAndSet(false, true)) { - log.warn("processRequests: Already running"); + log.warn("processDevices: Already running"); return CompletableFuture.completedFuture("ALREADY RUNNING"); } - log.debug("processRequests: Processing devices"); + log.debug("processDevices: Processing devices"); // Process requests try { + if (processorProperties.isAutomaticFailedDetection()) + processFailedDevices(); if (processorProperties.isAutomaticArchivingEnabled()) archiveDevices(); } catch (Throwable t) { - log.error("processRequests: ERROR while processing devices: ", t); + log.error("processDevices: ERROR while processing devices: ", t); } - log.debug("processRequests: Processing completed"); + log.debug("processDevices: Processing completed"); return CompletableFuture.completedFuture("DONE"); } catch (Throwable e) { - log.error("processRequests: EXCEPTION: ", e); + log.error("processDevices: EXCEPTION: ", e); return CompletableFuture.completedFuture("ERROR: "+e.getMessage()); } finally { // Clear running flag @@ -82,6 +96,49 @@ public Future processRequests() { } } + private void processFailedDevices() { + Instant suspectDeviceThreshold = Instant.now().minus(processorProperties.getSuspectDeviceThreshold(), ChronoUnit.MINUTES); + Instant failedDeviceThreshold = Instant.now().minus(processorProperties.getFailedDeviceThreshold(), ChronoUnit.MINUTES); + log.trace("processFailedDevices: BEGIN: suspect-threshold={}, failed-threshold={}", + suspectDeviceThreshold, failedDeviceThreshold); + List suspectDevices = deviceManagementService.getAll().stream() + .filter(r -> ! STATUSES_TO_EXCLUDE_FROM_SUSPECT_CHECK.contains(r.getStatus())) + .filter(r -> r.getStatusUpdate()==null || r.getStatusUpdate().getStateLastUpdate().isBefore(suspectDeviceThreshold)) + .filter(r -> r.getMetrics()==null || r.getMetrics().getTimestamp().isBefore(suspectDeviceThreshold)) + .filter(r -> r.getCreationDate().isBefore(suspectDeviceThreshold)) + .toList(); + + if (log.isDebugEnabled()) + log.debug("processFailedDevices: Found {} suspect devices: {}", + suspectDevices.size(), suspectDevices.stream().map(Device::getId).toList()); + + for (Device device : suspectDevices) { + // Mark device as suspect + log.debug("processFailedDevices: Marking as suspect device with Id: {}", device.getId()); + device.setStatus(DeviceStatus.SUSPECT); + if (device.getSuspectTimestamp()==null) { + device.setSuspectTimestamp(Instant.now()); + device.setRetries(0); + log.info("processFailedDevices: Marked as suspect device with Id: {}", device.getId()); + } else { + device.incrementRetries(); + } + + // If fail threshold exceeded the mark device as PROBLEMATIC + if ( (device.getStatusUpdate()==null || device.getStatusUpdate().getStateLastUpdate().isBefore(failedDeviceThreshold)) + && (device.getMetrics()==null || device.getMetrics().getTimestamp().isBefore(failedDeviceThreshold)) + && device.getCreationDate().isBefore(failedDeviceThreshold) ) + { + device.setStatus(DeviceStatus.FAILED); + log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); + } + + deviceManagementService.update(device); + } + + log.trace("processProblematicDevices: END"); + } + private void archiveDevices() { Instant archiveThreshold = Instant.now().minus(processorProperties.getArchivingThreshold(), ChronoUnit.MINUTES); log.trace("archiveDevices: BEGIN: archive-threshold: {}", archiveThreshold); @@ -94,9 +151,9 @@ private void archiveDevices() { devicesForArchiving.size(), devicesForArchiving.stream().map(Device::getId).toList()); for (Device device : devicesForArchiving) { - log.debug("archiveDevices: Archiving request with Id: {}", device.getId()); - deviceManagementService.archiveRequestBySystem(device.getId()); - log.info("archiveDevices: Archived request with Id: {}", device.getId()); + log.debug("archiveDevices: Archiving device with Id: {}", device.getId()); + deviceManagementService.archiveDeviceBySystem(device.getId()); + log.info("archiveDevices: Archived device with Id: {}", device.getId()); } log.trace("archiveDevices: END"); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 6a1ebaf..5c63d27 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -122,7 +122,7 @@ public String requestUpdate() { @GetMapping(value = "/device/process") public Map processDevices() throws ExecutionException, InterruptedException { - Future future = deviceProcessor.processRequests(); + Future future = deviceProcessor.processDevices(); return Map.of("result", future.isDone() ? future.get() : "STARTED"); } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index 1125c56..bb2e293 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -1,7 +1,6 @@ package eu.nebulous.resource.discovery.monitor.model; import com.fasterxml.jackson.annotation.JsonProperty; -import eu.nebulous.resource.discovery.registration.model.RegistrationRequest; import lombok.*; import lombok.experimental.SuperBuilder; import org.springframework.data.mongodb.core.mapping.Document; @@ -43,4 +42,11 @@ public class Device { private DeviceStatusUpdate statusUpdate; private DeviceMetrics metrics; + + private Instant suspectTimestamp; + private int retries; + + public void incrementRetries() { + retries++; + } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java index 1b45cf7..e124edf 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java @@ -3,6 +3,7 @@ public enum DeviceStatus { NEW_DEVICE, ON_HOLD, ONBOARDING, ONBOARDED, ONBOARD_ERROR, - HEALTHY, BUSY, IDLE, + HEALTHY, SUSPECT, FAILED, + BUSY, IDLE, OFFBOARDING, OFFBOARDED, OFFBOARD_ERROR } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java index bb46c6a..46da18c 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; -import eu.nebulous.resource.discovery.monitor.model.DeviceStatusUpdate; +import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; import jakarta.jms.Message; import jakarta.jms.MessageConsumer; import jakarta.jms.MessageListener; @@ -17,7 +17,6 @@ import org.apache.activemq.ActiveMQConnectionFactory; import org.apache.activemq.command.ActiveMQTextMessage; import org.apache.activemq.command.ActiveMQTopic; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.annotation.EnableAsync; @@ -27,7 +26,6 @@ import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.Optional; @Slf4j @Service @@ -88,5 +86,11 @@ public void onMessage(Message message) { } } + public void setHealthyStatus(Device device) { + device.setStatus(DeviceStatus.HEALTHY); + device.setSuspectTimestamp(null); + device.setRetries(0); + } + protected abstract void processPayload(Map dataMap); } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index 7488e4f..54fd41d 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -134,10 +134,10 @@ private boolean canAccess(@NonNull Device device, Authentication authentication) } private boolean canAccess(@NonNull Device device, Authentication authentication, boolean sameUserOnly) { - String requester = device.getOwner(); - if (requester == null && authentication.getName() == null) return true; - return requester != null && ( - requester.equals(authentication.getName()) || + String owner = device.getOwner(); + if (owner == null && authentication.getName() == null) return true; + return owner != null && ( + owner.equals(authentication.getName()) || !sameUserOnly && authentication.getAuthorities().stream() .map(GrantedAuthority::getAuthority).toList().contains("ROLE_ADMIN") ); @@ -182,10 +182,10 @@ public List getArchivedByIpAddress(@NonNull String ipAddress) { } public void archiveDevice(String id) { - archiveRequestBySystem(id); + archiveDeviceBySystem(id); } - public void archiveRequestBySystem(String id) { + public void archiveDeviceBySystem(String id) { Optional result = getById(id); if (result.isEmpty()) throw new DeviceException( diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 615d8aa..8d7add4 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -107,7 +107,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Update device data device.setMetrics(metrics); - device.setStatus(DeviceStatus.ONBOARDED); + setHealthyStatus(device); deviceManagementService.update(device); log.debug("DeviceMetricsMonitorService: Device metrics updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), metrics); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java index 7ae3382..e7c38c2 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java @@ -84,6 +84,7 @@ private void updateDeviceInfo(@NonNull Map infoMap) { // Update device data device.setStatusUpdate(deviceStatusUpdate); + setHealthyStatus(device); deviceManagementService.update(device); log.debug("DeviceStatusMonitorService: Device status updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), deviceStatusUpdate); diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived.html b/management/src/main/resources/static/freebees_webdesign_6/archived.html index 425d0aa..cc20857 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived.html @@ -96,6 +96,7 @@ function unarchiveRequest(reqId) { if (! confirm('Restore request?')) return; + if (! confirm('You will need to manually provide Device credentials in Registration Requests page')) return; $.ajax({ url: `/discovery/request/${reqId}/unarchive` }) .done(function(data, status) { @@ -165,17 +166,20 @@ } function getDeviceStatusColor(status) { + if (status=='NEW_DEVICE') return ''; if (status.indexOf('ERROR')>0) return 'table-danger'; + if (status=='FAILED') return 'table-danger'; + if (status=='SUSPECT') return 'table-warning'; if (status.indexOf('BOARDING')>0) return 'table-warning'; - if (status=='NEW_DEVICE') return ''; if (status=='ONBOARDED') return 'table-success'; - if (status=='OFFBOARDED') return 'table-secondary'; + if (status=='HEALTHY' || status=='BUSY' || status=='IDLE') return 'table-success'; + if (status=='OFFBOARDED' || status=='ON_HOLD') return 'table-secondary'; return 'table-info'; } function unarchiveDevice(devId) { if (! confirm('Restore device?')) return; - if (! confirm('You will need to manually provide Device credentials in Devices pages')) return; + if (! confirm('You will need to manually provide Device credentials in Devices page')) return; $.ajax({ url: `/monitor/device/${devId}/unarchive`, dataType: 'text' }) .done(function(data, status) { diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index 095daf1..210921d 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -122,11 +122,14 @@ } function getStatusColor(status) { + if (status=='NEW_DEVICE') return ''; if (status.indexOf('ERROR')>0) return 'table-danger'; + if (status=='FAILED') return 'table-danger'; + if (status=='SUSPECT') return 'table-warning'; if (status.indexOf('BOARDING')>0) return 'table-warning'; - if (status=='NEW_DEVICE') return ''; if (status=='ONBOARDED') return 'table-success'; - if (status=='OFFBOARDED') return 'table-secondary'; + if (status=='HEALTHY' || status=='BUSY' || status=='IDLE') return 'table-success'; + if (status=='OFFBOARDED' || status=='ON_HOLD') return 'table-secondary'; return 'table-info'; } From a54736e14dca7cad15a8beebf899760b4a21ea48 Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 13 Oct 2023 23:34:34 +0300 Subject: [PATCH 030/132] RD: Added shortcuts (top-right corner) to all detail pages --- .../static/freebees_webdesign_6/archived-device-view.html | 5 +++++ .../static/freebees_webdesign_6/archived-request-view.html | 5 +++++ .../resources/static/freebees_webdesign_6/device-view.html | 5 +++++ .../resources/static/freebees_webdesign_6/request-edit.html | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html index 3cd9680..69fc81e 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html @@ -191,6 +191,11 @@
+ + + + +             diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html index c3ae664..5a0c767 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html @@ -146,6 +146,11 @@
+ + + + +             diff --git a/management/src/main/resources/static/freebees_webdesign_6/device-view.html b/management/src/main/resources/static/freebees_webdesign_6/device-view.html index 5eaad64..22b111e 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/device-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -304,6 +304,11 @@
+ + + + +             diff --git a/management/src/main/resources/static/freebees_webdesign_6/request-edit.html b/management/src/main/resources/static/freebees_webdesign_6/request-edit.html index bd57c0e..287b56f 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/management/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -307,6 +307,11 @@
+ + + + +             From f1fab9c1366f2e06aa760163154540fdaa0e01ed Mon Sep 17 00:00:00 2001 From: ipatini Date: Sat, 14 Oct 2023 00:19:46 +0300 Subject: [PATCH 031/132] RD: Minor change in index.html (grayed out settings image) --- .../src/main/resources/static/freebees_webdesign_6/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/src/main/resources/static/freebees_webdesign_6/index.html b/management/src/main/resources/static/freebees_webdesign_6/index.html index 640ee83..8f90e19 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/index.html +++ b/management/src/main/resources/static/freebees_webdesign_6/index.html @@ -68,7 +68,7 @@

NebulOuS Resource Discovery Dashboard

More
-
+

Settings

View the Resource Discovery service settings. Admins can also manage the service, From 21be064a31bdac30c65df2debf704f64db395791 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 16 Oct 2023 18:45:28 +0300 Subject: [PATCH 032/132] RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. --- .../service/AbstractMonitorService.java | 59 +++++++++++-------- .../DeviceLifeCycleResponseService.java | 2 +- .../service/DeviceMetricsMonitorService.java | 2 +- .../service/DeviceStatusMonitorService.java | 2 +- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java index 46da18c..bdab2aa 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java @@ -6,10 +6,7 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; -import jakarta.jms.Message; -import jakarta.jms.MessageConsumer; -import jakarta.jms.MessageListener; -import jakarta.jms.Session; +import jakarta.jms.*; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -33,9 +30,12 @@ @EnableScheduling @RequiredArgsConstructor public abstract class AbstractMonitorService implements InitializingBean, MessageListener { + @NonNull protected final String name; protected final ResourceDiscoveryProperties monitorProperties; protected final TaskScheduler taskScheduler; protected final ObjectMapper objectMapper; + protected ActiveMQConnection connection; + protected Session session; @Override public void afterPropertiesSet() throws Exception { @@ -45,21 +45,32 @@ public void afterPropertiesSet() throws Exception { } private void initializeDeviceStatusListener() { - getTopicsToMonitor().forEach(topic -> { - try { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - monitorProperties.getBrokerUsername(), monitorProperties.getBrokerPassword(), monitorProperties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); - Session session = conn.createSession(); - MessageConsumer consumer = session.createConsumer( - new ActiveMQTopic(topic)); - consumer.setMessageListener(this); - conn.start(); - } catch (Exception e) { - log.error("AbstractMonitorService: ERROR while subscribing to Message broker for Device status updates: ", e); - taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); - } - }); + try { + openBrokerConnection(); + getTopicsToMonitor().forEach(topic -> { + try { + MessageConsumer consumer = session.createConsumer(new ActiveMQTopic(topic)); + consumer.setMessageListener(this); + log.debug("{}: Subscribed to topic: {}", name, topic); + } catch (Exception e) { + log.error("{}: ERROR while subscribing to topic: {}\n", name, topic, e); + taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); + } + }); + } catch (JMSException e) { + log.error("{}: ERROR while opening connection to Message broker: ", name, e); + taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); + } + } + + private void openBrokerConnection() throws JMSException { + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( + monitorProperties.getBrokerUsername(), monitorProperties.getBrokerPassword(), monitorProperties.getBrokerURL()); + ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); + Session ses = conn.createSession(); + this.connection = conn; + this.session = ses; + connection.start(); } protected abstract @NonNull List getTopicsToMonitor(); @@ -67,22 +78,22 @@ private void initializeDeviceStatusListener() { @Override public void onMessage(Message message) { try { - log.debug("AbstractMonitorService: Received a JMS message: {}", message); + log.debug("{}: Received a JMS message: {}", name, message); if (message instanceof ActiveMQTextMessage textMessage) { String payload = textMessage.getText(); - log.trace("AbstractMonitorService: Message payload: {}", payload); + log.trace("{}: Message payload: {}", name, payload); TypeReference> typeRef = new TypeReference<>() { }; Object obj = objectMapper.readerFor(typeRef).readValue(payload); if (obj instanceof Map dataMap) { processPayload(dataMap); } else { - log.debug("AbstractMonitorService: Message payload is not recognized. Expected Map but got: type={}, object={}", obj.getClass().getName(), obj); + log.debug("{}: Message payload is not recognized. Expected Map but got: type={}, object={}", name, obj.getClass().getName(), obj); } } else { - log.debug("AbstractMonitorService: Message type is not supported: {}", message); + log.debug("{}: Message type is not supported: {}", name, message); } } catch (Exception e) { - log.warn("AbstractMonitorService: ERROR while processing message: {}\nException: ", message, e); + log.warn("{}: ERROR while processing message: {}\nException: ", name, message, e); } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java index 7b76f13..615f955 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java @@ -23,7 +23,7 @@ public class DeviceLifeCycleResponseService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; public DeviceLifeCycleResponseService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super(monitorProperties, taskScheduler, objectMapper); + super("DeviceLifeCycleResponseService", monitorProperties, taskScheduler, objectMapper); this.deviceManagementService = deviceManagementService; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 8d7add4..9406c3a 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -26,7 +26,7 @@ public class DeviceMetricsMonitorService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super(monitorProperties, taskScheduler, objectMapper); + super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper); this.deviceManagementService = deviceManagementService; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java index e7c38c2..9993755 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java @@ -21,7 +21,7 @@ public class DeviceStatusMonitorService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; public DeviceStatusMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super(monitorProperties, taskScheduler, objectMapper); + super("DeviceStatusMonitorService", monitorProperties, taskScheduler, objectMapper); this.deviceManagementService = deviceManagementService; } From 7cd97280c0cb9e1a30de28f29b51a7485fa47f8f Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 16 Oct 2023 18:46:34 +0300 Subject: [PATCH 033/132] RD: Minor code (mostly logging) and GUI improvements --- .../monitor/controller/ArchivedDeviceManagementController.java | 1 - .../monitor/service/DeviceLifeCycleResponseService.java | 2 +- .../discovery/monitor/service/DeviceMetricsMonitorService.java | 2 +- .../discovery/monitor/service/DeviceStatusMonitorService.java | 2 +- .../discovery/registration/RegistrationRequestProcessor.java | 3 ++- .../main/resources/static/freebees_webdesign_6/devices.html | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java index 5b2724d..b421a6b 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java @@ -6,7 +6,6 @@ import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java index 615f955..5decde7 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java @@ -57,7 +57,7 @@ protected void processPayload(Map dataMap) { // Check if we process the indicated requestType REQUEST_TYPE requestType = REQUEST_TYPE.valueOf(requestTypeStr); if (requestType!=REQUEST_TYPE.REINSTALL && requestType!=REQUEST_TYPE.UNINSTALL) { - log.warn("DeviceLifeCycleResponseService: Ignoring message due to its requestType: requestType={}, status={}, requestId={}, deviceId={}, ipAddress={}, reference={}", + log.debug("DeviceLifeCycleResponseService: Ignoring message due to its requestType: requestType={}, status={}, requestId={}, deviceId={}, ipAddress={}, reference={}", requestType, status, requestId, deviceId, ipAddress, reference); return; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 9406c3a..7e0a471 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -69,7 +69,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Get registered device using IP address Optional result = deviceManagementService.getByIpAddress(ipAddress); if (result.isEmpty()) { - log.warn("DeviceMetricsMonitorService: Device metrics IP address does not match any registered device: {}", infoMap); + log.debug("DeviceMetricsMonitorService: Device metrics IP address does not match any registered device: {}", infoMap); return; } Device device = result.get(); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java index 9993755..bb88d44 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java @@ -54,7 +54,7 @@ private void updateDeviceInfo(@NonNull Map infoMap) { String ipAddress = deviceStatusUpdate.getIpAddress(); Optional result = deviceManagementService.getByIpAddress(ipAddress); if (result.isEmpty()) { - log.warn("DeviceStatusMonitorService: Device status update IP address does not match any registered device: {}", infoMap); + log.debug("DeviceStatusMonitorService: Device status update IP address does not match any registered device: {}", infoMap); return; } Device device = result.get(); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index de2138a..f9ac8f7 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -278,6 +278,7 @@ public void onMessage(Message message) { } private void processResponse(@NonNull Map response) { + String requestType = response.getOrDefault("requestType", "").toString().trim(); String requestId = response.getOrDefault("requestId", "").toString().trim(); String reference = response.getOrDefault("reference", "").toString().trim(); String responseStatus = response.getOrDefault("status", "").toString().trim(); @@ -396,7 +397,7 @@ private void processResponse(@NonNull Map response) { registrationRequestService.archiveRequestBySystem(registrationRequest.getId()); } } else { - log.warn("processResponse: Request not found: id={}", requestId); + log.debug("processResponse: Request not found: id={}, requestType={}", requestId, requestType); } } diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/management/src/main/resources/static/freebees_webdesign_6/devices.html index 210921d..363dffe 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/management/src/main/resources/static/freebees_webdesign_6/devices.html @@ -65,7 +65,7 @@ ? '' : ` `: ''; @@ -94,11 +94,21 @@ return 'table-info'; } -function unarchiveRequest(reqId) { +function unarchiveRequest(reqId, ipAddress) { if (! confirm('Restore request?')) return; - if (! confirm('You will need to manually provide Device credentials in Registration Requests page')) return; + showModal('Request: '+ipAddress+' ['+reqId+']', 'REQUEST', reqId); +} + +function _unarchiveRequest(reqId, credentials) { + //if (! confirm('Restore request?')) return; + //if (! confirm('You will need to manually provide Device credentials in Registration Requests page')) return; - $.ajax({ url: `/discovery/request/${reqId}/unarchive` }) + $.ajax({ + type: 'post', + url: `/discovery/request/${reqId}/unarchive`, + contentType:"application/json; charset=utf-8", + data: credentials + }) .done(function(data, status) { console.log('unarchiveRequest: OK: ', data); updateRequestsList(true); @@ -110,6 +120,51 @@ // ---------------------------------------------------------------------------- +function showModal(info, type, id) { + // Set credentials modal data and clear form + $('#credentialsModal_info').html(info); + $(`[id="device#type"]`).val(type); + $(`[id="device#id"]`).val(id); + + $(`[id="device#username"]`).val(''); + $(`[id="device#password"]`).val(''); + $(`[id="device#publicKey"]`).val(''); + + // Show credentials modal + var modal = new bootstrap.Modal(document.getElementById('credentialsModal'), { + //keyboard: true + }); + modal.show(); +} + +function doRestore() { + // Get credentialsModal form data + var type = $(`[id="device#type"]`).val(); + var id = $(`[id="device#id"]`).val(); + + var username = $(`[id="device#username"]`).val(); + var password = $(`[id="device#password"]`).val(); + var publicKey = $(`[id="device#publicKey"]`).val(); + + // Close credentials modal + var modal = bootstrap.Modal.getInstance(document.getElementById('credentialsModal')); + modal.hide(); + + // Check form input + if (username.trim()==='' || password.trim()==='' && publicKey.trim()==='') { + alert('You must provide username, and either password or SSH key'); + return; + } + + var credentials = JSON.stringify({ username: username, password: password, publicKey: publicKey}); + if (type==='REQUEST') + _unarchiveRequest(id, credentials); + else + _unarchiveDevice(id, credentials); +} + +// ---------------------------------------------------------------------------- + function updateDevicesList(asAdmin) { if (asAdmin === undefined) asAdmin = lastUpdateAsAdmin; else lastUpdateAsAdmin = asAdmin; @@ -134,7 +189,7 @@ var status = item.status; var color = getDeviceStatusColor(status); var adminActions = (isAdmin) ? ` - `: ''; @@ -177,11 +232,22 @@ return 'table-info'; } -function unarchiveDevice(devId) { +function unarchiveDevice(devId, ipAddress) { if (! confirm('Restore device?')) return; - if (! confirm('You will need to manually provide Device credentials in Devices page')) return; + showModal('Device: '+ipAddress+' ['+devId+']', 'DEVICE', devId); +} + +function _unarchiveDevice(devId, credentials) { + //if (! confirm('Restore device?')) return; + //if (! confirm('You will need to manually provide Device credentials in Devices page')) return; - $.ajax({ url: `/monitor/device/${devId}/unarchive`, dataType: 'text' }) + $.ajax({ + type: 'post', + url: `/monitor/device/${devId}/unarchive`, + contentType:"application/json; charset=utf-8", + data: credentials, + dataType: 'text' + }) .done(function(data, status) { console.log('unarchiveDevice: OK: ', data); updateDevicesList(true); @@ -312,6 +378,45 @@

Archived Devices

+ + +
From 151fb1622f253904bdfe0a6ac4cbd2d36c0355d0 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 17 Oct 2023 08:17:35 +0300 Subject: [PATCH 036/132] RD: Moved REQUEST_TYPE enum to 'common' package and updated code --- .../nebulous/resource/discovery/{ => common}/REQUEST_TYPE.java | 2 +- .../monitor/service/DeviceLifeCycleRequestService.java | 2 +- .../monitor/service/DeviceLifeCycleResponseService.java | 2 +- .../monitor/service/UnknownDeviceRegistrationService.java | 2 +- .../discovery/registration/RegistrationRequestProcessor.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename management/src/main/java/eu/nebulous/resource/discovery/{ => common}/REQUEST_TYPE.java (68%) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java b/management/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java similarity index 68% rename from management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java rename to management/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java index 2cf96a5..25bfd39 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/REQUEST_TYPE.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java @@ -1,4 +1,4 @@ -package eu.nebulous.resource.discovery; +package eu.nebulous.resource.discovery.common; public enum REQUEST_TYPE { INSTALL, REINSTALL, UNINSTALL, NODE_DETAILS, INFO, DIAGNOSTICS, OTHER diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java index d11bf94..2b13a9a 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java @@ -1,7 +1,7 @@ package eu.nebulous.resource.discovery.monitor.service; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.REQUEST_TYPE; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java index 5decde7..c6ee3eb 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java @@ -2,7 +2,7 @@ package eu.nebulous.resource.discovery.monitor.service; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.REQUEST_TYPE; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index a124afe..6458ba2 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -2,7 +2,7 @@ package eu.nebulous.resource.discovery.monitor.service; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.REQUEST_TYPE; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index f9ac8f7..0e14383 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.REQUEST_TYPE; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; From ccd667ae147ebb5081a5137f6d521f266a96dbda Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 17 Oct 2023 10:52:51 +0300 Subject: [PATCH 037/132] RD: Changed authentication to use BCrypt-encrypted passwords --- .../resource/discovery/SecurityConfig.java | 38 ++++++++++++------- management/src/main/resources/application.yml | 6 ++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java b/management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java index 07b2216..399e40e 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java @@ -4,18 +4,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; import org.springframework.security.core.userdetails.User; -import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.provisioning.InMemoryUserDetailsManager; import org.springframework.security.web.SecurityFilterChain; -import java.util.ArrayList; -import java.util.List; +import java.security.SecureRandom; import static org.springframework.security.config.Customizer.withDefaults; @@ -39,21 +39,31 @@ public SecurityFilterChain securityFilterChain(HttpSecurity httpSecurity) throws return httpSecurity.build(); } + public void configAuthentication(AuthenticationManagerBuilder auth) throws Exception { + auth.inMemoryAuthentication().passwordEncoder(passwordEncoder()); + } + @Bean - public InMemoryUserDetailsManager userDetailsService() { - List users = new ArrayList<>(); - properties.getUsers().forEach(userData -> { - UserDetails user = User.withUsername(userData.getUsername()) - .password(encoder().encode(userData.getPassword())) - .roles(userData.getRoles().toArray(new String[0])) - .build(); - users.add(user); - }); - return new InMemoryUserDetailsManager(users.toArray(new UserDetails[0])); + public InMemoryUserDetailsManager inMemoryUserDetailsManager() { + return new InMemoryUserDetailsManager( + properties.getUsers().stream() + .map(userData -> User.builder() + .username(userData.getUsername()) + .password(userData.getPassword()) + .roles(userData.getRoles().toArray(new String[0])) + .build()) + .toList()); } @Bean + public PasswordEncoder passwordEncoder() { + int strength = 10; // iterations + return new BCryptPasswordEncoder(strength, new SecureRandom()); + } + + /*@Bean public PasswordEncoder encoder() { + // Clear-text password encoder return new PasswordEncoder() { @Override public String encode(CharSequence rawPassword) { @@ -65,5 +75,5 @@ public boolean matches(CharSequence rawPassword, String encodedPassword) { return rawPassword.toString().equals(encodedPassword); } }; - } + }*/ } diff --git a/management/src/main/resources/application.yml b/management/src/main/resources/application.yml index fae7a02..8bb7c81 100644 --- a/management/src/main/resources/application.yml +++ b/management/src/main/resources/application.yml @@ -22,12 +22,14 @@ discovery: brokerURL: "tcp://localhost:61616?daemon=true&trace=false&useInactivityMonitor=false&connectionTimeout=0&keepAlive=true" allowedDeviceInfoKeys: - '*' + # NOTE: + # To generate BCrypt encrypted passwords you can use: https://bcrypt-generator.com/ users: - username: admin - password: admin1 + password: '$2a$10$5jzrhbVKq.W2J1PMGYeHyeydQtlw71PoVgryzDP0VZ.88FsPlq1ne' # admin1 (BCrypt; 10 iterations) roles: [ ADMIN ] - username: user - password: user1 + password: '$2a$10$I6GSOKiY5n4/Ql0LA7Js0.4HT4UXVCNaNpGv5UdZt/brEdv/F.ttG' # user1 (BCrypt; 10 iterations) roles: [ USER ] #logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE \ No newline at end of file From 53052694891bc6233edda077a2b655f9a2251678 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 17 Oct 2023 13:32:00 +0300 Subject: [PATCH 038/132] RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. --- .../ResourceDiscoveryProperties.java | 12 + .../resource/discovery/common/BrokerUtil.java | 215 ++++++++++++++++ .../discovery/common/EncryptionUtil.java | 231 ++++++++++++++++++ .../service/AbstractMonitorService.java | 65 ++--- .../DeviceLifeCycleRequestService.java | 64 +---- .../DeviceLifeCycleResponseService.java | 8 +- .../service/DeviceManagementService.java | 2 - .../service/DeviceMetricsMonitorService.java | 8 +- .../service/DeviceStatusMonitorService.java | 8 +- .../UnknownDeviceRegistrationService.java | 61 ++--- .../RegistrationRequestProcessor.java | 80 ++---- 11 files changed, 541 insertions(+), 213 deletions(-) create mode 100644 management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java create mode 100644 management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index b5c2c5b..c07e102 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -19,6 +20,8 @@ public class ResourceDiscoveryProperties { @ToString.Exclude private String brokerPassword; private String brokerURL; + private int connectionHealthCheckPeriod = 60; // in seconds + private String healthCheckTopic = "_HEALTH_CHECK"; // Subscription to Broker settings private long subscriptionStartupDelay = 10; @@ -63,6 +66,15 @@ public class ResourceDiscoveryProperties { private boolean immediatelyArchiveSuccessRequests = true; private boolean immediatelyArchiveOffboardedDevices = true; + // Encryption settings + private boolean enableEncryption; // Set to 'true' to enable message encryption +// private boolean requireEncryption; + private boolean usePasswordGeneratedKey = true; + private String generatedKeyFile; // NOTE: If blank, the key will be logged + private String keyPasswordFile; // If provided, it will override the next settings + private char[] symmetricKeyPassword; + private byte[] salt; + // Users private List users; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java b/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java new file mode 100644 index 0000000..dfe0f44 --- /dev/null +++ b/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java @@ -0,0 +1,215 @@ + +package eu.nebulous.resource.discovery.common; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import jakarta.jms.*; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.activemq.ActiveMQConnection; +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.apache.activemq.command.ActiveMQTopic; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.util.*; + +@Slf4j +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON) +@RequiredArgsConstructor +public class BrokerUtil implements InitializingBean, MessageListener { + private final ResourceDiscoveryProperties properties; + private final EncryptionUtil encryptionUtil; + private final TaskScheduler taskScheduler; + private final ObjectMapper objectMapper; + private final Map producers = new HashMap<>(); + private final Map consumers = new HashMap<>(); + private final Map> listeners = new HashMap<>(); + private ActiveMQConnection connection; + private Session session; + + @Override + public void afterPropertiesSet() throws Exception { + // Initialize broker connection + taskScheduler.schedule(this::initializeBrokerConnection, + Instant.now().plusSeconds(properties.getSubscriptionStartupDelay())); + + // Initialize connection health check + int healthCheckPeriod = properties.getConnectionHealthCheckPeriod(); + if (healthCheckPeriod>0) { + taskScheduler.scheduleAtFixedRate(this::connectionHealthCheck, + Instant.now().plusSeconds(properties.getSubscriptionStartupDelay()), + Duration.ofSeconds(healthCheckPeriod)); + log.info("BrokerUtil: Enabled connection health check: period={}s", healthCheckPeriod); + } + } + + private void initializeBrokerConnection() { + try { + openBrokerConnection(); + } catch (JMSException e) { + log.error("BrokerUtil: ERROR while opening connection to Message broker: ", e); + taskScheduler.schedule(this::initializeBrokerConnection, + Instant.now().plusSeconds(properties.getSubscriptionRetryDelay())); + } + } + + private void openBrokerConnection() throws JMSException { + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( + properties.getBrokerUsername(), properties.getBrokerPassword(), + properties.getBrokerURL()); + ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); + Session ses = conn.createSession(); + conn.start(); + this.connection = conn; + this.session = ses; + log.info("BrokerUtil: Opened connection to Message broker: {}", properties.getBrokerURL()); + } + + private void closeBrokerConnection() throws JMSException { + producers.clear(); + consumers.clear(); + listeners.clear(); + if (session!=null) this.session.close(); + if (connection!=null && ! connection.isClosed() && ! connection.isClosing()) + this.connection.close(); + this.session = null; + this.connection = null; + log.info("BrokerUtil: Closed connection to Message broker: {}", properties.getBrokerURL()); + } + + public void connectionHealthCheck() { + log.debug("BrokerUtil: Checking connection health: {}", properties.getBrokerURL()); + boolean error = false; + try { + sendMessage(properties.getHealthCheckTopic(), Map.of("ping", "pong")); + } catch (JMSException | JsonProcessingException e) { + log.warn("BrokerUtil: EXCEPTION during connection health: ", e); + error = true; + } + + if (error) { + // Close connection + try { + closeBrokerConnection(); + } catch (JMSException e) { + log.error("BrokerUtil: ERROR while closing connection to Message broker: ", e); + this.session = null; + this.connection = null; + } + + // Try to re-connect + taskScheduler.schedule(this::initializeBrokerConnection, + Instant.now().plusSeconds(1)); + } + } + + // ------------------------------------------------------------------------ + + public void sendMessage(@NonNull String topic, @NonNull Map message) throws JMSException, JsonProcessingException { + sendMessage(topic, message, false); + } + + public void sendMessage(@NonNull String topic, @NonNull Map message, boolean encrypt) throws JMSException, JsonProcessingException { + String jsonMessage = objectMapper.writer().writeValueAsString(message); + sendMessage(topic, jsonMessage, encrypt); + } + + public void sendMessage(@NonNull String topic, @NonNull String message) throws JMSException, JsonProcessingException { + sendMessage(topic, message, false); + } + + public void sendMessage(@NonNull String topic, @NonNull String message, boolean encrypt) throws JMSException, JsonProcessingException { + ActiveMQTextMessage textMessage = new ActiveMQTextMessage(); + if (encrypt) { + sendMessage(topic, Map.of("encrypted-message", encryptionUtil.encryptText(message)), false); + } else { + textMessage.setText(message); + getOrCreateProducer(topic).send(textMessage); + } + } + + // ------------------------------------------------------------------------ + + public MessageProducer getOrCreateProducer(@NonNull String topic) throws JMSException { + MessageProducer producer = producers.get(topic); + if (producer==null) { producer = createProducer(topic); producers.put(topic, producer); } + return producer; + } + + public MessageConsumer getOrCreateConsumer(@NonNull String topic) throws JMSException { + MessageConsumer consumer = consumers.get(topic); + if (consumer==null) { consumer = createConsumer(topic); consumers.put(topic, consumer); } + return consumer; + } + + public MessageProducer createProducer(@NonNull String topic) throws JMSException { + if (session==null) initializeBrokerConnection(); + return session.createProducer(new ActiveMQTopic(topic)); + } + + public MessageConsumer createConsumer(@NonNull String topic) throws JMSException { + if (session==null) initializeBrokerConnection(); + return session.createConsumer(new ActiveMQTopic(topic)); + } + + // ------------------------------------------------------------------------ + + public void subscribe(@NonNull String topic, @NonNull Listener listener) throws JMSException { + Set set = listeners.computeIfAbsent(topic, t -> new HashSet<>()); + if (set.contains(listener)) return; + set.add(listener); + getOrCreateConsumer(topic).setMessageListener(this); + } + + @Override + public void onMessage(Message message) { + try { + log.debug("BrokerUtil: Received a message from broker: {}", message); + if (message instanceof ActiveMQTextMessage textMessage) { + String payload = textMessage.getText(); + log.trace("BrokerUtil: Message payload: {}", payload); + + TypeReference> typeRef = new TypeReference<>() { }; + Object obj = objectMapper.readerFor(typeRef).readValue(payload); + + if (obj instanceof Map dataMap) { + handlePayload(((ActiveMQTextMessage) message).getDestination().getPhysicalName(), dataMap); + } else { + log.debug("BrokerUtil: Message payload is not recognized. Expected Map but got: type={}, object={}", obj.getClass().getName(), obj); + } + } else { + log.debug("BrokerUtil: Message type is not supported: {}", message); + } + } catch (Exception e) { + log.warn("BrokerUtil: ERROR while processing message: {}\nException: ", message, e); + } + } + + private void handlePayload(@NonNull String topic, @NonNull Map dataMap) { + // Decrypt message (if encrypted) + Object encryptedMessage = dataMap.get("encrypted-message"); + if (encryptedMessage!=null) + dataMap = encryptionUtil.decryptMap(encryptedMessage.toString()); + + // Dispatch message to listeners + Set set = listeners.get(topic); + if (set==null) return; + final Map immutableMap = Collections.unmodifiableMap(dataMap); + set.forEach(l -> l.onMessage(immutableMap)); + } + + public interface Listener { + void onMessage(Map map); + } +} diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java b/management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java new file mode 100644 index 0000000..c33c999 --- /dev/null +++ b/management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java @@ -0,0 +1,231 @@ + +package eu.nebulous.resource.discovery.common; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.tomcat.util.codec.binary.Base64; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.crypto.*; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import javax.security.auth.DestroyFailedException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.util.Arrays; +import java.util.Map; + +/* + * SEE: + * https://www.baeldung.com/java-aes-encryption-decryption + * https://stackoverflow.com/questions/6538485/java-using-aes-256-and-128-symmetric-key-encryption + */ + +@Slf4j +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON) +@RequiredArgsConstructor +public class EncryptionUtil implements InitializingBean { + public final static String KEY_FACTORY_ALGORITHM = "PBKDF2WithHmacSHA256"; + public final static String KEY_GEN_ALGORITHM = "AES"; + public final static String CIPHER_ALGORITHM = "AES"; + //public final static String CIPHER_ALGORITHM = "AES/CTR/PKCS5Padding"; + public final static int SIZE = 256; + public final static int ITERATION_COUNT = 65536; + + public final static byte SEPARATOR = 32; + public final static byte NULL = 0; + public final static char NULL_CHAR = '\0'; + + private final ResourceDiscoveryProperties properties; + private final ObjectMapper objectMapper; + private SecretKey key; + + @Override + public void afterPropertiesSet() throws Exception { + if (properties.isEnableEncryption()) + initializeSymmetricKey(); + } + + private void initializeSymmetricKey() throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { + if (properties.isUsePasswordGeneratedKey()) { + char[] password; + byte[] salt; + if (StringUtils.isNotBlank(properties.getKeyPasswordFile())) { + // Read key password from file + File file = Paths.get(properties.getKeyPasswordFile()).toFile(); + try (FileInputStream in = new FileInputStream(file)) { + byte[] bytes = in.readAllBytes(); + // find separator + int ii = Arrays.binarySearch(bytes, SEPARATOR); + password = new char[ii/2]; + for (int j=0, k=0; j message) { + try { + if (! properties.isEnableEncryption()) return objectMapper.writeValueAsString(message); + + byte[] bytes = objectMapper.writeValueAsBytes(message); + + try { + return encrypt(bytes, key); + } catch (IllegalBlockSizeException | NoSuchPaddingException | BadPaddingException | + NoSuchAlgorithmException | InvalidKeyException e) + { + log.warn("EncryptionUtil: ERROR while encrypting message: ", e); + } finally { + Arrays.fill(bytes, NULL); + } + } catch (JsonProcessingException e) { + log.warn("EncryptionUtil: ERROR while converting message (Map) to bytes: ", e); + } + return null; + } + + public Map decryptMap(@NonNull String message) { + try { + if (! properties.isEnableEncryption()) return objectMapper.readValue(message, Map.class); + + byte[] bytes = decrypt(message, key); + + try { + TypeReference> tr = new TypeReference<>() {}; + return objectMapper.readValue(bytes, tr); + } catch (IOException e) { + log.warn("EncryptionUtil: ERROR while converting decrypted bytes to Map: ", e); + } finally { + Arrays.fill(bytes, NULL); + } + } catch (NoSuchPaddingException | IllegalBlockSizeException | NoSuchAlgorithmException + | BadPaddingException | InvalidKeyException | IOException e) + { + log.warn("EncryptionUtil: ERROR while decrypting message: ", e); + } + return null; + } + + private String encrypt(byte[] bytesToEncrypt, SecretKey secretKey) + throws IllegalBlockSizeException, BadPaddingException, NoSuchPaddingException, + NoSuchAlgorithmException, InvalidKeyException + { + Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + return Base64.encodeBase64String( + cipher.doFinal(bytesToEncrypt)); + } + + private byte[] decrypt(final String encryptedMessage, SecretKey secretKey) + throws NoSuchPaddingException, NoSuchAlgorithmException, IllegalBlockSizeException, + BadPaddingException, InvalidKeyException + { + //final SecretKeySpec secretKey = new SecretKeySpec(keyStr.getBytes(StandardCharsets.UTF_8), "AES"); + final Cipher c = Cipher.getInstance(CIPHER_ALGORITHM); + c.init(Cipher.DECRYPT_MODE, secretKey); + return c.doFinal( + Base64.decodeBase64(encryptedMessage)); + } +} diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java index bdab2aa..f36e530 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java @@ -1,19 +1,14 @@ package eu.nebulous.resource.discovery.monitor.service; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; -import jakarta.jms.*; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.activemq.ActiveMQConnection; -import org.apache.activemq.ActiveMQConnectionFactory; -import org.apache.activemq.command.ActiveMQTextMessage; -import org.apache.activemq.command.ActiveMQTopic; import org.springframework.beans.factory.InitializingBean; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.annotation.EnableAsync; @@ -29,13 +24,12 @@ @EnableAsync @EnableScheduling @RequiredArgsConstructor -public abstract class AbstractMonitorService implements InitializingBean, MessageListener { +public abstract class AbstractMonitorService implements InitializingBean, BrokerUtil.Listener { @NonNull protected final String name; protected final ResourceDiscoveryProperties monitorProperties; protected final TaskScheduler taskScheduler; protected final ObjectMapper objectMapper; - protected ActiveMQConnection connection; - protected Session session; + protected final BrokerUtil brokerUtil; @Override public void afterPropertiesSet() throws Exception { @@ -45,53 +39,24 @@ public void afterPropertiesSet() throws Exception { } private void initializeDeviceStatusListener() { - try { - openBrokerConnection(); - getTopicsToMonitor().forEach(topic -> { - try { - MessageConsumer consumer = session.createConsumer(new ActiveMQTopic(topic)); - consumer.setMessageListener(this); - log.debug("{}: Subscribed to topic: {}", name, topic); - } catch (Exception e) { - log.error("{}: ERROR while subscribing to topic: {}\n", name, topic, e); - taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); - } - }); - } catch (JMSException e) { - log.error("{}: ERROR while opening connection to Message broker: ", name, e); - taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); - } - } - - private void openBrokerConnection() throws JMSException { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - monitorProperties.getBrokerUsername(), monitorProperties.getBrokerPassword(), monitorProperties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); - Session ses = conn.createSession(); - this.connection = conn; - this.session = ses; - connection.start(); + getTopicsToMonitor().forEach(topic -> { + try { + brokerUtil.subscribe(topic, this); + log.debug("{}: Subscribed to topic: {}", name, topic); + } catch (Exception e) { + log.error("{}: ERROR while subscribing to topic: {}\n", name, topic, e); + taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); + } + }); } protected abstract @NonNull List getTopicsToMonitor(); @Override - public void onMessage(Message message) { + public void onMessage(Map message) { try { - log.debug("{}: Received a JMS message: {}", name, message); - if (message instanceof ActiveMQTextMessage textMessage) { - String payload = textMessage.getText(); - log.trace("{}: Message payload: {}", name, payload); - TypeReference> typeRef = new TypeReference<>() { }; - Object obj = objectMapper.readerFor(typeRef).readValue(payload); - if (obj instanceof Map dataMap) { - processPayload(dataMap); - } else { - log.debug("{}: Message payload is not recognized. Expected Map but got: type={}, object={}", name, obj.getClass().getName(), obj); - } - } else { - log.debug("{}: Message type is not supported: {}", name, message); - } + log.debug("{}: Received a message: {}", name, message); + processPayload(message); } catch (Exception e) { log.warn("{}: ERROR while processing message: {}\nException: ", name, message, e); } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java index 2b13a9a..160a321 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java @@ -1,28 +1,20 @@ package eu.nebulous.resource.discovery.monitor.service; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; -import jakarta.jms.JMSException; -import jakarta.jms.MessageNotWriteableException; -import jakarta.jms.MessageProducer; -import jakarta.jms.Session; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.activemq.ActiveMQConnection; -import org.apache.activemq.ActiveMQConnectionFactory; -import org.apache.activemq.command.ActiveMQMessage; -import org.apache.activemq.command.ActiveMQTextMessage; -import org.apache.activemq.command.ActiveMQTopic; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import java.time.Instant; -import java.util.*; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; @Slf4j @Service @@ -30,7 +22,7 @@ public class DeviceLifeCycleRequestService { private final ResourceDiscoveryProperties properties; private final DeviceManagementService deviceManagementService; - private final ObjectMapper objectMapper; + private final BrokerUtil brokerUtil; // ------------------------------------------------------------------------ @@ -46,18 +38,11 @@ public void reinstallRequest(String id) { // Prepare request log.debug("reinstallRequest: Requesting device re-onboarding with Id: {}", device.getId()); Map onboardingRequest = prepareRequestPayload(REQUEST_TYPE.REINSTALL, device); - String jsonMessage = objectMapper.writer().writeValueAsString(onboardingRequest); - - // Connect to Message broker - Pair connAndProducer = connectToBroker(); // Send request - connAndProducer.getSecond().send(createMessage(jsonMessage)); + brokerUtil.sendMessage(properties.getDeviceLifeCycleRequestsTopic(), onboardingRequest); device.setStatus(DeviceStatus.ONBOARDING); - // Close connection to Message broker - connAndProducer.getFirst().close(); - log.debug("reinstallRequest: Save updated device: id={}, device={}", device.getId(), device); deviceManagementService.update(device); log.debug("reinstallRequest: Onboarding request sent for device with Id: {}", device.getId()); @@ -83,18 +68,11 @@ public void uninstallRequest(String id) { // Prepare request log.debug("uninstallRequest: Requesting device off-onboarding with Id: {}", device.getId()); Map offboardingRequest = prepareRequestPayload(REQUEST_TYPE.UNINSTALL, device); - String jsonMessage = objectMapper.writer().writeValueAsString(offboardingRequest); - - // Connect to Message broker - Pair connAndProducer = connectToBroker(); // Send request - connAndProducer.getSecond().send(createMessage(jsonMessage)); + brokerUtil.sendMessage(properties.getDeviceLifeCycleRequestsTopic(), offboardingRequest); device.setStatus(DeviceStatus.OFFBOARDING); - // Close connection to Message broker - connAndProducer.getFirst().close(); - log.debug("uninstallRequest: Save updated device: id={}, device={}", device.getId(), device); deviceManagementService.update(device); log.debug("uninstallRequest: Off-boarding request sent for device with Id: {}", device.getId()); @@ -113,16 +91,9 @@ public void requestInfoUpdate() { // Prepare request log.debug("requestInfoUpdate: Requesting device info and metrics update"); Map updateRequest = prepareRequestPayload(REQUEST_TYPE.INFO, null); - String jsonMessage = objectMapper.writer().writeValueAsString(updateRequest); - - // Connect to Message broker - Pair connAndProducer = connectToBroker(); // Send request - connAndProducer.getSecond().send(createMessage(jsonMessage)); - - // Close connection to Message broker - connAndProducer.getFirst().close(); + brokerUtil.sendMessage(properties.getDeviceLifeCycleRequestsTopic(), updateRequest); log.debug("requestInfoUpdate: Update request sent"); } catch (Exception e) { @@ -163,21 +134,4 @@ private static Map prepareRequestPayload(@NonNull REQUEST_TYPE r throw e; } } - - protected Pair connectToBroker() throws JMSException { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - properties.getBrokerUsername(), properties.getBrokerPassword(), - properties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); - Session session = conn.createSession(); - MessageProducer producer = session.createProducer( - new ActiveMQTopic( properties.getDeviceLifeCycleRequestsTopic() )); - return Pair.of(conn, producer); - } - - protected ActiveMQMessage createMessage(String message) throws MessageNotWriteableException { - ActiveMQTextMessage textMessage = new ActiveMQTextMessage(); - textMessage.setText(message); - return textMessage; - } } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java index c6ee3eb..498202e 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java @@ -2,6 +2,7 @@ package eu.nebulous.resource.discovery.monitor.service; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.Device; @@ -22,8 +23,11 @@ public class DeviceLifeCycleResponseService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; - public DeviceLifeCycleResponseService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super("DeviceLifeCycleResponseService", monitorProperties, taskScheduler, objectMapper); + public DeviceLifeCycleResponseService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, + ObjectMapper objectMapper, DeviceManagementService deviceManagementService, + BrokerUtil brokerUtil) + { + super("DeviceLifeCycleResponseService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index 07bf8a2..8bf7802 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -1,6 +1,5 @@ package eu.nebulous.resource.discovery.monitor.service; -import com.fasterxml.jackson.databind.ObjectMapper; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.monitor.model.ArchivedDevice; import eu.nebulous.resource.discovery.monitor.model.Device; @@ -28,7 +27,6 @@ public class DeviceManagementService { private final DeviceRepository deviceRepository; private final ArchivedDeviceRepository archivedDeviceRepository; private final DeviceConversionService deviceConversionService; - private final ObjectMapper objectMapper; // ------------------------------------------------------------------------ diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 7e0a471..0b355df 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceMetrics; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; @@ -25,8 +26,11 @@ public class DeviceMetricsMonitorService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; - public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper); + public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, + ObjectMapper objectMapper, DeviceManagementService deviceManagementService, + BrokerUtil brokerUtil) + { + super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java index bb88d44..dcd7b21 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatusUpdate; import lombok.NonNull; @@ -20,8 +21,11 @@ public class DeviceStatusMonitorService extends AbstractMonitorService { private final DeviceManagementService deviceManagementService; - public DeviceStatusMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super("DeviceStatusMonitorService", monitorProperties, taskScheduler, objectMapper); + public DeviceStatusMonitorService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, + ObjectMapper objectMapper, DeviceManagementService deviceManagementService, + BrokerUtil brokerUtil) + { + super("DeviceStatusMonitorService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; } diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index 6458ba2..039dd6b 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -2,16 +2,13 @@ package eu.nebulous.resource.discovery.monitor.service; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; -import jakarta.jms.JMSException; -import jakarta.jms.MessageProducer; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.activemq.command.ActiveMQTextMessage; -import org.apache.activemq.command.ActiveMQTopic; import org.apache.commons.lang3.StringUtils; import org.springframework.scheduling.TaskScheduler; import org.springframework.stereotype.Service; @@ -32,8 +29,11 @@ public class UnknownDeviceRegistrationService extends AbstractMonitorService { private final Map detectedDevices = Collections.synchronizedMap(new LinkedHashMap<>()); private final List deviceDetailsQueue = Collections.synchronizedList(new LinkedList<>()); - public UnknownDeviceRegistrationService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService) { - super("UnknownDeviceRegistrationService", monitorProperties, taskScheduler, objectMapper); + public UnknownDeviceRegistrationService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, + ObjectMapper objectMapper, DeviceManagementService deviceManagementService, + BrokerUtil brokerUtil) + { + super("UnknownDeviceRegistrationService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; } @@ -135,34 +135,25 @@ private void processDetectedDevices() { } private void processUnknownDevices(LinkedHashMap unknownDevices) { - try { - log.info("UnknownDeviceRegistrationService: Unknown devices: {}", unknownDevices); - MessageProducer producer = session.createProducer( - new ActiveMQTopic(monitorProperties.getDeviceInfoRequestsTopic())); - - unknownDevices.forEach((ipAddress, reference) -> { - try { - // Query EMS for device info - log.debug("UnknownDeviceRegistrationService: Sending Node-Details-Request Message: ipAddress={}, reference={}", - ipAddress, reference); - Map request = new LinkedHashMap<>(Map.of( - "requestType", REQUEST_TYPE.NODE_DETAILS.name(), - "deviceIpAddress", ipAddress, - "reference", reference - )); - ActiveMQTextMessage message = new ActiveMQTextMessage(); - message.setText(objectMapper.writeValueAsString(request)); - - log.debug("UnknownDeviceRegistrationService: Sending Node-Details-Request Message: request={}", message.getText()); - producer.send(message); - log.debug("UnknownDeviceRegistrationService: Node-Details-Request Message sent: ipAddress={}", ipAddress); - } catch (Exception e) { - log.error("UnknownDeviceRegistrationService: ERROR while creating Node-Details-Request Message: ", e); - } - }); - } catch (JMSException e) { - log.error("UnknownDeviceRegistrationService: ERROR while creating Message producer: ", e); - } + log.info("UnknownDeviceRegistrationService: Unknown devices: {}", unknownDevices); + unknownDevices.forEach((ipAddress, reference) -> { + try { + // Query EMS for device info + log.debug("UnknownDeviceRegistrationService: Sending Node-Details-Request Message: ipAddress={}, reference={}", + ipAddress, reference); + Map request = new LinkedHashMap<>(Map.of( + "requestType", REQUEST_TYPE.NODE_DETAILS.name(), + "deviceIpAddress", ipAddress, + "reference", reference + )); + + log.debug("UnknownDeviceRegistrationService: Sending Node-Details-Request Message: request={}", request); + brokerUtil.sendMessage(monitorProperties.getDeviceInfoRequestsTopic(), request); + log.debug("UnknownDeviceRegistrationService: Node-Details-Request Message sent: ipAddress={}", ipAddress); + } catch (Exception e) { + log.error("UnknownDeviceRegistrationService: ERROR while creating Node-Details-Request Message: ", e); + } + }); } private void processDeviceDetailsResponses() { diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 0e14383..f8871f7 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -1,23 +1,17 @@ package eu.nebulous.resource.discovery.registration; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.common.BrokerUtil; +import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; import eu.nebulous.resource.discovery.registration.model.RegistrationRequest; import eu.nebulous.resource.discovery.registration.model.RegistrationRequestStatus; import eu.nebulous.resource.discovery.registration.service.RegistrationRequestService; -import jakarta.jms.*; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.activemq.ActiveMQConnection; -import org.apache.activemq.ActiveMQConnectionFactory; -import org.apache.activemq.command.ActiveMQMessage; -import org.apache.activemq.command.ActiveMQTextMessage; -import org.apache.activemq.command.ActiveMQTopic; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.scheduling.TaskScheduler; @@ -42,7 +36,7 @@ @EnableAsync @EnableScheduling @RequiredArgsConstructor -public class RegistrationRequestProcessor implements IRegistrationRequestProcessor, InitializingBean, MessageListener { +public class RegistrationRequestProcessor implements IRegistrationRequestProcessor, InitializingBean, BrokerUtil.Listener { private final static List STATUSES_TO_ARCHIVE = List.of( RegistrationRequestStatus.PRE_AUTHORIZATION_REJECT, RegistrationRequestStatus.PRE_AUTHORIZATION_ERROR, @@ -58,6 +52,7 @@ public class RegistrationRequestProcessor implements IRegistrationRequestProcess private final DeviceManagementService deviceManagementService; private final TaskScheduler taskScheduler; private final ObjectMapper objectMapper; + private final BrokerUtil brokerUtil; private final AtomicBoolean isRunning = new AtomicBoolean(false); @Override @@ -88,28 +83,16 @@ public Future processRequests() { } log.debug("processRequests: Processing registration requests"); - // Connect to Message broker - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - processorProperties.getBrokerUsername(), processorProperties.getBrokerPassword(), - processorProperties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); - Session session = conn.createSession(); - MessageProducer producer = session.createProducer( - new ActiveMQTopic(processorProperties.getDataCollectionRequestTopic())); - // Process requests try { - processNewRequests(producer); - processOnboardingRequests(producer); + processNewRequests(); + processOnboardingRequests(); if (processorProperties.isAutomaticArchivingEnabled()) archiveRequests(); } catch (Throwable t) { log.error("processRequests: ERROR processing requests: ", t); } - // Close connection to Message broker - conn.close(); - log.debug("processRequests: Processing completed"); return CompletableFuture.completedFuture("DONE"); @@ -122,8 +105,8 @@ public Future processRequests() { } } - private void processNewRequests(MessageProducer producer) { - log.trace("processNewRequests: BEGIN: {}", producer); + private void processNewRequests() { + log.trace("processNewRequests: BEGIN"); List newRequests = registrationRequestService.getAll().stream() .filter(r -> r.getStatus() == RegistrationRequestStatus.NEW_REQUEST).toList(); @@ -134,8 +117,7 @@ private void processNewRequests(MessageProducer producer) { try { log.debug("processNewRequests: Requesting collection of device data for request with Id: {}", registrationRequest.getId()); Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE.DIAGNOSTICS, registrationRequest); - String jsonMessage = objectMapper.writer().writeValueAsString(dataCollectionRequest); - producer.send(createMessage(jsonMessage)); + brokerUtil.sendMessage(processorProperties.getDataCollectionRequestTopic(), dataCollectionRequest); registrationRequest.setStatus(RegistrationRequestStatus.DATA_COLLECTION_REQUESTED); log.debug("processNewRequests: Save updated request: id={}, request={}", registrationRequest.getId(), registrationRequest); @@ -151,8 +133,8 @@ private void processNewRequests(MessageProducer producer) { log.trace("processNewRequests: END"); } - private void processOnboardingRequests(MessageProducer producer) { - log.trace("processOnboardingRequests: BEGIN: {}", producer); + private void processOnboardingRequests() { + log.trace("processOnboardingRequests: BEGIN"); List onboardingRequests = registrationRequestService.getAll().stream() .filter(r -> r.getStatus() == RegistrationRequestStatus.PENDING_ONBOARDING).toList(); @@ -169,8 +151,7 @@ private void processOnboardingRequests(MessageProducer producer) { log.debug("processOnboardingRequests: Requesting device onboarding for request with Id: {}", registrationRequest.getId()); Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE.INSTALL, registrationRequest); - String jsonMessage = objectMapper.writer().writeValueAsString(dataCollectionRequest); - producer.send(createMessage(jsonMessage)); + brokerUtil.sendMessage(processorProperties.getDataCollectionRequestTopic(), dataCollectionRequest); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_REQUESTED); log.debug("processOnboardingRequests: Save updated request: id={}, request={}", registrationRequest.getId(), registrationRequest); @@ -211,12 +192,6 @@ private static Map prepareRequestPayload(@NonNull REQUEST_TYPE r } } - protected ActiveMQMessage createMessage(String message) throws MessageNotWriteableException { - ActiveMQTextMessage textMessage = new ActiveMQTextMessage(); - textMessage.setText(message); - return textMessage; - } - private void archiveRequests() { Instant archiveThreshold = Instant.now().minus(processorProperties.getArchivingThreshold(), ChronoUnit.MINUTES); log.trace("archiveRequests: BEGIN: archive-threshold: {}", archiveThreshold); @@ -241,40 +216,15 @@ private void archiveRequests() { protected void initializeResultsListener() { try { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - processorProperties.getBrokerUsername(), processorProperties.getBrokerPassword(), processorProperties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); - Session session = conn.createSession(); - MessageConsumer consumer = session.createConsumer( - new ActiveMQTopic(processorProperties.getDataCollectionResponseTopic())); - consumer.setMessageListener(this); - conn.start(); + brokerUtil.subscribe(processorProperties.getDataCollectionResponseTopic(), this); } catch (Exception e) { log.error("RegistrationRequestProcessor: ERROR while subscribing to Message broker for Device info announcements: ", e); taskScheduler.schedule(this::initializeResultsListener, Instant.now().plusSeconds(processorProperties.getSubscriptionRetryDelay())); } } - @Override - public void onMessage(Message message) { - try { - log.debug("RegistrationRequestProcessor: Received a JMS message: {}", message); - if (message instanceof ActiveMQTextMessage textMessage) { - String payload = textMessage.getText(); - log.trace("RegistrationRequestProcessor: Message payload: {}", payload); - TypeReference> typeRef = new TypeReference<>() { }; - Object obj = objectMapper.readerFor(typeRef).readValue(payload); - if (obj instanceof Map response) { - processResponse(response); - } else { - log.debug("RegistrationRequestProcessor: Message payload is not recognized. Expected Map: type={}, object={}", obj.getClass().getName(), obj); - } - } else { - log.debug("RegistrationRequestProcessor: Message type is not supported: {}", message); - } - } catch (Exception e) { - log.warn("RegistrationRequestProcessor: ERROR while processing message: {}\nException: ", message, e); - } + public void onMessage(Map message) { + processResponse(message); } private void processResponse(@NonNull Map response) { From 101f2c886a3a6654e3694ce6064cc7b30277cb24 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 17 Oct 2023 13:35:29 +0300 Subject: [PATCH 039/132] RD: Minor code tidy up --- .../discovery/ResourceDiscoveryProperties.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index c07e102..625aefe 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -20,7 +20,7 @@ public class ResourceDiscoveryProperties { @ToString.Exclude private String brokerPassword; private String brokerURL; - private int connectionHealthCheckPeriod = 60; // in seconds + private int connectionHealthCheckPeriod = 60; // in seconds private String healthCheckTopic = "_HEALTH_CHECK"; // Subscription to Broker settings @@ -53,8 +53,8 @@ public class ResourceDiscoveryProperties { // Failed devices detection private boolean automaticFailedDetection = true; - private long suspectDeviceThreshold = 5; // in minutes - private long failedDeviceThreshold = 10; // in minutes + private long suspectDeviceThreshold = 5; // in minutes + private long failedDeviceThreshold = 10; // in minutes // Device detailed data settings private String deviceInfoRequestsTopic = "ems.client.info.requests"; @@ -62,16 +62,15 @@ public class ResourceDiscoveryProperties { // Archiving settings private boolean automaticArchivingEnabled; - private long archivingThreshold; // in minutes + private long archivingThreshold; // in minutes private boolean immediatelyArchiveSuccessRequests = true; private boolean immediatelyArchiveOffboardedDevices = true; // Encryption settings - private boolean enableEncryption; // Set to 'true' to enable message encryption -// private boolean requireEncryption; + private boolean enableEncryption; // Set to 'true' to enable message encryption private boolean usePasswordGeneratedKey = true; - private String generatedKeyFile; // NOTE: If blank, the key will be logged - private String keyPasswordFile; // If provided, it will override the next settings + private String generatedKeyFile; // NOTE: If blank, the key will be logged + private String keyPasswordFile; // If provided, it will override the next settings private char[] symmetricKeyPassword; private byte[] salt; From fc3c1afd774292d088c1e05c0fbf5f96fd0aa2d8 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 17 Oct 2023 20:33:21 +0300 Subject: [PATCH 040/132] RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml --- .../ResourceDiscoveryProperties.java | 10 ++++++- .../resource/discovery/common/BrokerUtil.java | 30 ++++++++++++++----- management/src/main/resources/application.yml | 5 +++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 625aefe..1a9ba44 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -16,10 +16,18 @@ @ConfigurationProperties(prefix = "discovery") public class ResourceDiscoveryProperties { // Broker configuration + private String brokerURL; private String brokerUsername; @ToString.Exclude private String brokerPassword; - private String brokerURL; + + private String keyStoreFile; + private String keyStorePassword; + private String keyStoreType = "PKCS12"; + private String trustStoreFile; + private String trustStorePassword; + private String trustStoreType = "PKCS12"; + private int connectionHealthCheckPeriod = 60; // in seconds private String healthCheckTopic = "_HEALTH_CHECK"; diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java b/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java index dfe0f44..d308d64 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java @@ -10,9 +10,10 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.activemq.ActiveMQConnection; -import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.ActiveMQSslConnectionFactory; import org.apache.activemq.command.ActiveMQTextMessage; import org.apache.activemq.command.ActiveMQTopic; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -57,18 +58,33 @@ public void afterPropertiesSet() throws Exception { private void initializeBrokerConnection() { try { openBrokerConnection(); - } catch (JMSException e) { + } catch (Exception e) { log.error("BrokerUtil: ERROR while opening connection to Message broker: ", e); taskScheduler.schedule(this::initializeBrokerConnection, Instant.now().plusSeconds(properties.getSubscriptionRetryDelay())); } } - private void openBrokerConnection() throws JMSException { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory( - properties.getBrokerUsername(), properties.getBrokerPassword(), - properties.getBrokerURL()); - ActiveMQConnection conn = (ActiveMQConnection) connectionFactory.createConnection(); + private void openBrokerConnection() throws Exception { + ActiveMQSslConnectionFactory cf = new ActiveMQSslConnectionFactory(properties.getBrokerURL()); + cf.setUserName(properties.getBrokerUsername()); + cf.setPassword(properties.getBrokerPassword()); + + log.debug("BrokerUtil: Keystore and Truststore settings: keystore-file={}, keystore-type={}, truststore-file={}, truststore-type={}", + properties.getKeyStoreFile(), properties.getKeyStoreType(), properties.getTrustStoreFile(), properties.getTrustStoreType()); + if (StringUtils.isNotBlank(properties.getKeyStoreFile())) { + cf.setKeyStore(properties.getKeyStoreFile()); + cf.setKeyStorePassword(properties.getKeyStorePassword()); + cf.setKeyStoreType(properties.getKeyStoreType()); + } + if (StringUtils.isNotBlank(properties.getTrustStoreFile())) { + cf.setTrustStore(properties.getTrustStoreFile()); + cf.setTrustStorePassword(properties.getTrustStorePassword()); + cf.setTrustStoreType(properties.getKeyStoreType()); + } + cf.setWatchTopicAdvisories(true); + + ActiveMQConnection conn = (ActiveMQConnection) cf.createConnection(); Session ses = conn.createSession(); conn.start(); this.connection = conn; diff --git a/management/src/main/resources/application.yml b/management/src/main/resources/application.yml index 8bb7c81..0c6ee01 100644 --- a/management/src/main/resources/application.yml +++ b/management/src/main/resources/application.yml @@ -17,9 +17,12 @@ spring.data.mongodb.uri: mongodb://root:example@localhost:27017/admin spring.data.mongodb.database: resource_discovery discovery: + brokerURL: "ssl://localhost:61617?daemon=true&trace=false&useInactivityMonitor=false&connectionTimeout=0&keepAlive=true" brokerUsername: "aaa" brokerPassword: "111" - brokerURL: "tcp://localhost:61616?daemon=true&trace=false&useInactivityMonitor=false&connectionTimeout=0&keepAlive=true" + trustStoreFile: tests/config/broker-truststore.p12 + trustStorePassword: melodic + trustStoreType: PKCS12 allowedDeviceInfoKeys: - '*' # NOTE: From e75198e06da5addefc92cc013724e58be30f3fae Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 18 Oct 2023 12:03:45 +0300 Subject: [PATCH 041/132] RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does --- .../UnknownDeviceRegistrationService.java | 34 +++++++++++++------ .../service/RegistrationRequestService.java | 4 +++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index 039dd6b..7bb4fc1 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -7,6 +7,8 @@ import eu.nebulous.resource.discovery.common.REQUEST_TYPE; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; +import eu.nebulous.resource.discovery.registration.model.RegistrationRequest; +import eu.nebulous.resource.discovery.registration.service.RegistrationRequestService; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -25,15 +27,17 @@ public class UnknownDeviceRegistrationService extends AbstractMonitorService { REQUEST_TYPE.INSTALL.name(), REQUEST_TYPE.REINSTALL.name() ); + private final RegistrationRequestService registrationRequestService; private final DeviceManagementService deviceManagementService; private final Map detectedDevices = Collections.synchronizedMap(new LinkedHashMap<>()); private final List deviceDetailsQueue = Collections.synchronizedList(new LinkedList<>()); public UnknownDeviceRegistrationService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService, - BrokerUtil brokerUtil) + RegistrationRequestService registrationRequestService, BrokerUtil brokerUtil) { super("UnknownDeviceRegistrationService", monitorProperties, taskScheduler, objectMapper, brokerUtil); + this.registrationRequestService = registrationRequestService; this.deviceManagementService = deviceManagementService; } @@ -115,16 +119,26 @@ private void processDetectedDevices() { map.forEach((ipAddress, reference) -> { log.trace("UnknownDeviceRegistrationService: Processing device data: ipAddress={}, reference={}", ipAddress, reference); - // Check if device is registered - Optional device = deviceManagementService.getByIpAddress(ipAddress.trim()); - if (device.isEmpty() || !reference.equalsIgnoreCase(device.get().getNodeReference())) { - // Device is not registered - log.trace("UnknownDeviceRegistrationService: Unknown device: ipAddress={}, reference={}, device={}", ipAddress, reference, device.orElse(null)); - unknownDevices.put(ipAddress, reference); + // Check if there is a registration request for the device + List requests = registrationRequestService.getByDeviceIpAddress(ipAddress.trim()); + if (requests.isEmpty()) { + // No registration request found with this IP address + + // Check if device is registered + Optional device = deviceManagementService.getByIpAddress(ipAddress.trim()); + if (device.isEmpty() || ! reference.equalsIgnoreCase(device.get().getNodeReference())) { + // Device is not registered + log.trace("UnknownDeviceRegistrationService: Unknown device: ipAddress={}, reference={}, device={}", ipAddress, reference, device.orElse(null)); + unknownDevices.put(ipAddress, reference); + } else { + // Device is already registered + log.trace("UnknownDeviceRegistrationService: Device is already registered: ipAddress={}, device={}", + ipAddress, device.get()); + } } else { - // Device is already registered - log.trace("UnknownDeviceRegistrationService: Device is already registered: ipAddress={}, device={}", - ipAddress, device.get()); + // There is a registration request for Device + log.trace("UnknownDeviceRegistrationService: Device is already registered: ipAddress={}, request-id={}", + ipAddress, requests.stream().map(RegistrationRequest::getId).toList()); } }); diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java index 6361bb8..e775cb3 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java @@ -41,6 +41,10 @@ public Optional getById(@NonNull String id) { return registrationRequestRepository.findById(id); } + public List getByDeviceIpAddress(@NonNull String ipAddress) { + return registrationRequestRepository.findByDeviceIpAddress(ipAddress); + } + public List getAll() { return Collections.unmodifiableList(registrationRequestRepository.findAll()); } From 362e0070a93e09ea0f76755fd9f46120292b356a Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 18 Oct 2023 22:49:48 +0300 Subject: [PATCH 042/132] RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version --- management/pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/management/pom.xml b/management/pom.xml index e055c31..0f50430 100644 --- a/management/pom.xml +++ b/management/pom.xml @@ -18,6 +18,7 @@ 17 + ${project.groupId}:${project.version} @@ -88,6 +89,16 @@ org.springframework.boot spring-boot-maven-plugin + + + + build-image + + + + + ${imageName} + From ede964c40e07065cc4828140e416723bbffe7ed4 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 23 Oct 2023 16:24:24 +0300 Subject: [PATCH 043/132] RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. --- .../discovery/common/DeviceLocation.java | 25 +++++++++++++++++++ .../discovery/monitor/model/Device.java | 2 ++ .../discovery/registration/model/Device.java | 2 ++ .../archived-device-view.html | 15 +++++++++++ .../archived-request-view.html | 15 +++++++++++ .../static/freebees_webdesign_6/archived.html | 4 ++- .../freebees_webdesign_6/device-view.html | 15 +++++++++++ .../freebees_webdesign_6/request-edit.html | 15 +++++++++++ 8 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java b/management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java new file mode 100644 index 0000000..dff7074 --- /dev/null +++ b/management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java @@ -0,0 +1,25 @@ +package eu.nebulous.resource.discovery.common; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@NoArgsConstructor +public class DeviceLocation { + private String id; + private String name; + private String continent; + private String continentCode; + private String country; + private String countryCode; + private String state; + private String stateCode; + private String city; + private String zipcode; + private String address; + private String extra; + private double latitude; + private double longitude; +} diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index bb2e293..7aa5ded 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -1,6 +1,7 @@ package eu.nebulous.resource.discovery.monitor.model; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.nebulous.resource.discovery.common.DeviceLocation; import lombok.*; import lombok.experimental.SuperBuilder; import org.springframework.data.mongodb.core.mapping.Document; @@ -20,6 +21,7 @@ public class Device { private String name; private String owner; private String ipAddress; + private DeviceLocation location; private String username; @ToString.Exclude @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java b/management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java index db6d703..cdf04fa 100644 --- a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java +++ b/management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java @@ -1,6 +1,7 @@ package eu.nebulous.resource.discovery.registration.model; import com.fasterxml.jackson.annotation.JsonProperty; +import eu.nebulous.resource.discovery.common.DeviceLocation; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; @@ -17,6 +18,7 @@ public class Device { private String name; private String owner; private String ipAddress; + private DeviceLocation location; private String username; @ToString.Exclude @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html index 69fc81e..6b4e34e 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html @@ -287,6 +287,21 @@
Device details
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html index 5a0c767..16d70ba 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html @@ -300,6 +300,21 @@
Device details
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived.html b/management/src/main/resources/static/freebees_webdesign_6/archived.html index 1312872..a97df0b 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/management/src/main/resources/static/freebees_webdesign_6/archived.html @@ -130,6 +130,8 @@ $(`[id="device#password"]`).val(''); $(`[id="device#publicKey"]`).val(''); + $('#btn-restore-req-or-dev').html(type==='REQUEST' ? 'Restore Request' : 'Restore Device'); + // Show credentials modal var modal = new bootstrap.Modal(document.getElementById('credentialsModal'), { //keyboard: true @@ -411,7 +413,7 @@
- +
diff --git a/management/src/main/resources/static/freebees_webdesign_6/device-view.html b/management/src/main/resources/static/freebees_webdesign_6/device-view.html index 22b111e..37b9a9c 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/device-view.html +++ b/management/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -404,6 +404,21 @@
Device details
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
diff --git a/management/src/main/resources/static/freebees_webdesign_6/request-edit.html b/management/src/main/resources/static/freebees_webdesign_6/request-edit.html index 287b56f..05dd881 100644 --- a/management/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/management/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -458,6 +458,21 @@
Device details
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
From 024686ef75101546ae4f31a741b20fb85847a7dc Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 30 Oct 2023 14:47:45 +0200 Subject: [PATCH 044/132] RD: Added StatusController to report current application status (currently returns 'OK') --- .../resource/discovery/StatusController.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 management/src/main/java/eu/nebulous/resource/discovery/StatusController.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/StatusController.java b/management/src/main/java/eu/nebulous/resource/discovery/StatusController.java new file mode 100644 index 0000000..2158ff8 --- /dev/null +++ b/management/src/main/java/eu/nebulous/resource/discovery/StatusController.java @@ -0,0 +1,14 @@ +package eu.nebulous.resource.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +public class StatusController { + @GetMapping(value = "/status") + public String status() { + return "OK"; + } +} From 7ad2cf02b9850b6991b358158fdc187a418f475f Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 30 Oct 2023 14:54:29 +0200 Subject: [PATCH 045/132] RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly --- management/pom.xml | 10 ++++++++-- management/src/main/resources/application.yml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/management/pom.xml b/management/pom.xml index 0f50430..6340b3d 100644 --- a/management/pom.xml +++ b/management/pom.xml @@ -18,7 +18,8 @@ 17 - ${project.groupId}:${project.version} + + resource-discovery:${project.version} @@ -97,7 +98,12 @@ - ${imageName} + + ${imageName} + + C.UTF-8 + + diff --git a/management/src/main/resources/application.yml b/management/src/main/resources/application.yml index 0c6ee01..3683668 100644 --- a/management/src/main/resources/application.yml +++ b/management/src/main/resources/application.yml @@ -1,7 +1,7 @@ application.version: 1.0.0 spring.output.ansi.enabled: ALWAYS -spring.web.resources.static-locations: file:resource-discovery/management/src/main/resources/static/freebees_webdesign_6 +spring.web.resources.static-locations: file:resource-discovery/management/src/main/resources/static/freebees_webdesign_6, classpath:/static/freebees_webdesign_6/ #security.ignored: /** #security.basic.enable: false From 8960460dbbe2763bcf1cbb510bbe40c6a2248625 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 30 Oct 2023 15:13:02 +0200 Subject: [PATCH 046/132] RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version --- management/pom.xml | 11 +++++------ management/src/main/resources/application.yml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/management/pom.xml b/management/pom.xml index 6340b3d..01a052d 100644 --- a/management/pom.xml +++ b/management/pom.xml @@ -10,16 +10,15 @@ - eu.nebulous.resource-discovery - management + eu.nebulous.resource-management + resource-discovery 1.0.0-SNAPSHOT - Resource discovery management service - Nebulous resource discovery management service + Resource discovery service + Nebulous resource discovery service 17 - - resource-discovery:${project.version} + ${project.artifactId}:${project.version} diff --git a/management/src/main/resources/application.yml b/management/src/main/resources/application.yml index 3683668..ec64793 100644 --- a/management/src/main/resources/application.yml +++ b/management/src/main/resources/application.yml @@ -1,5 +1,5 @@ -application.version: 1.0.0 +application.version: @project.version@ spring.output.ansi.enabled: ALWAYS spring.web.resources.static-locations: file:resource-discovery/management/src/main/resources/static/freebees_webdesign_6, classpath:/static/freebees_webdesign_6/ From 6969d68bc1fadef773cb71f443858b853d227ef7 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 30 Oct 2023 15:21:29 +0200 Subject: [PATCH 047/132] RD: Renamed 'management' module to 'resource-discovery' --- {management => resource-discovery}/.gitignore | 0 .../.mvn/wrapper/maven-wrapper.jar | Bin .../.mvn/wrapper/maven-wrapper.properties | 0 {management => resource-discovery}/mvnw | 0 {management => resource-discovery}/mvnw.cmd | 0 {management => resource-discovery}/pom.xml | 0 .../discovery/ResourceDiscoveryApplication.java | 0 .../resource/discovery/ResourceDiscoveryConfig.java | 0 .../discovery/ResourceDiscoveryProperties.java | 0 .../nebulous/resource/discovery/SecurityConfig.java | 0 .../resource/discovery/StatusController.java | 0 .../resource/discovery/common/BrokerUtil.java | 0 .../resource/discovery/common/DeviceLocation.java | 0 .../resource/discovery/common/EncryptionUtil.java | 0 .../resource/discovery/common/REQUEST_TYPE.java | 0 .../resource/discovery/monitor/DeviceProcessor.java | 0 .../ArchivedDeviceManagementController.java | 0 .../controller/DeviceManagementController.java | 0 .../discovery/monitor/model/ArchivedDevice.java | 0 .../resource/discovery/monitor/model/Device.java | 0 .../discovery/monitor/model/DeviceException.java | 0 .../discovery/monitor/model/DeviceMetrics.java | 0 .../discovery/monitor/model/DeviceStatus.java | 0 .../discovery/monitor/model/DeviceStatusUpdate.java | 0 .../repository/ArchivedDeviceRepository.java | 0 .../monitor/repository/DeviceRepository.java | 0 .../monitor/service/AbstractMonitorService.java | 0 .../monitor/service/DeviceConversionService.java | 0 .../service/DeviceLifeCycleRequestService.java | 0 .../service/DeviceLifeCycleResponseService.java | 0 .../monitor/service/DeviceManagementService.java | 0 .../service/DeviceMetricsMonitorService.java | 0 .../monitor/service/DeviceStatusMonitorService.java | 0 .../service/UnknownDeviceRegistrationService.java | 0 .../registration/IRegistrationRequestProcessor.java | 0 .../registration/RegistrationRequestProcessor.java | 0 ...egistrationRequestService_SampleDataCreator.java | 0 .../controller/RegistrationRequestController.java | 0 .../model/ArchivedRegistrationRequest.java | 0 .../discovery/registration/model/Device.java | 0 .../registration/model/RegistrationRequest.java | 0 .../model/RegistrationRequestException.java | 0 .../model/RegistrationRequestHistoryEntry.java | 0 .../model/RegistrationRequestStatus.java | 0 .../ArchivedRegistrationRequestRepository.java | 0 .../InMemoryRegistrationRequestRepository.java | 0 .../repository/RegistrationRequestRepository.java | 0 .../RegistrationRequestConversionService.java | 0 .../service/RegistrationRequestService.java | 0 .../src/main/resources/application.yml | 0 .../src/main/resources/banner.txt | 0 .../freebees_webdesign_6/archived-device-view.html | 0 .../freebees_webdesign_6/archived-request-view.html | 0 .../static/freebees_webdesign_6/archived.html | 0 .../static/freebees_webdesign_6/css/style.css | 0 .../static/freebees_webdesign_6/css/style.css.map | 0 .../static/freebees_webdesign_6/device-view.html | 0 .../static/freebees_webdesign_6/devices.html | 0 .../static/freebees_webdesign_6/img/arrow-left.svg | 0 .../freebees_webdesign_6/img/arrow-right-color.svg | 0 .../static/freebees_webdesign_6/img/arrow-right.svg | 0 .../static/freebees_webdesign_6/img/circle.svg | 0 .../freebees_webdesign_6/img/icon/Group 1802.svg | 0 .../freebees_webdesign_6/img/icon/Group 1953.svg | 0 .../freebees_webdesign_6/img/icon/Group 1954.svg | 0 .../freebees_webdesign_6/img/icon/Group 1955.svg | 0 .../img/nebulous-logo-basic.png | Bin .../img/nebulous-logo-white.png | Bin .../static/freebees_webdesign_6/img/user-icon.png | Bin .../static/freebees_webdesign_6/index.html | 0 .../static/freebees_webdesign_6/js/addshadow.js | 0 .../static/freebees_webdesign_6/request-edit.html | 0 .../static/freebees_webdesign_6/requests.html | 0 .../static/freebees_webdesign_6/sass/_colors.scss | 0 .../static/freebees_webdesign_6/sass/style.scss | 0 .../src/main/resources/static/index.html | 0 .../ResourceManagementApplicationTests.java | 0 77 files changed, 0 insertions(+), 0 deletions(-) rename {management => resource-discovery}/.gitignore (100%) rename {management => resource-discovery}/.mvn/wrapper/maven-wrapper.jar (100%) rename {management => resource-discovery}/.mvn/wrapper/maven-wrapper.properties (100%) rename {management => resource-discovery}/mvnw (100%) rename {management => resource-discovery}/mvnw.cmd (100%) rename {management => resource-discovery}/pom.xml (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryApplication.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryConfig.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/StatusController.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/ArchivedDevice.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceException.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceMetrics.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatusUpdate.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/repository/ArchivedDeviceRepository.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/repository/DeviceRepository.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceConversionService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/IRegistrationRequestProcessor.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/ArchivedRegistrationRequest.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestException.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestHistoryEntry.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestStatus.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/repository/ArchivedRegistrationRequestRepository.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/repository/InMemoryRegistrationRequestRepository.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/repository/RegistrationRequestRepository.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestConversionService.java (100%) rename {management => resource-discovery}/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java (100%) rename {management => resource-discovery}/src/main/resources/application.yml (100%) rename {management => resource-discovery}/src/main/resources/banner.txt (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/archived-device-view.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/archived-request-view.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/archived.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/css/style.css (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/css/style.css.map (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/device-view.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/devices.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/arrow-left.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/arrow-right-color.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/arrow-right.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/circle.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1802.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1953.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1954.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1955.svg (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-basic.png (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-white.png (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/img/user-icon.png (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/index.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/js/addshadow.js (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/request-edit.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/requests.html (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/sass/_colors.scss (100%) rename {management => resource-discovery}/src/main/resources/static/freebees_webdesign_6/sass/style.scss (100%) rename {management => resource-discovery}/src/main/resources/static/index.html (100%) rename {management => resource-discovery}/src/test/java/eu/nebulous/resource/discovery/registration/ResourceManagementApplicationTests.java (100%) diff --git a/management/.gitignore b/resource-discovery/.gitignore similarity index 100% rename from management/.gitignore rename to resource-discovery/.gitignore diff --git a/management/.mvn/wrapper/maven-wrapper.jar b/resource-discovery/.mvn/wrapper/maven-wrapper.jar similarity index 100% rename from management/.mvn/wrapper/maven-wrapper.jar rename to resource-discovery/.mvn/wrapper/maven-wrapper.jar diff --git a/management/.mvn/wrapper/maven-wrapper.properties b/resource-discovery/.mvn/wrapper/maven-wrapper.properties similarity index 100% rename from management/.mvn/wrapper/maven-wrapper.properties rename to resource-discovery/.mvn/wrapper/maven-wrapper.properties diff --git a/management/mvnw b/resource-discovery/mvnw similarity index 100% rename from management/mvnw rename to resource-discovery/mvnw diff --git a/management/mvnw.cmd b/resource-discovery/mvnw.cmd similarity index 100% rename from management/mvnw.cmd rename to resource-discovery/mvnw.cmd diff --git a/management/pom.xml b/resource-discovery/pom.xml similarity index 100% rename from management/pom.xml rename to resource-discovery/pom.xml diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryApplication.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryApplication.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryApplication.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryApplication.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryConfig.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryConfig.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryConfig.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryConfig.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/StatusController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/StatusController.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/StatusController.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/StatusController.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/BrokerUtil.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/DeviceLocation.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/EncryptionUtil.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/common/REQUEST_TYPE.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/ArchivedDeviceManagementController.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/ArchivedDevice.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/ArchivedDevice.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/ArchivedDevice.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/ArchivedDevice.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceException.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceException.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceException.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceException.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceMetrics.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceMetrics.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceMetrics.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceMetrics.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatusUpdate.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatusUpdate.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatusUpdate.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatusUpdate.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/repository/ArchivedDeviceRepository.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/repository/ArchivedDeviceRepository.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/repository/ArchivedDeviceRepository.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/repository/ArchivedDeviceRepository.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/repository/DeviceRepository.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/repository/DeviceRepository.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/repository/DeviceRepository.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/repository/DeviceRepository.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceConversionService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceConversionService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceConversionService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceConversionService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleResponseService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceStatusMonitorService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/IRegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/IRegistrationRequestProcessor.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/IRegistrationRequestProcessor.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/IRegistrationRequestProcessor.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/ArchivedRegistrationRequest.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/ArchivedRegistrationRequest.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/ArchivedRegistrationRequest.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/ArchivedRegistrationRequest.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestException.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestException.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestException.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestException.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestHistoryEntry.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestHistoryEntry.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestHistoryEntry.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestHistoryEntry.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestStatus.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestStatus.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestStatus.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequestStatus.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/repository/ArchivedRegistrationRequestRepository.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/ArchivedRegistrationRequestRepository.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/repository/ArchivedRegistrationRequestRepository.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/ArchivedRegistrationRequestRepository.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/repository/InMemoryRegistrationRequestRepository.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/InMemoryRegistrationRequestRepository.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/repository/InMemoryRegistrationRequestRepository.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/InMemoryRegistrationRequestRepository.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/repository/RegistrationRequestRepository.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/RegistrationRequestRepository.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/repository/RegistrationRequestRepository.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/repository/RegistrationRequestRepository.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestConversionService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestConversionService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestConversionService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestConversionService.java diff --git a/management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java similarity index 100% rename from management/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java rename to resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java diff --git a/management/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml similarity index 100% rename from management/src/main/resources/application.yml rename to resource-discovery/src/main/resources/application.yml diff --git a/management/src/main/resources/banner.txt b/resource-discovery/src/main/resources/banner.txt similarity index 100% rename from management/src/main/resources/banner.txt rename to resource-discovery/src/main/resources/banner.txt diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/archived-device-view.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/archived-request-view.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/archived.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/archived.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/css/style.css b/resource-discovery/src/main/resources/static/freebees_webdesign_6/css/style.css similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/css/style.css rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/css/style.css diff --git a/management/src/main/resources/static/freebees_webdesign_6/css/style.css.map b/resource-discovery/src/main/resources/static/freebees_webdesign_6/css/style.css.map similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/css/style.css.map rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/css/style.css.map diff --git a/management/src/main/resources/static/freebees_webdesign_6/device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/device-view.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/devices.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/devices.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/arrow-left.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-left.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/arrow-left.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-left.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/arrow-right-color.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-right-color.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/arrow-right-color.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-right-color.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/arrow-right.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-right.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/arrow-right.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/arrow-right.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/circle.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/circle.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/circle.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/circle.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1802.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1802.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1802.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1802.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1953.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1953.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1953.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1953.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1954.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1954.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1954.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1954.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1955.svg b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1955.svg similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1955.svg rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/icon/Group 1955.svg diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-basic.png b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-basic.png similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-basic.png rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-basic.png diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-white.png b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-white.png similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-white.png rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/nebulous-logo-white.png diff --git a/management/src/main/resources/static/freebees_webdesign_6/img/user-icon.png b/resource-discovery/src/main/resources/static/freebees_webdesign_6/img/user-icon.png similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/img/user-icon.png rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/img/user-icon.png diff --git a/management/src/main/resources/static/freebees_webdesign_6/index.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/index.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/js/addshadow.js b/resource-discovery/src/main/resources/static/freebees_webdesign_6/js/addshadow.js similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/js/addshadow.js rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/js/addshadow.js diff --git a/management/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/request-edit.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/requests.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/requests.html rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html diff --git a/management/src/main/resources/static/freebees_webdesign_6/sass/_colors.scss b/resource-discovery/src/main/resources/static/freebees_webdesign_6/sass/_colors.scss similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/sass/_colors.scss rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/sass/_colors.scss diff --git a/management/src/main/resources/static/freebees_webdesign_6/sass/style.scss b/resource-discovery/src/main/resources/static/freebees_webdesign_6/sass/style.scss similarity index 100% rename from management/src/main/resources/static/freebees_webdesign_6/sass/style.scss rename to resource-discovery/src/main/resources/static/freebees_webdesign_6/sass/style.scss diff --git a/management/src/main/resources/static/index.html b/resource-discovery/src/main/resources/static/index.html similarity index 100% rename from management/src/main/resources/static/index.html rename to resource-discovery/src/main/resources/static/index.html diff --git a/management/src/test/java/eu/nebulous/resource/discovery/registration/ResourceManagementApplicationTests.java b/resource-discovery/src/test/java/eu/nebulous/resource/discovery/registration/ResourceManagementApplicationTests.java similarity index 100% rename from management/src/test/java/eu/nebulous/resource/discovery/registration/ResourceManagementApplicationTests.java rename to resource-discovery/src/test/java/eu/nebulous/resource/discovery/registration/ResourceManagementApplicationTests.java From 096122958eeeee9057767e1a80664c2d4f9efad4 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 2 Nov 2023 14:24:31 +0200 Subject: [PATCH 048/132] RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore --- resource-discovery/.dockerignore | 6 ++++ resource-discovery/Dockerfile | 48 ++++++++++++++++++++++++++++ resource-discovery/pom.xml | 12 ++++++- resource-discovery/run.sh | 55 ++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 resource-discovery/.dockerignore create mode 100644 resource-discovery/Dockerfile create mode 100644 resource-discovery/run.sh diff --git a/resource-discovery/.dockerignore b/resource-discovery/.dockerignore new file mode 100644 index 0000000..218dadc --- /dev/null +++ b/resource-discovery/.dockerignore @@ -0,0 +1,6 @@ +.mvn +src/ +target/ +!target/resource-discovery-*.jar +mvn* +pom.xml diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile new file mode 100644 index 0000000..fe684e5 --- /dev/null +++ b/resource-discovery/Dockerfile @@ -0,0 +1,48 @@ + +ARG BUILDER_IMAGE=eclipse-temurin:17.0.8.1_1-jre +ARG RUN_IMAGE=eclipse-temurin:17.0.8.1_1-jre + +# ----------------- Builder image ----------------- +FROM $BUILDER_IMAGE as rd-builder +WORKDIR /app +COPY target/resource-discovery-*.jar . +RUN java -Djarmode=layertools -jar resource-discovery-*.jar extract + +# ----------------- Runtime image ----------------- +FROM $RUN_IMAGE + +# Setup environment +ENV BASEDIR /opt/resource-discovery +ENV RD_HOME ${BASEDIR} + +# Install required and optional packages +RUN apt-get update && apt-get install -y \ + dumb-init \ + netcat \ + vim \ + iputils-ping \ + && rm -rf /var/lib/apt/lists/* + +# Add RD user +ARG RD_USER=rd +RUN mkdir ${RD_HOME} ; \ + addgroup ${RD_USER} ; \ + adduser --home ${RD_HOME} --no-create-home --ingroup ${RD_USER} --disabled-password ${RD_USER} ; \ + chown ${RD_USER}:${RD_USER} ${RD_HOME} + +# Set User and Workdir +USER ${RD_USER} +WORKDIR ${BASEDIR} + +# Copy files from builder container +COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/dependencies ${BASEDIR} +COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/spring-boot-loader ${BASEDIR} +COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/snapshot-dependencies ${BASEDIR} +COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/application ${BASEDIR} + +# Copy entrypoint script +COPY --chown=${RD_USER}:${RD_USER} run.sh ${BASEDIR} + +EXPOSE 8080 + +ENTRYPOINT ["dumb-init", "./run.sh"] \ No newline at end of file diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index 01a052d..bf395bb 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -85,6 +85,16 @@ + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/resource-discovery/run.sh b/resource-discovery/run.sh new file mode 100644 index 0000000..1dc69b8 --- /dev/null +++ b/resource-discovery/run.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Change directory to RD home +PREVWORKDIR=`pwd` +BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +cd ${BASEDIR} + +# Read JASYPT password (decrypts encrypted configuration settings) +#if [[ -z "$JASYPT_PASSWORD" ]]; then +# printf "Configuration Password: " +# read -s JASYPT_PASSWORD +# export JASYPT_PASSWORD +#fi +# Use this online service to encrypt/decrypt passwords: +# https://www.devglan.com/online-tools/jasypt-online-encryption-decryption + + +# Setup TERM & INT signal handler +trap 'echo "Signaling server to exit"; kill -TERM "${pid}"; wait "${pid}"; ' SIGTERM SIGINT + +# Set JRE command and options +JRE=/opt/java/openjdk/bin/java +#JAVA_ADD_OPENS="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.base/sun.nio.cs=ALL-UNNAMED --add-opens java.base/java.nio.charset=ALL-UNNAMED" + +# Set shell encoding to UTF-8 (in order to display banner correctly) +export LANG=C.UTF-8 + +# Print basic env. info +echo "--------------------------------------------------------------------------------" +echo "Env. info:" +echo "LANG: ${LANG}" +echo "USER: $( whoami )" +echo "IP address: `hostname -I`" +echo "--------------------------------------------------------------------------------" +echo "JRE:" +${JRE} -version +echo "--------------------------------------------------------------------------------" +echo "Starting Resource Discovery server..." + +# Run RD server +${JRE} \ + $JAVA_OPTS \ + $JAVA_ADD_OPENS \ + -Djasypt.encryptor.password=$JASYPT_PASSWORD \ + -Djava.security.egd=file:/dev/urandom \ + org.springframework.boot.loader.JarLauncher \ + $* & + +# Get PID and wait it to exit +pid=$! +echo "Pid: $pid" +wait $pid +echo "Server exited" + +cd $PREVWORKDIR From db290cf7c128c294badedf4ac397e7a7365d3558 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 2 Nov 2023 15:41:24 +0200 Subject: [PATCH 049/132] RD: Modified base images in Dockerfile --- resource-discovery/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index fe684e5..c7266ab 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -1,6 +1,6 @@ -ARG BUILDER_IMAGE=eclipse-temurin:17.0.8.1_1-jre -ARG RUN_IMAGE=eclipse-temurin:17.0.8.1_1-jre +ARG BUILDER_IMAGE=docker.io/eclipse-temurin:17.0.8.1_1-jre +ARG RUN_IMAGE=docker.io/eclipse-temurin:17.0.8.1_1-jre # ----------------- Builder image ----------------- FROM $BUILDER_IMAGE as rd-builder From 1c25f5e0a0dd5d747e819a6d0da9548876e7af5a Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 3 Nov 2023 09:05:48 +0200 Subject: [PATCH 050/132] RD: Modified Dockerfile in order to both compile the app and build the Docker image --- resource-discovery/.dockerignore | 5 ----- resource-discovery/Dockerfile | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/resource-discovery/.dockerignore b/resource-discovery/.dockerignore index 218dadc..2f7896d 100644 --- a/resource-discovery/.dockerignore +++ b/resource-discovery/.dockerignore @@ -1,6 +1 @@ -.mvn -src/ target/ -!target/resource-discovery-*.jar -mvn* -pom.xml diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index c7266ab..9781cc2 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -1,12 +1,16 @@ -ARG BUILDER_IMAGE=docker.io/eclipse-temurin:17.0.8.1_1-jre -ARG RUN_IMAGE=docker.io/eclipse-temurin:17.0.8.1_1-jre +ARG BUILDER_IMAGE=docker.io/library/maven:3.9.5-eclipse-temurin-17 +ARG RUN_IMAGE=docker.io/library/eclipse-temurin:17.0.8.1_1-jre # ----------------- Builder image ----------------- FROM $BUILDER_IMAGE as rd-builder -WORKDIR /app -COPY target/resource-discovery-*.jar . -RUN java -Djarmode=layertools -jar resource-discovery-*.jar extract +ENV BASEDIR /app +WORKDIR ${BASEDIR} +COPY src ${BASEDIR}/src +COPY pom.xml ${BASEDIR}/ +COPY run.sh ${BASEDIR}/ +RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install +RUN java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract # ----------------- Runtime image ----------------- FROM $RUN_IMAGE @@ -39,9 +43,7 @@ COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/dependencies COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/spring-boot-loader ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/snapshot-dependencies ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/application ${BASEDIR} - -# Copy entrypoint script -COPY --chown=${RD_USER}:${RD_USER} run.sh ${BASEDIR} +COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/run.sh ${BASEDIR} EXPOSE 8080 From 94260c4522ac605ce077836d7fb0fb1cd4898a5b Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 3 Nov 2023 15:01:43 +0200 Subject: [PATCH 051/132] RD: Improved Dockerfile --- resource-discovery/Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index 9781cc2..a361279 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -9,8 +9,8 @@ WORKDIR ${BASEDIR} COPY src ${BASEDIR}/src COPY pom.xml ${BASEDIR}/ COPY run.sh ${BASEDIR}/ -RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install -RUN java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract +RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install && \ + java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract # ----------------- Runtime image ----------------- FROM $RUN_IMAGE @@ -20,18 +20,19 @@ ENV BASEDIR /opt/resource-discovery ENV RD_HOME ${BASEDIR} # Install required and optional packages -RUN apt-get update && apt-get install -y \ - dumb-init \ - netcat \ - vim \ - iputils-ping \ - && rm -rf /var/lib/apt/lists/* +RUN wget --progress=dot:giga -O /usr/local/bin/dumb-init \ + https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 && \ + chmod +x /usr/local/bin/dumb-init + +#RUN apt-get update && \ +# apt-get install -y netcat vim iputils-ping && \ +# rm -rf /var/lib/apt/lists/* # Add RD user ARG RD_USER=rd -RUN mkdir ${RD_HOME} ; \ - addgroup ${RD_USER} ; \ - adduser --home ${RD_HOME} --no-create-home --ingroup ${RD_USER} --disabled-password ${RD_USER} ; \ +RUN mkdir ${RD_HOME} && \ + addgroup ${RD_USER} && \ + adduser --home ${RD_HOME} --no-create-home --ingroup ${RD_USER} --disabled-password ${RD_USER} && \ chown ${RD_USER}:${RD_USER} ${RD_HOME} # Set User and Workdir From 7800bde1e10366ea6d6378361849ebf305d38545 Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 8 Nov 2023 10:25:17 +0200 Subject: [PATCH 052/132] RD: Improved Dockerfile --- resource-discovery/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index a361279..8bc3766 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -45,6 +45,7 @@ COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/spring-boot-loader COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/snapshot-dependencies ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/application ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/run.sh ${BASEDIR} +RUN chmod +x run.sh EXPOSE 8080 From cb9b3f45be1c9aa9dc2f3a03727841b1fce049e3 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 15 Jan 2024 13:22:02 +0200 Subject: [PATCH 053/132] RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. --- .../.mvn/wrapper/maven-wrapper.properties | 2 +- resource-discovery/Dockerfile | 10 ++++++---- resource-discovery/pom.xml | 10 +++++----- .../resource/discovery/monitor/model/Device.java | 1 + .../registration/model/RegistrationRequest.java | 7 +++---- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/resource-discovery/.mvn/wrapper/maven-wrapper.properties b/resource-discovery/.mvn/wrapper/maven-wrapper.properties index 2e76e18..e70e7bc 100644 --- a/resource-discovery/.mvn/wrapper/maven-wrapper.properties +++ b/resource-discovery/.mvn/wrapper/maven-wrapper.properties @@ -1,2 +1,2 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.4/apache-maven-3.9.4-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index 8bc3766..2b51053 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -1,9 +1,11 @@ -ARG BUILDER_IMAGE=docker.io/library/maven:3.9.5-eclipse-temurin-17 -ARG RUN_IMAGE=docker.io/library/eclipse-temurin:17.0.8.1_1-jre +ARG BUILDER_IMAGE=docker.io/library/maven +ARG BUILDER_IMAGE_TAG=3.9.6-eclipse-temurin-21 +ARG RUN_IMAGE=docker.io/library/eclipse-temurin +ARG RUN_IMAGE=21.0.1_12-jre # ----------------- Builder image ----------------- -FROM $BUILDER_IMAGE as rd-builder +FROM $BUILDER_IMAGE:$BUILDER_IMAGE_TAG as rd-builder ENV BASEDIR /app WORKDIR ${BASEDIR} COPY src ${BASEDIR}/src @@ -13,7 +15,7 @@ RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install && \ java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract # ----------------- Runtime image ----------------- -FROM $RUN_IMAGE +FROM $RUN_IMAGE:$RUN_IMAGE_TAG # Setup environment ENV BASEDIR /opt/resource-discovery diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index bf395bb..6a46ad4 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.1.4 + 3.2.1 @@ -17,7 +17,7 @@ Nebulous resource discovery service - 17 + 21 ${project.artifactId}:${project.version} @@ -45,12 +45,12 @@ org.projectlombok lombok - 1.18.26 + 1.18.30 org.apache.commons commons-lang3 - 3.13.0 + 3.14.0 @@ -79,7 +79,7 @@ org.yaml snakeyaml - 2.0 + 2.2 diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index 7aa5ded..32d9fb1 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -40,6 +40,7 @@ public class Device { private String nodeReference; @Setter(AccessLevel.NONE) + @Builder.Default private List messages = new ArrayList<>(); private DeviceStatusUpdate statusUpdate; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java index 210cf10..664c741 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/RegistrationRequest.java @@ -1,9 +1,6 @@ package eu.nebulous.resource.discovery.registration.model; -import lombok.AccessLevel; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.*; import lombok.experimental.SuperBuilder; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; @@ -25,9 +22,11 @@ public class RegistrationRequest { private Instant lastUpdateDate; private Instant archiveDate; private RegistrationRequestStatus status; + @Builder.Default private List history = new ArrayList<>(); private String nodeReference; @Setter(AccessLevel.NONE) + @Builder.Default private List messages = new ArrayList<>(); // Required in order BeanUtils.copyProperties() to also copy this From ef4e8407d3d4164556fa8197fd6955b854fe6f9f Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 6 Mar 2024 22:11:08 +0200 Subject: [PATCH 054/132] RD: Fixed Dockerfile --- resource-discovery/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index 2b51053..93d40ff 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -2,7 +2,7 @@ ARG BUILDER_IMAGE=docker.io/library/maven ARG BUILDER_IMAGE_TAG=3.9.6-eclipse-temurin-21 ARG RUN_IMAGE=docker.io/library/eclipse-temurin -ARG RUN_IMAGE=21.0.1_12-jre +ARG RUN_IMAGE_TAG=21.0.1_12-jre # ----------------- Builder image ----------------- FROM $BUILDER_IMAGE:$BUILDER_IMAGE_TAG as rd-builder From d8dc54de0dab60ca361b29750dd92ae208af9b19 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Mon, 11 Mar 2024 15:56:13 +0200 Subject: [PATCH 055/132] Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 --- resource-discovery/pom.xml | 25 ++ .../ResourceDiscoveryProperties.java | 6 + .../broker_communication/BrokerPublisher.java | 109 +++++++ .../BrokerSubscriber.java | 193 +++++++++++ .../BrokerSubscriptionDetails.java | 69 ++++ .../CustomConnectorHandler.java | 24 ++ .../ExtendedConnector.java | 99 ++++++ .../broker_communication/SALCommunicator.java | 302 ++++++++++++++++++ .../SynchronousBrokerPublisher.java | 106 ++++++ .../discovery/monitor/DeviceProcessor.java | 10 + .../DeviceManagementController.java | 4 + .../UnknownDeviceRegistrationService.java | 10 +- .../RegistrationRequestProcessor.java | 3 + .../service/SALRegistrationService.java | 89 ++++++ .../src/main/resources/application.yml | 4 + 15 files changed, 1052 insertions(+), 1 deletion(-) create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriptionDetails.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index 6a46ad4..c6df3fb 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -22,6 +22,14 @@ + + + eu.nebulouscloud + exn-connector-java + 1.0-SNAPSHOT + + + org.springframework.boot spring-boot-starter-security @@ -72,6 +80,11 @@ com.fasterxml.jackson.datatype jackson-datatype-jsr310 + + gr.ntua.imu.morphemic + amq-message-java-library + 4.0.0-SNAPSHOT + @@ -93,6 +106,18 @@ + + + + maven-central + https://repo1.maven.org/maven2/ + + + nexus-nebulous + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + + +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html index 16d70ba..ea9e61d 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html @@ -300,6 +300,13 @@
Device details
+ +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html index 37b9a9c..c39924e 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -404,6 +404,13 @@
Device details
+ +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html index 05dd881..20266ce 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -458,6 +458,13 @@
Device details
+ +
+ +
+ +
+
From 8170b97feb59758c7a1645a0fe5115d0dff51eb2 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 14 Mar 2024 18:57:06 +0200 Subject: [PATCH 058/132] RD: Added two TODOs --- .../discovery/monitor/service/DeviceManagementService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index 8bf7802..bcf4a67 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -192,6 +192,7 @@ public void archiveDeviceBySystem(String id) { result.get().setArchiveDate(Instant.now()); archivedDeviceRepository.save(deviceConversionService.toArchivedDevice(result.get())); deviceRepository.delete(result.get()); + //XXX:TODO: Send notification to SAL to deregister Device } public void unarchiveDevice(String id, Map credentials) { @@ -208,6 +209,7 @@ public void unarchiveDevice(String id, Map credentials) { restoredDevice.setPublicKey(credentials.get("publicKey").toCharArray()); deviceRepository.save(restoredDevice); archivedDeviceRepository.deleteById(result.get().getId()); + //XXX:TODO: Send notification to SAL to re-register Device } private void checkCredentials(String id, Map credentials) { From 38ea2cb7e1d4d1bce06cd4a6077307122c8ba8c5 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 14 Mar 2024 19:51:45 +0200 Subject: [PATCH 059/132] RD: Deactivated UnknownDeviceRegistrationService --- .../monitor/service/UnknownDeviceRegistrationService.java | 4 ++-- .../discovery/registration/RegistrationRequestProcessor.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index 7bb4fc1..6d32b0c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -13,14 +13,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.scheduling.TaskScheduler; -import org.springframework.stereotype.Service; +//import org.springframework.stereotype.Service; import java.time.Duration; import java.time.Instant; import java.util.*; @Slf4j -@Service +//@Service public class UnknownDeviceRegistrationService extends AbstractMonitorService { private final static List MONITORED_REQUEST_TYPES = List.of( REQUEST_TYPE.INFO.name(), diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 825aa1c..ccae7d3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -258,7 +258,7 @@ private void processResponse(@NonNull Map response) { String ipAddress = registrationRequest.getDevice().getIpAddress(); boolean isError = false; if (StringUtils.isNotBlank(deviceIpAddress) && ! StringUtils.equals(ipAddress, deviceIpAddress)) { - String mesg = String.format("Device IP address in RESPONSE does not match with that in the request: id=%s, ip-address-response=%s != ip-address-in-request%s", requestId, deviceIpAddress, ipAddress); + String mesg = String.format("Device IP address in RESPONSE does not match with that in the request: id=%s, ip-address-response=%s != ip-address-in-request=%s", requestId, deviceIpAddress, ipAddress); log.warn("processResponse: {}", mesg); registrationRequest.getMessages().add(mesg); isError = true; From d807458937433c34422b8e696fe17e595ca646ae Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Fri, 22 Mar 2024 23:09:12 +0200 Subject: [PATCH 060/132] Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 --- resource-discovery/pom.xml | 52 ++++++++- .../broker_communication/BrokerPublisher.java | 25 +++-- .../BrokerSubscriber.java | 23 ++-- .../ExtendedConnector.java | 10 +- .../broker_communication/JsonFileParser.java | 27 +++++ .../broker_communication/SALCommunicator.java | 101 +++++++++++++----- .../SynchronousBrokerPublisher.java | 52 ++++++--- .../service/SALRegistrationService.java | 19 ++-- 8 files changed, 232 insertions(+), 77 deletions(-) create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/JsonFileParser.java diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index c6df3fb..ff87f62 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -80,11 +80,57 @@ com.fasterxml.jackson.datatype jackson-datatype-jsr310 + + + com.googlecode.json-simple + json-simple + 1.1.1 + + + + commons-io + commons-io + 2.15.1 + + + + org.apache.httpcomponents + httpcore + 4.4.16 + + + org.apache.httpcomponents + httpclient + 4.5.14 + - gr.ntua.imu.morphemic - amq-message-java-library - 4.0.0-SNAPSHOT + org.apache.httpcomponents + httpmime + 4.5.14 + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-logging + + + + diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 7ecf73d..9713117 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -7,15 +7,13 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.*; +@Slf4j public class BrokerPublisher { public static String EMPTY=""; private static HashMap> broker_and_topics_to_publish_to = new HashMap<>(); @@ -51,8 +49,10 @@ public BrokerPublisher(String topic, String broker_ip, String brokerUsername, St if (publisher_configuration_changed){ // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ - Logger.getGlobal().log(Level.INFO,"Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); - active_connector.stop(new ArrayList<>(),publishers); + log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); + if (active_connector!=null) { + active_connector.stop(new ArrayList<>(), publishers); + } publishers.clear(); for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ //ArrayList publishers = new ArrayList<>(); @@ -82,14 +82,12 @@ public BrokerPublisher(String topic, String broker_ip, String brokerUsername, St ) ); active_connector.start(); - //Logger.getGlobal().log(INFO,"Sending from EXTERIOR"); - //private_publisher_instance.send(new JSONObject()); } } //TODO The methods below assume that the only content to be sent is json-like - public void publish (String json_string_content, Iterable application_names){ + public void publish (String json_string_content, Collection application_names){ for (String application_name : application_names) { JSONParser parser = new JSONParser(); @@ -97,12 +95,13 @@ public void publish (String json_string_content, Iterable application_na try { json_object = (JSONObject) parser.parse(json_string_content); } catch (ParseException p) { - Logger.getGlobal().log(Level.WARNING, "Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); + log.warn( "Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); } if (private_publisher_instance != null) { private_publisher_instance.send(json_object); + log.info("Sent new message\n"+json_object.toJSONString()); } else { - Logger.getGlobal().log(Level.SEVERE, "Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + log.error( "Could not send message to AMQP broker, as the broker ip to be used has not been specified"); } } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java index bcce7c5..1e1f713 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java @@ -4,32 +4,31 @@ import eu.nebulouscloud.exn.core.Context; import eu.nebulouscloud.exn.core.Handler; import eu.nebulouscloud.exn.settings.StaticExnConfig; +import lombok.extern.slf4j.Slf4j; import org.apache.qpid.protonj2.client.Message; import org.json.simple.JSONValue; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; -import java.util.logging.Level; -import java.util.logging.Logger; import static eu.nebulous.resource.discovery.broker_communication.BrokerPublisher.EMPTY; -import static java.util.logging.Level.INFO; +@Slf4j public class BrokerSubscriber { private class MessageProcessingHandler extends Handler { private BrokerSubscriptionDetails broker_details; private static final BiFunction temporary_function = (Object o, Object o2) -> { //System.out.println(""); - Logger.getGlobal().log(INFO, "REPLACE_TEMPORARY_HANDLING_FUNCTIONALITY"); + log.info("REPLACE_TEMPORARY_HANDLING_FUNCTIONALITY"); return "IN_PROCESSING"; }; private BiFunction processing_function; @Override public void onMessage(String key, String address, Map body, Message message, Context context) { - Logger.getGlobal().log(INFO, "Handling message for address " + address); + log.info("Handling message for address " + address); processing_function.apply(broker_details, JSONValue.toJSONString(body)); } @@ -70,7 +69,7 @@ public BrokerSubscriber(String topic, String broker_ip, String brokerUsername, S } catch (Exception e) { String message = "Topic is " + topic + " broker ip is " + broker_ip + " broker username/pass are " + brokerUsername + "," + brokerPassword; - Logger.getGlobal().log(INFO, message); + log.info(message); throw new RuntimeException(e); } } @@ -98,11 +97,11 @@ public BrokerSubscriber(String topic, String broker_ip, String brokerUsername, S if (subscriber_configuration_changed) { Consumer current_consumer; if (application_name != null && !application_name.equals(EMPTY)) { //Create a consumer for one application - Logger.getGlobal().log(INFO, "APP level subscriber " + topic); + log.info("APP level subscriber " + topic); current_consumer = new Consumer(topic, topic, new MessageProcessingHandler(broker_details), application_name, true, true); } else { //Allow the consumer to get information from any publisher current_consumer = new Consumer(topic, topic, new MessageProcessingHandler(broker_details), true, true); - Logger.getGlobal().log(INFO, "HIGH level subscriber " + topic); + log.info("HIGH level subscriber " + topic); } active_consumers_per_topic_per_broker_ip.get(broker_ip).put(topic, current_consumer); @@ -152,7 +151,7 @@ private void remove_topic_from_broker_connector(String topic_key) { public int subscribe(BiFunction function, String application_name, AtomicBoolean stop_signal) { int exit_status = -1; - Logger.getGlobal().log(INFO, "ESTABLISHING SUBSCRIPTION for " + topic); + log.info("ESTABLISHING SUBSCRIPTION for " + topic); //First remove any leftover consumer if (active_consumers_per_topic_per_broker_ip.containsKey(broker_ip)) { active_consumers_per_topic_per_broker_ip.get(broker_ip).remove(topic); @@ -172,17 +171,17 @@ public int subscribe(BiFunction function, String application_name, AtomicBoolean active_consumers_per_topic_per_broker_ip.get(broker_ip).put(topic, new_consumer); add_topic_consumer_to_broker_connector(new_consumer); - Logger.getGlobal().log(INFO, "ESTABLISHED SUBSCRIPTION to topic " + topic); + log.info("ESTABLISHED SUBSCRIPTION to topic " + topic); synchronized (stop_signal) { while (!stop_signal.get()) { try { stop_signal.wait(); } catch (Exception e) { - Logger.getGlobal().log(Level.WARNING, e.toString() + " in thread " + Thread.currentThread().getName()); + log.warn( e.toString() + " in thread " + Thread.currentThread().getName()); break; } } - Logger.getGlobal().log(INFO, "Stopping subscription for broker " + broker_ip + " and topic " + topic + "at thread " + Thread.currentThread().getName()); + log.info("Stopping subscription for broker " + broker_ip + " and topic " + topic + "at thread " + Thread.currentThread().getName()); stop_signal.set(false); } active_consumers_per_topic_per_broker_ip.get(broker_ip).remove(topic); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java index 3dd5c09..1e09f3e 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java @@ -6,13 +6,13 @@ import eu.nebulouscloud.exn.core.Publisher; import eu.nebulouscloud.exn.handlers.ConnectorHandler; import eu.nebulouscloud.exn.settings.ExnConfig; +import lombok.extern.slf4j.Slf4j; import org.apache.catalina.util.CustomObjectInputStream; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +@Slf4j public class ExtendedConnector extends Connector { private CustomConnectorHandler handler; @@ -45,7 +45,7 @@ public void remove_consumer_with_key(String key) { Context context = ((CustomConnectorHandler)handler).getContext(); context.unregisterConsumer(key); }catch (ClassCastException c){ - Logger.getAnonymousLogger().log(Level.WARNING,"Could not unregister consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); + log.warn("Could not unregister consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); } } @@ -54,7 +54,7 @@ private void remove_publisher_with_key(String key) { Context context = ((CustomConnectorHandler)handler).getContext(); context.unregisterPublisher(key); }catch (ClassCastException c){ - Logger.getAnonymousLogger().log(Level.WARNING,"Could not unregister consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); + log.warn("Could not unregister consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); } } @@ -63,7 +63,7 @@ public void add_consumer(Consumer newConsumer) { try { ((CustomConnectorHandler)handler).getContext().registerConsumer(newConsumer); }catch (ClassCastException c){ - Logger.getAnonymousLogger().log(Level.WARNING,"Could not register consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); + log.warn("Could not register consumer, as the handler of the Connector it belongs to is not a CustomConnectorHandler"); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/JsonFileParser.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/JsonFileParser.java new file mode 100644 index 0000000..210d10b --- /dev/null +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/JsonFileParser.java @@ -0,0 +1,27 @@ +package eu.nebulous.resource.discovery.broker_communication; + + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import java.io.FileReader; + +public class JsonFileParser { + public static JSONObject parse(String file_name){ + JSONParser parser = new JSONParser(); + try { + Object obj = parser.parse(new FileReader(file_name)); + JSONObject jsonObject = (JSONObject) obj; + // Access properties of the parsed JSON object here + return jsonObject; + } catch (Exception e) { + e.printStackTrace(); + } + return new JSONObject(); + } + public static void main(String[] args) { + String currentDir = System.getProperty("user.dir"); + System.out.println("Current Directory: " + currentDir); + String parsed_file_string = parse("./src/main/java/eu/nebulous/resource/discovery/broker_communication/file.json").toString(); + System.out.println(parsed_file_string); + } +} diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 57cf0c4..10f28da 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.broker_communication; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -23,12 +24,12 @@ import java.util.HashMap; import java.util.Map; import java.util.Random; -import java.util.logging.Level; -import java.util.logging.Logger; + /** * Assuming that only one SALCommunicator will exist - otherwise some variables should have their characterization as 'static' be removed */ +@Slf4j public class SALCommunicator { private static String sal_host = "localhost"; @@ -50,7 +51,7 @@ public static String get_connection_id(String sal_host, int sal_port, String sal authentication_map.put("username",sal_username); authentication_map.put("password",sal_password); String sessionID = sendPOSTRequest("http://"+sal_host+":"+sal_port+"/sal/pagateway/connect", new HashMap<>(), authentication_map); - Logger.getGlobal().log(Level.INFO,"Retrieved session id "+sessionID); + log.info("Retrieved session id "+sessionID); return sessionID; } @@ -62,11 +63,11 @@ public static void main(String[] args) { //String contentType = "application/json"; String sessionID = get_connection_id("localhost",9000,"admin","admin"); - Logger.getGlobal().log(Level.INFO,"The session id is "+sessionID); + log.info("The session id is "+sessionID); ArrayList applications = get_running_applications(request_running_applications_REST(sessionID)); - Logger.getGlobal().log(Level.INFO,"The running apps are "+applications.toString()); + log.info("The running apps are "+applications.toString()); - register_devices("./src/main/java/eu/nebulous/resource/discovery/broker_communication/sal_device_registration_base_payload.json", sessionID, applications,"10.100.100","100.100.100.",10,10,10,"test12","test_provider","Athens","Greece",1); + register_devices("./src/main/resources/sal_device_registration_base_payload.json", sessionID, applications,"10.100.100","100.100.100.",10,10,10,"test12","test_provider","Athens","Greece",100); // Request 4 //String payload4 = "{\"key3\": \"value3\"}"; //sendRequest("https://api.example.com/endpoint3", sessionID, contentType, payload4); @@ -79,13 +80,13 @@ public static String request_running_applications_AMQP() { return null; } - private static String request_running_applications_REST(String sessionID) { + private static String request_running_applications_REST(String sessionID) { // Request 2 - Get available jobs String get_jobs_payload = "{\"sessionid\": \""+sessionID+"\"}"; HashMap session_id_headers = new HashMap<>(); session_id_headers.put("sessionid",sessionID); - Logger.getGlobal().log(Level.INFO,"Using temporary \"job\" endpoint to get the jobs from SAL..."); + log.info("Using temporary \"job\" endpoint to get the jobs from SAL..."); String get_jobs_string = sendGETRequest("http://localhost:9000/sal/job/",session_id_headers ); return get_jobs_string; } @@ -141,21 +142,73 @@ private static void register_devices(String request_body_file, String sessionID, } - public static String get_device_registration_json(String request_body_file,String internal_ip_address, String external_ip_address, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name) { - - JSONObject json = JsonFileParser.parse(request_body_file); - json.put("name", device_name); - ((JSONObject) ((JSONArray) json.get("ipAddresses")).get(0)).put("value", internal_ip_address); - ((JSONObject) ((JSONArray) json.get("ipAddresses")).get(1)).put("value", external_ip_address); - ((JSONObject) json.get("nodeProperties")).put("disk", disk_gb); - ((JSONObject) json.get("nodeProperties")).put("memory", ram_gb); - ((JSONObject) json.get("nodeProperties")).put("providerId", provider_id); - ((JSONObject) json.get("nodeProperties")).put("numberOfCores", cpu_cores); - ((JSONObject) ((JSONObject) json.get("nodeProperties")).get("geoLocation")).put("country", country_name); - ((JSONObject) ((JSONObject) json.get("nodeProperties")).get("geoLocation")).put("city", city_name); - ((JSONObject) ((JSONObject) json.get("nodeProperties")).get("geoLocation")).put("latitude", new Random().nextFloat(-90, 90)); - ((JSONObject) ((JSONObject) json.get("nodeProperties")).get("geoLocation")).put("longitude", new Random().nextFloat(-90, 90)); - return(json.toJSONString()); + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password) { + + JSONObject root_json_object = new JSONObject(); + JSONObject loginCredential = new JSONObject(); + JSONObject ipAddress1 = new JSONObject(); + JSONObject ipAddress2 = new JSONObject(); + JSONObject operatingSystem = new JSONObject(); + JSONObject geoLocation = new JSONObject(); + JSONObject nodeProperties = new JSONObject(); + + loginCredential.put("username", device_username); + loginCredential.put("password", device_password); + loginCredential.put("privateKey", ""); + + + ipAddress1.put("IpAddressType", "PUBLIC_IP"); + ipAddress1.put("IpVersion", "V4"); + ipAddress1.put("value", external_ip_address); + + ipAddress2.put("IpAddressType", "PRIVATE_IP"); + ipAddress2.put("IpVersion", "V4"); + ipAddress2.put("value", internal_ip_address); + + + operatingSystem.put("operatingSystemFamily", "UBUNTU"); + operatingSystem.put("operatingSystemArchitecture", "ARMv8"); + operatingSystem.put("operatingSystemVersion", 1804); + + geoLocation.put("city", city_name); + geoLocation.put("country", country_name); + geoLocation.put("latitude", new Random().nextFloat(-90, 90)); + geoLocation.put("longitude", new Random().nextFloat(-90, 90)); + + nodeProperties.put("providerId", provider_id); + nodeProperties.put("numberOfCores", cpu_cores); + nodeProperties.put("memory", ram_gb); + nodeProperties.put("disk", disk_gb); + nodeProperties.put("operatingSystem", operatingSystem); + nodeProperties.put("geoLocation", geoLocation); + + root_json_object.put("name", device_name); + root_json_object.put("loginCredential", loginCredential); + + JSONArray ipAddresses = new JSONArray(); + ipAddresses.add(ipAddress1); + ipAddresses.add(ipAddress2); + root_json_object.put("ipAddresses", ipAddresses); + + root_json_object.put("nodeProperties", nodeProperties); + root_json_object.put("systemArch", "ARMv8"); + root_json_object.put("scriptURL", "https://www.google.com"); + root_json_object.put("jarURL", "https://www.activeeon.com/public_content/7cde3381417ff3784639dc41fa7e7cd0544a5234-morphemic-7bulls/node_13.1.0-SNAPSHOT_armv8.jar"); + + + //JSONObject root_json_object = JsonFileParser.parse(request_body_file); + //root_json_object.put("name", device_name); + //((JSONObject) ((JSONArray) root_json_object.get("ipAddresses")).get(0)).put("value", internal_ip_address); + //((JSONObject) ((JSONArray) root_json_object.get("ipAddresses")).get(1)).put("value", external_ip_address); + //((JSONObject) root_json_object.get("nodeProperties")).put("disk", disk_gb); + //((JSONObject) root_json_object.get("nodeProperties")).put("memory", ram_gb); + //((JSONObject) root_json_object.get("nodeProperties")).put("providerId", provider_id); + //((JSONObject) root_json_object.get("nodeProperties")).put("numberOfCores", cpu_cores); + //((JSONObject) ((JSONObject) root_json_object.get("nodeProperties")).get("geoLocation")).put("country", country_name); + //((JSONObject) ((JSONObject) root_json_object.get("nodeProperties")).get("geoLocation")).put("city", city_name); + //((JSONObject) ((JSONObject) root_json_object.get("nodeProperties")).get("geoLocation")).put("latitude", new Random().nextFloat(-90, 90)); + //((JSONObject) ((JSONObject) root_json_object.get("nodeProperties")).get("geoLocation")).put("longitude", new Random().nextFloat(-90, 90)); + return(root_json_object.toJSONString()); } @@ -177,7 +230,7 @@ public static ArrayList get_running_applications(String running_jobs_str } }catch (Exception e){ e.printStackTrace(); - System.out.println("This is the input json jobs string\n\n\n\n"); + System.out.println("This is the input json jobs string\n\n"); System.out.println(running_jobs_string); } return applications; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 5e4d3a2..b42874a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -3,15 +3,15 @@ import eu.nebulouscloud.exn.core.Publisher; import eu.nebulouscloud.exn.core.SyncedPublisher; import eu.nebulouscloud.exn.settings.StaticExnConfig; +import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; +@Slf4j public class SynchronousBrokerPublisher { public static String EMPTY=""; private static HashMap> broker_and_topics_to_publish_to = new HashMap<>(); @@ -23,6 +23,12 @@ public class SynchronousBrokerPublisher { private String broker_ip; public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { + log.error("SynchronousBrokerPublisherParams"); + log.error("topic is "+topic); + log.error("broker ip is "+broker_ip); + log.error("username is "+brokerUsername); + log.error("password is "+brokerPassword); + boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerPublisher){ @@ -33,10 +39,12 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerU HashSet topics_to_publish_to = new HashSet<>(); topics_to_publish_to.add(topic); broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); + log.error("changed1"); publisher_configuration_changed = true; }else{ if (!broker_and_topics_to_publish_to.get(broker_ip).contains(topic)){ broker_and_topics_to_publish_to.get(broker_ip).add(topic); + log.error("changed2"); publisher_configuration_changed = true; } else{ @@ -44,17 +52,22 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerU } } - + log.error("preliminary_outside"); if (publisher_configuration_changed){ + log.error("preliminary_inside1"); // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ - Logger.getGlobal().log(Level.INFO,"Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); - active_connector.stop(new ArrayList<>(),publishers); + log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); + if (active_connector!=null) { + active_connector.stop(new ArrayList<>(), publishers); + } publishers.clear(); for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ + log.error("preliminary_inside2"); //ArrayList publishers = new ArrayList<>(); SyncedPublisher publisher = new SyncedPublisher("resource_manager_"+broker_topic, broker_topic, true, true); publishers.add(publisher); if (broker_topic.equals(topic)){ + log.error("inside_assignment_to_private_publisher_instance"); this.private_publisher_instance = (SyncedPublisher) publishers.get(publishers.size()-1); this.topic = broker_topic; this.broker_ip = broker_ip; @@ -78,27 +91,42 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerU ) ); active_connector.start(); - //Logger.getGlobal().log(INFO,"Sending from EXTERIOR"); - //private_publisher_instance.send(new JSONObject()); } } - public Map publish_for_response (String json_string_content, Iterable application_names){ + public Map publish_for_response (String json_string_content, Collection application_names){ Map reply = null; - for (String application_name : application_names) { + if (application_names!=null && !application_names.isEmpty()) { + for (String application_name : application_names) { + JSONParser parser = new JSONParser(); + JSONObject json_object = new JSONObject(); + try { + json_object = (JSONObject) parser.parse(json_string_content); + } catch (ParseException p) { + log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); + } + if (private_publisher_instance != null) { + reply = private_publisher_instance.sendSync(json_object, application_name, null, false); + } else { + log.error("Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + } + } + }else{ //Send an empty string for application JSONParser parser = new JSONParser(); JSONObject json_object = new JSONObject(); try { json_object = (JSONObject) parser.parse(json_string_content); } catch (ParseException p) { - Logger.getGlobal().log(Level.WARNING, "Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); + log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); } if (private_publisher_instance != null) { - reply = private_publisher_instance.sendSync(json_object,application_name,null,false); + log.info("Sending new synchronous message\n"+json_object.toJSONString()); + reply = private_publisher_instance.sendSync(json_object,EMPTY, null, false); + log.info("Sent new synchronous message\n"+json_object.toJSONString()); } else { - Logger.getGlobal().log(Level.SEVERE, "Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + log.error("Could not send message to AMQP broker, as the broker ip to be used has not been specified"); } } return reply; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index ed68331..5df46ac 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -1,7 +1,6 @@ package eu.nebulous.resource.discovery.registration.service; import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; -import eu.nebulous.resource.discovery.broker_communication.BrokerPublisher; import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; import eu.nebulous.resource.discovery.monitor.model.Device; import lombok.extern.slf4j.Slf4j; @@ -48,9 +47,12 @@ public void register(Device device) { Integer cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); Integer ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; - Integer disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL"))/1000000; + Integer disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; String external_ip_address = device.getIpAddress(); - + String device_username = device.getUsername(); + String device_password = new String(device.getPassword()); + String device_pub_key = new String(device.getPublicKey()); //TODO get here private key instead and pass this to device registration + //TODO implement provider here: String provider = device.getProvider(); //String network_rx =device_info.get("RX"); //String network_tx = device_info.get("TX"); @@ -59,15 +61,16 @@ public void register(Device device) { JSONObject register_device_message = new JSONObject(); register_device_message.put("device_name",device_name); register_device_message.put("timestamp",(int)(clock.millis()/1000)); - get_device_registration_json("./src/main/java/eu/nebulous/resource/discovery/broker_communication/sal_device_registration_base_payload.json","10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece"); - String sal_running_applications_reply = request_running_applications_AMQP(); //TODO handle the response inside the called function - ArrayList applications = get_running_applications(sal_running_applications_reply); - for (String application_name:applications) { + get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); + //String sal_running_applications_reply = request_running_applications_AMQP(); + //ArrayList applications = get_running_applications(sal_running_applications_reply); + //for (String application_name:applications) { + String application_name = ""; SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); log.info("The response received while trying to register device "+device_name); - } + //} /* This is some realtime information, could be retrieved with a different call to the EMS. CurrDateTime: 1709207141 From e00d3ff94a2471c21a86d40fc62f32eb74953601 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 14:56:20 +0200 Subject: [PATCH 061/132] Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 --- .../registration/service/SALRegistrationService.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 5df46ac..9ba0c76 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -3,6 +3,7 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; import eu.nebulous.resource.discovery.monitor.model.Device; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; import org.springframework.stereotype.Service; @@ -16,13 +17,10 @@ @Slf4j @Service +@RequiredArgsConstructor public class SALRegistrationService { private final ResourceDiscoveryProperties processorProperties; - public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { - this.processorProperties = processorProperties; - } - public void register(Device device) { Map device_info = device.getDeviceInfo(); From cc0943cd9fea85a996f8ead90227426bfc6f78dc Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 14:56:20 +0200 Subject: [PATCH 062/132] Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 --- .../SynchronousBrokerPublisher.java | 5 ---- .../service/SALRegistrationService.java | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index b42874a..151029d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -23,11 +23,6 @@ public class SynchronousBrokerPublisher { private String broker_ip; public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { - log.error("SynchronousBrokerPublisherParams"); - log.error("topic is "+topic); - log.error("broker ip is "+broker_ip); - log.error("username is "+brokerUsername); - log.error("password is "+brokerPassword); boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 5df46ac..ad19c4d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -3,8 +3,10 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; import eu.nebulous.resource.discovery.monitor.model.Device; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; +import org.springframework.beans.factory.InitializingBean; import org.springframework.stereotype.Service; import java.time.Clock; @@ -16,12 +18,10 @@ @Slf4j @Service -public class SALRegistrationService { +@RequiredArgsConstructor +public class SALRegistrationService implements InitializingBean { private final ResourceDiscoveryProperties processorProperties; - - public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { - this.processorProperties = processorProperties; - } + private String application_name = ""; public void register(Device device) { @@ -65,7 +65,6 @@ public void register(Device device) { //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - String application_name = ""; SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); @@ -89,4 +88,19 @@ private String get_registration_topic_name(String application_name) { return "eu.nebulouscloud.exn.sal.node.put"; //return ("eu.nebulouscloud.exn.sal.edge." + application_name); } + + @Override + public void afterPropertiesSet() throws Exception { + if ( processorProperties.getNebulous_broker_password()!=null && + processorProperties.getNebulous_broker_username()!=null && + processorProperties.getNebulous_broker_ip_address()!=null + ){ + log.info("Successful setting of properties for communication with SAL"); + }else{ + log.error("topic is "+get_registration_topic_name(application_name)); + log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); + log.error("username is "+processorProperties.getNebulous_broker_username()); + log.error("password is "+processorProperties.getNebulous_broker_password()); + } + } } From e9a62f34aa3110fa9d9fe4fc1be1544baf5128e5 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 16:23:57 +0200 Subject: [PATCH 063/132] Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 --- .../service/SALRegistrationService.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index ad19c4d..178ccb9 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -22,6 +22,7 @@ public class SALRegistrationService implements InitializingBean { private final ResourceDiscoveryProperties processorProperties; private String application_name = ""; + private boolean properties_set = false; public void register(Device device) { @@ -65,10 +66,14 @@ public void register(Device device) { //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); - //TODO handle the response here - Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); - log.info("The response received while trying to register device "+device_name); + if (properties_set) { + SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + //TODO handle the response here + Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); + log.info("The response received while trying to register device " + device_name); + }else{ + log.error("The necessary properties for the initialization of the SynchronousBrokerPublisher have not been set"); + } //} /* This is some realtime information, could be retrieved with a different call to the EMS. @@ -96,11 +101,13 @@ public void afterPropertiesSet() throws Exception { processorProperties.getNebulous_broker_ip_address()!=null ){ log.info("Successful setting of properties for communication with SAL"); + properties_set =true; }else{ log.error("topic is "+get_registration_topic_name(application_name)); log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); log.error("username is "+processorProperties.getNebulous_broker_username()); log.error("password is "+processorProperties.getNebulous_broker_password()); + throw new Exception("Required data is null"); } } } From da0fee361cdce9f1052efafed27f969e0658a06d Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 16:55:28 +0200 Subject: [PATCH 064/132] Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 --- .../registration/service/SALRegistrationService.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 178ccb9..33c9f7e 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -21,11 +21,10 @@ @RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { private final ResourceDiscoveryProperties processorProperties; - private String application_name = ""; - private boolean properties_set = false; public void register(Device device) { + String application_name = ""; //TODO decide on this Map device_info = device.getDeviceInfo(); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS @@ -63,17 +62,17 @@ public void register(Device device) { register_device_message.put("device_name",device_name); register_device_message.put("timestamp",(int)(clock.millis()/1000)); get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); + log.error("topic is "+get_registration_topic_name(application_name)); + log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); + log.error("username is "+processorProperties.getNebulous_broker_username()); + log.error("password is "+processorProperties.getNebulous_broker_password()); //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - if (properties_set) { SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); log.info("The response received while trying to register device " + device_name); - }else{ - log.error("The necessary properties for the initialization of the SynchronousBrokerPublisher have not been set"); - } //} /* This is some realtime information, could be retrieved with a different call to the EMS. From b5b7bae7801d46bf2f16486c31aaa0f8f1628f15 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 16:56:59 +0200 Subject: [PATCH 065/132] Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 --- .../discovery/registration/service/SALRegistrationService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 33c9f7e..1ec78a7 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -100,9 +100,7 @@ public void afterPropertiesSet() throws Exception { processorProperties.getNebulous_broker_ip_address()!=null ){ log.info("Successful setting of properties for communication with SAL"); - properties_set =true; }else{ - log.error("topic is "+get_registration_topic_name(application_name)); log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); log.error("username is "+processorProperties.getNebulous_broker_username()); log.error("password is "+processorProperties.getNebulous_broker_password()); From 6f4e49b1ac68cb4c3e76ee52c59b0396d8366b56 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 17:25:36 +0200 Subject: [PATCH 066/132] Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 --- resource-discovery/src/main/resources/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index df9f031..070360d 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -21,6 +21,8 @@ discovery: brokerUsername: "aaa" brokerPassword: "111" nebulous_broker_ip_address: "nebulous-activemq" + nebulous_broker_username: "admin" + nebulous_broker_password: "admin" sal_host: "localhost" sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" From 40479c16ed9163306458b937d67fb21a42eb347f Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 17:28:06 +0200 Subject: [PATCH 067/132] Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 --- .../registration/service/SALRegistrationService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1ec78a7..e01cf87 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -7,6 +7,7 @@ import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.time.Clock; @@ -18,10 +19,14 @@ @Slf4j @Service -@RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { + @Autowired private final ResourceDiscoveryProperties processorProperties; + public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { + this.processorProperties = processorProperties; + } + public void register(Device device) { String application_name = ""; //TODO decide on this From 13b8c8b7d2d12139d660ae58cd4c586e6f70c3dc Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Sat, 23 Mar 2024 17:28:06 +0200 Subject: [PATCH 068/132] Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 --- .../service/SALRegistrationService.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1ec78a7..ef6b73e 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -7,6 +7,7 @@ import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.time.Clock; @@ -18,13 +19,17 @@ @Slf4j @Service -@RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { + @Autowired private final ResourceDiscoveryProperties processorProperties; + public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { + this.processorProperties = processorProperties; + } + public void register(Device device) { - String application_name = ""; //TODO decide on this + String application_name = "default-app"; //TODO decide on this Map device_info = device.getDeviceInfo(); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS @@ -71,7 +76,7 @@ public void register(Device device) { //for (String application_name:applications) { SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here - Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton("")); + Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name); //} @@ -104,7 +109,7 @@ public void afterPropertiesSet() throws Exception { log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); log.error("username is "+processorProperties.getNebulous_broker_username()); log.error("password is "+processorProperties.getNebulous_broker_password()); - throw new Exception("Required data is null"); + throw new Exception("Required data is null - broker ip is "+processorProperties.getNebulous_broker_ip_address()+" username is "+processorProperties.getNebulous_broker_username()+" password is "+processorProperties.getNebulous_broker_password()); } } } From 8a307b69ccb33f653d1ab2dea46877959671d9e4 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 2 Apr 2024 14:08:42 +0300 Subject: [PATCH 069/132] Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 --- .../SynchronousBrokerPublisher.java | 26 ++++++++++++++++--- .../service/SALRegistrationService.java | 4 +-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 151029d..600ddf0 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -7,6 +7,7 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import org.slf4j.Marker; import java.util.*; @@ -93,19 +94,37 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerU public Map publish_for_response (String json_string_content, Collection application_names){ Map reply = null; + HashMap payload = new HashMap<>(); + HashMap metadata = new HashMap<>(); + metadata.put("user","admin"); + metadata.put("type","edge"); if (application_names!=null && !application_names.isEmpty()) { for (String application_name : application_names) { + + boolean successful_json_parsing = false; JSONParser parser = new JSONParser(); JSONObject json_object = new JSONObject(); try { json_object = (JSONObject) parser.parse(json_string_content); + successful_json_parsing = true; } catch (ParseException p) { log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); } + metadata.put("jobId",application_name); + payload.put("metaData",metadata); if (private_publisher_instance != null) { - reply = private_publisher_instance.sendSync(json_object, application_name, null, false); + //reply = private_publisher_instance.sendSync(json_object, application_name, null, false); + if (successful_json_parsing) { + json_object.put("jobId",application_name); + payload.put("body",json_object.toJSONString()); + reply = private_publisher_instance.sendSync(payload, application_name, null, false); + }else{ + payload.put("body",json_string_content); + log.warn(Marker.ANY_MARKER,"Sending the original json string without any modification as its parsing was not successful"); + reply = private_publisher_instance.sendSync(payload, application_name, null, false); + } } else { - log.error("Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + log.error("Could not send message to AMQP broker, as the private publisher instance is null (is broker ip specified?)"); } } }else{ //Send an empty string for application @@ -113,6 +132,7 @@ public Map publish_for_response (String json_string_content, Collection JSONObject json_object = new JSONObject(); try { json_object = (JSONObject) parser.parse(json_string_content); + } catch (ParseException p) { log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); } @@ -121,7 +141,7 @@ public Map publish_for_response (String json_string_content, Collection reply = private_publisher_instance.sendSync(json_object,EMPTY, null, false); log.info("Sent new synchronous message\n"+json_object.toJSONString()); } else { - log.error("Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + log.error("Could not send message to AMQP broker, as the private publisher instance is null (is broker ip specified?)"); } } return reply; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index ef6b73e..03e7ca4 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -29,7 +29,7 @@ public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { public void register(Device device) { - String application_name = "default-app"; //TODO decide on this + String application_name = "default-application"; //TODO decide on this Map device_info = device.getDeviceInfo(); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS @@ -94,7 +94,7 @@ public void register(Device device) { } private String get_registration_topic_name(String application_name) { - return "eu.nebulouscloud.exn.sal.node.put"; + return "eu.nebulouscloud.exn.sal.node.create"; //return ("eu.nebulouscloud.exn.sal.edge." + application_name); } From 6a8d7ec7d995ee54de84daad5c46203cc2d205c2 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 2 Apr 2024 18:36:59 +0300 Subject: [PATCH 070/132] Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 --- .../discovery/ResourceDiscoveryProperties.java | 1 + .../broker_communication/BrokerPublisher.java | 6 ++++-- .../broker_communication/BrokerSubscriber.java | 8 +++++--- .../BrokerSubscriptionDetails.java | 12 +++++++++++- .../SynchronousBrokerPublisher.java | 4 ++-- .../resource/discovery/monitor/DeviceProcessor.java | 2 +- .../registration/service/SALRegistrationService.java | 2 +- 7 files changed, 25 insertions(+), 10 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 8db5a1b..69610be 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -87,6 +87,7 @@ public class ResourceDiscoveryProperties { // Nebulous broker subscription details private String nebulous_broker_ip_address; + private int nebulous_broker_port; private String nebulous_broker_username; private String lost_device_topic; private String nebulous_broker_password; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 9713117..03e8665 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -23,8 +23,9 @@ public class BrokerPublisher { private ExtendedConnector active_connector; private String topic; private String broker_ip; + private int broker_port; - public BrokerPublisher(String topic, String broker_ip, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { + public BrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerPublisher){ @@ -62,6 +63,7 @@ public BrokerPublisher(String topic, String broker_ip, String brokerUsername, St this.private_publisher_instance = publishers.get(publishers.size()-1); this.topic = broker_topic; this.broker_ip = broker_ip; + this.broker_port = broker_port; } } //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); @@ -74,7 +76,7 @@ public BrokerPublisher(String topic, String broker_ip, String brokerUsername, St false, new StaticExnConfig( broker_ip, - 5672, + broker_port, brokerUsername, brokerPassword, 60, diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java index 1e1f713..7ff11d5 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java @@ -56,11 +56,12 @@ public void setProcessing_function(BiFunction processing_function) { private static HashMap current_connectors = new HashMap<>(); private String topic; private String broker_ip; + private int broker_port; private String brokerUsername; private String brokerPassword; BrokerSubscriptionDetails broker_details; - public BrokerSubscriber(String topic, String broker_ip, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation, String application_name) { + public BrokerSubscriber(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation, String application_name) { boolean able_to_initialize_BrokerSubscriber = topic != null && broker_ip != null && brokerUsername != null && brokerPassword != null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerSubscriber) { @@ -73,7 +74,7 @@ public BrokerSubscriber(String topic, String broker_ip, String brokerUsername, S throw new RuntimeException(e); } } - broker_details = new BrokerSubscriptionDetails(broker_ip, brokerUsername, brokerPassword, application_name, topic); + broker_details = new BrokerSubscriptionDetails(broker_ip, broker_port, brokerUsername, brokerPassword, application_name, topic); boolean subscriber_configuration_changed; if (!broker_and_topics_to_subscribe_to.containsKey(broker_ip)) { HashSet topics_to_subscribe_to = new HashSet<>(); @@ -107,6 +108,7 @@ public BrokerSubscriber(String topic, String broker_ip, String brokerUsername, S this.topic = topic; this.broker_ip = broker_ip; + this.broker_port = broker_port; this.brokerUsername = brokerUsername; this.brokerPassword = brokerPassword; add_topic_consumer_to_broker_connector(current_consumer); @@ -131,7 +133,7 @@ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { false, new StaticExnConfig( broker_ip, - 5672, + broker_port, brokerUsername, brokerPassword, 60, diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriptionDetails.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriptionDetails.java index c8113d3..65ba8f6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriptionDetails.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriptionDetails.java @@ -6,11 +6,13 @@ public class BrokerSubscriptionDetails { String broker_username = "admin"; String broker_password = "admin"; String broker_ip = "localhost"; + int broker_port = 5672; String application_name = "default_application"; String topic = EMPTY; - public BrokerSubscriptionDetails(String broker_ip, String broker_username, String broker_password,String application_name, String topic) { + public BrokerSubscriptionDetails(String broker_ip, int broker_port, String broker_username, String broker_password,String application_name, String topic) { this.broker_ip = broker_ip; + this.broker_port = broker_port; this.broker_username = broker_username; this.broker_password = broker_password; this.topic = topic; @@ -66,4 +68,12 @@ public String getTopic() { public void setTopic(String topic) { this.topic = topic; } + + public int getBroker_port() { + return broker_port; + } + + public void setBroker_port(int broker_port) { + this.broker_port = broker_port; + } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 600ddf0..3958e17 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -23,7 +23,7 @@ public class SynchronousBrokerPublisher { private String topic; private String broker_ip; - public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { + public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); @@ -79,7 +79,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, String brokerU false, new StaticExnConfig( broker_ip, - 5672, + broker_port, brokerUsername, brokerPassword, 60, diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 8e44849..8fe87bb 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -138,7 +138,7 @@ private void processFailedDevices() { lost_device_message.put("device_name",device.getName()); Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); - BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(),processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); device_lost_publisher.publish(lost_device_message.toJSONString(), Collections.singleton("")); log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 03e7ca4..0619fe8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -74,7 +74,7 @@ public void register(Device device) { //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name); From 135fb650e123b0358cfa1579ce94c06053de07f7 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 2 Apr 2024 18:45:41 +0300 Subject: [PATCH 071/132] Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 --- resource-discovery/src/main/resources/application.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 070360d..74ae20f 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -21,6 +21,7 @@ discovery: brokerUsername: "aaa" brokerPassword: "111" nebulous_broker_ip_address: "nebulous-activemq" + nebulous_broker_port: 5672 nebulous_broker_username: "admin" nebulous_broker_password: "admin" sal_host: "localhost" From ac13693b27132014bc28b3fb4e4278ac18e57951 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 2 Apr 2024 19:21:33 +0300 Subject: [PATCH 072/132] Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 --- .../discovery/registration/service/SALRegistrationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 0619fe8..1331a36 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -69,6 +69,7 @@ public void register(Device device) { get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); log.error("topic is "+get_registration_topic_name(application_name)); log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); + log.error("broker port is "+processorProperties.getNebulous_broker_port()); log.error("username is "+processorProperties.getNebulous_broker_username()); log.error("password is "+processorProperties.getNebulous_broker_password()); //String sal_running_applications_reply = request_running_applications_AMQP(); From 0bfe18d56b27734286725badc30d3b0deef34ab8 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 2 Apr 2024 19:36:51 +0300 Subject: [PATCH 073/132] Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 --- .../registration/service/SALRegistrationService.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1331a36..1e0d08d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -63,10 +63,11 @@ public void register(Device device) { Clock clock = Clock.systemUTC(); - JSONObject register_device_message = new JSONObject(); - register_device_message.put("device_name",device_name); - register_device_message.put("timestamp",(int)(clock.millis()/1000)); - get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); + //JSONObject register_device_message = new JSONObject(); + //register_device_message.put("device_name",device_name); + //register_device_message.put("timestamp",(int)(clock.millis()/1000)); + + String register_device_message_string = get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); log.error("topic is "+get_registration_topic_name(application_name)); log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); log.error("broker port is "+processorProperties.getNebulous_broker_port()); @@ -77,7 +78,7 @@ public void register(Device device) { //for (String application_name:applications) { SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here - Map response = register_device_publisher.publish_for_response(register_device_message.toJSONString(), Collections.singleton(application_name)); + Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name); //} From bbb747859383ffd7f873021cba9a51896b9d1f4b Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 4 Apr 2024 17:53:26 +0300 Subject: [PATCH 074/132] RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml --- resource-discovery/pom.xml | 21 +++++++------------ .../src/main/resources/application.yml | 9 ++++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index ff87f62..b0258b9 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -6,13 +6,13 @@ org.springframework.boot spring-boot-starter-parent - 3.2.1 + 3.2.4 eu.nebulous.resource-management resource-discovery - 1.0.0-SNAPSHOT + 1.0.2-SNAPSHOT Resource discovery service Nebulous resource discovery service @@ -37,6 +37,12 @@ org.springframework.boot spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-logging + + @@ -120,17 +126,6 @@ - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-logging - - - - diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 74ae20f..a7bff37 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -27,9 +27,9 @@ discovery: sal_host: "localhost" sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" - trustStoreFile: tests/config/broker-truststore.p12 - trustStorePassword: melodic - trustStoreType: PKCS12 +# trustStoreFile: tests/config/broker-truststore.p12 +# trustStorePassword: melodic +# trustStoreType: PKCS12 allowedDeviceInfoKeys: - '*' # NOTE: @@ -42,4 +42,5 @@ discovery: password: '$2a$10$I6GSOKiY5n4/Ql0LA7Js0.4HT4UXVCNaNpGv5UdZt/brEdv/F.ttG' # user1 (BCrypt; 10 iterations) roles: [ USER ] -#logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE \ No newline at end of file +#logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE +logging.level.eu.nebulous.resource.discovery.common: TRACE \ No newline at end of file From c1d06dd018a99d35fef495a3a58384e285ea5503 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 4 Apr 2024 18:33:53 +0300 Subject: [PATCH 075/132] RD: Fixed banner.txt --- resource-discovery/src/main/resources/banner.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-discovery/src/main/resources/banner.txt b/resource-discovery/src/main/resources/banner.txt index 1e12e7a..c8085cb 100644 --- a/resource-discovery/src/main/resources/banner.txt +++ b/resource-discovery/src/main/resources/banner.txt @@ -9,3 +9,4 @@ ${AnsiColor.012} ╚═╝ ╚═╝╚══════╝╚═════ ${AnsiColor.046} :: App version :: ${AnsiColor.87} (${application.version}) ${AnsiColor.046} :: Spring Boot :: ${AnsiColor.87} ${spring-boot.formatted-version} ${AnsiColor.046} :: Java (TM) :: ${AnsiColor.87} (${java.version}) +${AnsiColor.DEFAULT}${AnsiStyle.NORMAL} \ No newline at end of file From d85d65376d0d1707224e3d13eedef133b02e4a85 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 4 Apr 2024 18:57:19 +0300 Subject: [PATCH 076/132] RD: Modified application.yml --- resource-discovery/src/main/resources/application.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index a7bff37..2d707bf 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -42,5 +42,4 @@ discovery: password: '$2a$10$I6GSOKiY5n4/Ql0LA7Js0.4HT4UXVCNaNpGv5UdZt/brEdv/F.ttG' # user1 (BCrypt; 10 iterations) roles: [ USER ] -#logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE -logging.level.eu.nebulous.resource.discovery.common: TRACE \ No newline at end of file +#logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE \ No newline at end of file From 31b103c207dd900689c94fe5b608a0a03db8fc80 Mon Sep 17 00:00:00 2001 From: ipatini Date: Sat, 6 Apr 2024 18:59:23 +0300 Subject: [PATCH 077/132] RD: Added temp. debug messages --- .../registration/RegistrationRequestProcessor.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index ce40a47..ced3b5f 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -231,6 +231,7 @@ public void onMessage(Map message) { } private void processResponse(@NonNull Map response) { + log.warn("!!!!!! RRProcessor: processResponse: BEGIN: {}", response); String requestType = response.getOrDefault("requestType", "").toString().trim(); String requestId = response.getOrDefault("requestId", "").toString().trim(); String reference = response.getOrDefault("reference", "").toString().trim(); @@ -239,6 +240,7 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); + log.warn("!!!!!! RRProcessor: registrationRequest: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -250,6 +252,7 @@ private void processResponse(@NonNull Map response) { }; log.debug("processResponse: Temporary status change: {} --> {}", currStatus, newStatus); registrationRequest.setStatus(newStatus); + log.warn("!!!!!! RRProcessor: registrationRequest: status: {} --> {}", currStatus, newStatus); if (currStatus==RegistrationRequestStatus.SUCCESS) { log.error("ERROR: received response for a request with status SUCCESS. Will ignore response: request-id={}", requestId); @@ -287,6 +290,7 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); + log.warn("!!!!!! RRProcessor: devInfo: obj: {}", obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -318,12 +322,15 @@ private void processResponse(@NonNull Map response) { } // Set new status + log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", currStatus); + log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", registrationRequest.getStatus()); if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); } + log.warn("!!!!!! RRProcessor: devInfo: STATUS AFTER: {}", registrationRequest.getStatus()); log.debug("processResponse: Done processing response for request: id={}, timestamp={}", requestId, timestamp); } else { @@ -333,6 +340,7 @@ private void processResponse(@NonNull Map response) { // If request status is SUCCESS then copy Device in monitoring subsystem if (registrationRequest.getStatus() == RegistrationRequestStatus.SUCCESS) { try { + log.warn("!!!!!! RRProcessor: COPY TO DEVICES"); copyDeviceToMonitoring(registrationRequest); } catch (Exception e) { log.warn("processResponse: EXCEPTION: while copying device to monitoring subsystem: request={}\n", registrationRequest, e); @@ -344,14 +352,17 @@ private void processResponse(@NonNull Map response) { // Store changes log.debug("processResponse: Save updated request: id={}, request={}", requestId, registrationRequest); registrationRequestService.update(registrationRequest, false, true); + log.warn("!!!!!! RRProcessor: SAVED CHANGES!!!!"); // Archive success requests if (doArchive) { registrationRequestService.archiveRequestBySystem(registrationRequest.getId()); + log.warn("!!!!!! RRProcessor: ARCHIVED"); } } else { log.debug("processResponse: Request not found: id={}, requestType={}", requestId, requestType); } + log.warn("!!!!!! RRProcessor: END"); } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { From 738f6048a8b88b4cfc7410db2146c7d7855c03bd Mon Sep 17 00:00:00 2001 From: ipatini Date: Sat, 6 Apr 2024 19:20:44 +0300 Subject: [PATCH 078/132] RD: Added temp. debug messages 2 --- .../SynchronousBrokerPublisher.java | 5 +++++ .../registration/RegistrationRequestProcessor.java | 13 ++++++------- .../service/SALRegistrationService.java | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 3958e17..f6df4fd 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -71,6 +71,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); + log.warn("########## SynchronousBrokerPublisher: CREATING ExtendedConnector"); active_connector = new ExtendedConnector("resource_manager" , new CustomConnectorHandler() {} , publishers @@ -86,13 +87,16 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por EMPTY ) ); + log.warn("########## SynchronousBrokerPublisher: STARTING ExtendedConnector"); active_connector.start(); + log.warn("########## SynchronousBrokerPublisher: DONE"); } } public Map publish_for_response (String json_string_content, Collection application_names){ + log.warn("########## SynchronousBrokerPublisher: BEGIN: publish_for_response"); Map reply = null; HashMap payload = new HashMap<>(); HashMap metadata = new HashMap<>(); @@ -144,6 +148,7 @@ public Map publish_for_response (String json_string_content, Collection log.error("Could not send message to AMQP broker, as the private publisher instance is null (is broker ip specified?)"); } } + log.warn("########## SynchronousBrokerPublisher: END: {}", reply); return reply; } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index ced3b5f..6ebaf21 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -240,7 +240,6 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); - log.warn("!!!!!! RRProcessor: registrationRequest: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -252,7 +251,6 @@ private void processResponse(@NonNull Map response) { }; log.debug("processResponse: Temporary status change: {} --> {}", currStatus, newStatus); registrationRequest.setStatus(newStatus); - log.warn("!!!!!! RRProcessor: registrationRequest: status: {} --> {}", currStatus, newStatus); if (currStatus==RegistrationRequestStatus.SUCCESS) { log.error("ERROR: received response for a request with status SUCCESS. Will ignore response: request-id={}", requestId); @@ -290,7 +288,6 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); - log.warn("!!!!!! RRProcessor: devInfo: obj: {}", obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -322,15 +319,12 @@ private void processResponse(@NonNull Map response) { } // Set new status - log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", currStatus); - log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", registrationRequest.getStatus()); if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); } - log.warn("!!!!!! RRProcessor: devInfo: STATUS AFTER: {}", registrationRequest.getStatus()); log.debug("processResponse: Done processing response for request: id={}, timestamp={}", requestId, timestamp); } else { @@ -340,8 +334,9 @@ private void processResponse(@NonNull Map response) { // If request status is SUCCESS then copy Device in monitoring subsystem if (registrationRequest.getStatus() == RegistrationRequestStatus.SUCCESS) { try { - log.warn("!!!!!! RRProcessor: COPY TO DEVICES"); + log.warn("!!!!!! RRProcessor: COPING TO DEVICES"); copyDeviceToMonitoring(registrationRequest); + log.warn("!!!!!! RRProcessor: COPIED TO DEVICES"); } catch (Exception e) { log.warn("processResponse: EXCEPTION: while copying device to monitoring subsystem: request={}\n", registrationRequest, e); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_ERROR); @@ -366,6 +361,7 @@ private void processResponse(@NonNull Map response) { } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { + log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: BEGIN"); Device device = objectMapper.convertValue(registrationRequest.getDevice(), Device.class); // override values device.setId(null); @@ -380,7 +376,10 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { //device.setRequest(registrationRequest); device.setRequestId(registrationRequest.getId()); device.setNodeReference(registrationRequest.getNodeReference()); + log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: BEFORE SAVE"); deviceManagementService.save(device); + log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: AFTER SAVE -- BEFORE SAL"); salRegistrationService.register(device); + log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: END: AFTER SAL"); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1e0d08d..bb94836 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -28,6 +28,7 @@ public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { } public void register(Device device) { + log.warn("########## SALReg: BEGIN: {}", device); String application_name = "default-application"; //TODO decide on this Map device_info = device.getDeviceInfo(); @@ -76,8 +77,10 @@ public void register(Device device) { //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { + log.warn("########## SALReg: CREATING register_device_publisher"); SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here + log.warn("########## SALReg: PUBLISHING WITH register_device_publisher"); Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name); //} @@ -93,6 +96,7 @@ public void register(Device device) { TX: 0 */ + log.warn("########## SALReg: END"); } private String get_registration_topic_name(String application_name) { From 8b7e5f4bc00940dc88fee279d5ae19b4daa49701 Mon Sep 17 00:00:00 2001 From: ipatini Date: Sat, 6 Apr 2024 20:18:06 +0300 Subject: [PATCH 079/132] Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. --- .../SynchronousBrokerPublisher.java | 5 ----- .../registration/RegistrationRequestProcessor.java | 13 +++++++------ .../service/SALRegistrationService.java | 4 ---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index f6df4fd..3958e17 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -71,7 +71,6 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); - log.warn("########## SynchronousBrokerPublisher: CREATING ExtendedConnector"); active_connector = new ExtendedConnector("resource_manager" , new CustomConnectorHandler() {} , publishers @@ -87,16 +86,13 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por EMPTY ) ); - log.warn("########## SynchronousBrokerPublisher: STARTING ExtendedConnector"); active_connector.start(); - log.warn("########## SynchronousBrokerPublisher: DONE"); } } public Map publish_for_response (String json_string_content, Collection application_names){ - log.warn("########## SynchronousBrokerPublisher: BEGIN: publish_for_response"); Map reply = null; HashMap payload = new HashMap<>(); HashMap metadata = new HashMap<>(); @@ -148,7 +144,6 @@ public Map publish_for_response (String json_string_content, Collection log.error("Could not send message to AMQP broker, as the private publisher instance is null (is broker ip specified?)"); } } - log.warn("########## SynchronousBrokerPublisher: END: {}", reply); return reply; } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 6ebaf21..ced3b5f 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -240,6 +240,7 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); + log.warn("!!!!!! RRProcessor: registrationRequest: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -251,6 +252,7 @@ private void processResponse(@NonNull Map response) { }; log.debug("processResponse: Temporary status change: {} --> {}", currStatus, newStatus); registrationRequest.setStatus(newStatus); + log.warn("!!!!!! RRProcessor: registrationRequest: status: {} --> {}", currStatus, newStatus); if (currStatus==RegistrationRequestStatus.SUCCESS) { log.error("ERROR: received response for a request with status SUCCESS. Will ignore response: request-id={}", requestId); @@ -288,6 +290,7 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); + log.warn("!!!!!! RRProcessor: devInfo: obj: {}", obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -319,12 +322,15 @@ private void processResponse(@NonNull Map response) { } // Set new status + log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", currStatus); + log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", registrationRequest.getStatus()); if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); } + log.warn("!!!!!! RRProcessor: devInfo: STATUS AFTER: {}", registrationRequest.getStatus()); log.debug("processResponse: Done processing response for request: id={}, timestamp={}", requestId, timestamp); } else { @@ -334,9 +340,8 @@ private void processResponse(@NonNull Map response) { // If request status is SUCCESS then copy Device in monitoring subsystem if (registrationRequest.getStatus() == RegistrationRequestStatus.SUCCESS) { try { - log.warn("!!!!!! RRProcessor: COPING TO DEVICES"); + log.warn("!!!!!! RRProcessor: COPY TO DEVICES"); copyDeviceToMonitoring(registrationRequest); - log.warn("!!!!!! RRProcessor: COPIED TO DEVICES"); } catch (Exception e) { log.warn("processResponse: EXCEPTION: while copying device to monitoring subsystem: request={}\n", registrationRequest, e); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_ERROR); @@ -361,7 +366,6 @@ private void processResponse(@NonNull Map response) { } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { - log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: BEGIN"); Device device = objectMapper.convertValue(registrationRequest.getDevice(), Device.class); // override values device.setId(null); @@ -376,10 +380,7 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { //device.setRequest(registrationRequest); device.setRequestId(registrationRequest.getId()); device.setNodeReference(registrationRequest.getNodeReference()); - log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: BEFORE SAVE"); deviceManagementService.save(device); - log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: AFTER SAVE -- BEFORE SAL"); salRegistrationService.register(device); - log.warn("!!!!!! RRProcessor: copyDeviceToMonitoring: END: AFTER SAL"); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index bb94836..1e0d08d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -28,7 +28,6 @@ public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { } public void register(Device device) { - log.warn("########## SALReg: BEGIN: {}", device); String application_name = "default-application"; //TODO decide on this Map device_info = device.getDeviceInfo(); @@ -77,10 +76,8 @@ public void register(Device device) { //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - log.warn("########## SALReg: CREATING register_device_publisher"); SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here - log.warn("########## SALReg: PUBLISHING WITH register_device_publisher"); Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name); //} @@ -96,7 +93,6 @@ public void register(Device device) { TX: 0 */ - log.warn("########## SALReg: END"); } private String get_registration_topic_name(String application_name) { From 49ca35cd6be6c7f7ce66ebbc83218ddddfc54341 Mon Sep 17 00:00:00 2001 From: ipatini Date: Sat, 6 Apr 2024 20:18:06 +0300 Subject: [PATCH 080/132] Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. --- .../registration/RegistrationRequestProcessor.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index ced3b5f..ce40a47 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -231,7 +231,6 @@ public void onMessage(Map message) { } private void processResponse(@NonNull Map response) { - log.warn("!!!!!! RRProcessor: processResponse: BEGIN: {}", response); String requestType = response.getOrDefault("requestType", "").toString().trim(); String requestId = response.getOrDefault("requestId", "").toString().trim(); String reference = response.getOrDefault("reference", "").toString().trim(); @@ -240,7 +239,6 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); - log.warn("!!!!!! RRProcessor: registrationRequest: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -252,7 +250,6 @@ private void processResponse(@NonNull Map response) { }; log.debug("processResponse: Temporary status change: {} --> {}", currStatus, newStatus); registrationRequest.setStatus(newStatus); - log.warn("!!!!!! RRProcessor: registrationRequest: status: {} --> {}", currStatus, newStatus); if (currStatus==RegistrationRequestStatus.SUCCESS) { log.error("ERROR: received response for a request with status SUCCESS. Will ignore response: request-id={}", requestId); @@ -290,7 +287,6 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); - log.warn("!!!!!! RRProcessor: devInfo: obj: {}", obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -322,15 +318,12 @@ private void processResponse(@NonNull Map response) { } // Set new status - log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", currStatus); - log.warn("!!!!!! RRProcessor: devInfo: STATUS BEFORE: {}", registrationRequest.getStatus()); if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); } - log.warn("!!!!!! RRProcessor: devInfo: STATUS AFTER: {}", registrationRequest.getStatus()); log.debug("processResponse: Done processing response for request: id={}, timestamp={}", requestId, timestamp); } else { @@ -340,7 +333,6 @@ private void processResponse(@NonNull Map response) { // If request status is SUCCESS then copy Device in monitoring subsystem if (registrationRequest.getStatus() == RegistrationRequestStatus.SUCCESS) { try { - log.warn("!!!!!! RRProcessor: COPY TO DEVICES"); copyDeviceToMonitoring(registrationRequest); } catch (Exception e) { log.warn("processResponse: EXCEPTION: while copying device to monitoring subsystem: request={}\n", registrationRequest, e); @@ -352,17 +344,14 @@ private void processResponse(@NonNull Map response) { // Store changes log.debug("processResponse: Save updated request: id={}, request={}", requestId, registrationRequest); registrationRequestService.update(registrationRequest, false, true); - log.warn("!!!!!! RRProcessor: SAVED CHANGES!!!!"); // Archive success requests if (doArchive) { registrationRequestService.archiveRequestBySystem(registrationRequest.getId()); - log.warn("!!!!!! RRProcessor: ARCHIVED"); } } else { log.debug("processResponse: Request not found: id={}, requestType={}", requestId, requestType); } - log.warn("!!!!!! RRProcessor: END"); } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { From a681f2614615bceadaccaee69dc69e335f335f0f Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 8 Apr 2024 23:54:44 +0300 Subject: [PATCH 081/132] RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. --- .../ResourceDiscoveryProperties.java | 8 +- .../discovery/monitor/model/Device.java | 2 + .../RegistrationRequestProcessor.java | 3 +- .../service/SALRegistrationService.java | 111 ++++++++++++++---- 4 files changed, 95 insertions(+), 29 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 69610be..735eb4a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -6,7 +6,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -59,6 +58,11 @@ public class ResourceDiscoveryProperties { private String deviceLifeCycleRequestsTopic = "ems.client.installation.requests"; private String deviceLifeCycleResponsesTopic = "ems.client.installation.reports"; + // SAL registration settings + private boolean salRegistrationEnabled = true; + private long salRegistrationTimeout = 60*1000; + private String registration_topic_name = "eu.nebulouscloud.exn.sal.node.create"; + // Failed devices detection private boolean automaticFailedDetection = true; private long suspectDeviceThreshold = 5; // in minutes @@ -89,8 +93,8 @@ public class ResourceDiscoveryProperties { private String nebulous_broker_ip_address; private int nebulous_broker_port; private String nebulous_broker_username; - private String lost_device_topic; private String nebulous_broker_password; + private String lost_device_topic; @Data public static class UserData { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index dc41182..505c12b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -50,6 +50,8 @@ public class Device { private Instant suspectTimestamp; private int retries; + private boolean registeredToSAL; + public void incrementRetries() { retries++; } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index ce40a47..90052d9 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -370,6 +370,7 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { device.setRequestId(registrationRequest.getId()); device.setNodeReference(registrationRequest.getNodeReference()); deviceManagementService.save(device); - salRegistrationService.register(device); + if (processorProperties.isSalRegistrationEnabled()) + salRegistrationService.queueForRegistration(device); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1e0d08d..443a8ac 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -3,28 +3,36 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; import eu.nebulous.resource.discovery.monitor.model.Device; +import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.json.simple.JSONObject; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; import java.time.Clock; -import java.util.ArrayList; import java.util.Collections; import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; -import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.*; +import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.get_device_registration_json; @Slf4j @Service +@RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { - @Autowired + private final DeviceManagementService deviceManagementService; private final ResourceDiscoveryProperties processorProperties; + private final TaskExecutor taskExecutor; + private final LinkedBlockingDeque queue = new LinkedBlockingDeque<>(); + private Thread processQueueThread; + private long lastRegistrationStartTimestamp = -1L; - public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { - this.processorProperties = processorProperties; + public void queueForRegistration(@NonNull Device device) { + if (processorProperties.isSalRegistrationEnabled()) + queue.add(device); } public void register(Device device) { @@ -50,9 +58,9 @@ public void register(Device device) { */ String device_name = device.getName(); - Integer cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); - Integer ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; - Integer disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; + int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); + int ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; + int disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; String external_ip_address = device.getIpAddress(); String device_username = device.getUsername(); String device_password = new String(device.getPassword()); @@ -68,11 +76,11 @@ public void register(Device device) { //register_device_message.put("timestamp",(int)(clock.millis()/1000)); String register_device_message_string = get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); - log.error("topic is "+get_registration_topic_name(application_name)); - log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); - log.error("broker port is "+processorProperties.getNebulous_broker_port()); - log.error("username is "+processorProperties.getNebulous_broker_username()); - log.error("password is "+processorProperties.getNebulous_broker_password()); + log.info("topic is {}", get_registration_topic_name(application_name)); + log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); + log.info("broker port is {}", processorProperties.getNebulous_broker_port()); + log.info("username is {}", processorProperties.getNebulous_broker_username()); + log.info("password is {}", StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ? "" : ""); //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { @@ -96,22 +104,73 @@ public void register(Device device) { } private String get_registration_topic_name(String application_name) { - return "eu.nebulouscloud.exn.sal.node.create"; + return processorProperties.getRegistration_topic_name(); //return ("eu.nebulouscloud.exn.sal.edge." + application_name); } @Override - public void afterPropertiesSet() throws Exception { - if ( processorProperties.getNebulous_broker_password()!=null && - processorProperties.getNebulous_broker_username()!=null && - processorProperties.getNebulous_broker_ip_address()!=null - ){ + public void afterPropertiesSet() { + if (! processorProperties.isSalRegistrationEnabled()) { + log.info("SAL registration is disabled due to configuration"); + return; + } + + if ( StringUtils.isNotBlank(processorProperties.getNebulous_broker_ip_address()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_username()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ) + { log.info("Successful setting of properties for communication with SAL"); - }else{ - log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); - log.error("username is "+processorProperties.getNebulous_broker_username()); - log.error("password is "+processorProperties.getNebulous_broker_password()); - throw new Exception("Required data is null - broker ip is "+processorProperties.getNebulous_broker_ip_address()+" username is "+processorProperties.getNebulous_broker_username()+" password is "+processorProperties.getNebulous_broker_password()); + taskExecutor.execute(this::processQueue); + taskExecutor.execute(this::checkProcessQueue); + } else { + String message = String.format("Nebulous broker configuration is missing: ip-address=%s, username=%s, password=%s", + processorProperties.getNebulous_broker_ip_address(), + processorProperties.getNebulous_broker_username(), + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ? "" : ""); + log.error(message); + throw new RuntimeException(message); + } + } + + public void processQueue() { + processQueueThread = Thread.currentThread(); + while (true) { + Device device = null; + try { + device = queue.take(); + log.warn("SALRegistrationService: processQueue(): Will register device: {}", device); + lastRegistrationStartTimestamp = System.currentTimeMillis(); + register(device); + lastRegistrationStartTimestamp = -1L; + device.setRegisteredToSAL(true); + deviceManagementService.update(device); + log.warn("SALRegistrationService: processQueue(): Device registered to SAL: {}", device); + } catch (InterruptedException e) { + log.warn("SALRegistrationService: processQueue(): Interrupted. Will not register device to SAL: {}", device); + lastRegistrationStartTimestamp = -1L; +// break; + } catch (Exception e) { + log.warn("SALRegistrationService: processQueue(): EXCEPTION caught. Will not register device to SAL: {}", device, e); + lastRegistrationStartTimestamp = -1L; + } + } + } + + public void checkProcessQueue() { + while (true) { + try { + Thread.sleep(1000); + if (processQueueThread!=null && lastRegistrationStartTimestamp > 0) { + long runningTime = System.currentTimeMillis() - lastRegistrationStartTimestamp; + if (runningTime > processorProperties.getSalRegistrationTimeout()) { + log.warn("SALRegistrationService: checkProcessQueue(): Method 'processQueue' is running for too log. Will attempt to interrupt it"); + processQueueThread.interrupt(); + lastRegistrationStartTimestamp = -1L; + } + } + } catch (Exception e) { + log.warn("SALRegistrationService: checkProcessQueue(): EXCEPTION caught: ", e); + } } } } From 2dae6582de0359a4f33b0d8c066b1b55ecad2227 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 9 Apr 2024 10:33:42 +0300 Subject: [PATCH 082/132] RD: Added temp. log messages --- .../discovery/monitor/service/AbstractMonitorService.java | 2 ++ .../monitor/service/DeviceMetricsMonitorService.java | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java index f36e530..48a80dd 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java @@ -34,6 +34,7 @@ public abstract class AbstractMonitorService implements InitializingBean, Broker @Override public void afterPropertiesSet() throws Exception { // Initialize device status listener + log.warn("!!!!!!!!!!! AbstractMonitorService.afterPropertiesSet: {}", this.getClass()); taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionStartupDelay())); } @@ -43,6 +44,7 @@ private void initializeDeviceStatusListener() { try { brokerUtil.subscribe(topic, this); log.debug("{}: Subscribed to topic: {}", name, topic); + log.warn("!!!!!!!!!!! AbstractMonitorService.initializeDeviceStatusListener: {}: Subscribed to topic: {}", this.getClass(), topic); } catch (Exception e) { log.error("{}: ERROR while subscribing to topic: {}\n", name, topic, e); taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 0b355df..8492b16 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -32,6 +32,7 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties { super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.: {}", monitorProperties); } @Override @@ -41,10 +42,13 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties protected void processPayload(@NonNull Map dataMap) { Object obj = dataMap.get("message"); + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: MAP: {}", dataMap); + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: OBJ: {}", obj); if (obj==null) { log.debug("DeviceMetricsMonitorService: Message does not contain device metrics (message field is null): {}", dataMap); return; } + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: OBJ-CLASS: {}", obj.getClass()); if (obj instanceof Map infoMap) { if (infoMap.isEmpty()) log.debug("DeviceMetricsMonitorService: Device metrics map (message field) is empty: {}", dataMap); @@ -72,6 +76,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Get registered device using IP address Optional result = deviceManagementService.getByIpAddress(ipAddress); + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.updateDeviceMetrics: device: {}", result); if (result.isEmpty()) { log.debug("DeviceMetricsMonitorService: Device metrics IP address does not match any registered device: {}", infoMap); return; @@ -115,6 +120,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { deviceManagementService.update(device); log.debug("DeviceMetricsMonitorService: Device metrics updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), metrics); + log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.updateDeviceMetrics: DONE: {}", device); } catch (Exception e) { log.warn("DeviceMetricsMonitorService: EXCEPTION while processing device metrics map: {}\n", infoMap, e); } From 0f35b8263b14d9b06f33acf4c4de0b9842ca75fc Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 9 Apr 2024 10:34:21 +0300 Subject: [PATCH 083/132] Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. --- .../discovery/monitor/service/AbstractMonitorService.java | 2 -- .../monitor/service/DeviceMetricsMonitorService.java | 6 ------ 2 files changed, 8 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java index 48a80dd..f36e530 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/AbstractMonitorService.java @@ -34,7 +34,6 @@ public abstract class AbstractMonitorService implements InitializingBean, Broker @Override public void afterPropertiesSet() throws Exception { // Initialize device status listener - log.warn("!!!!!!!!!!! AbstractMonitorService.afterPropertiesSet: {}", this.getClass()); taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionStartupDelay())); } @@ -44,7 +43,6 @@ private void initializeDeviceStatusListener() { try { brokerUtil.subscribe(topic, this); log.debug("{}: Subscribed to topic: {}", name, topic); - log.warn("!!!!!!!!!!! AbstractMonitorService.initializeDeviceStatusListener: {}: Subscribed to topic: {}", this.getClass(), topic); } catch (Exception e) { log.error("{}: ERROR while subscribing to topic: {}\n", name, topic, e); taskScheduler.schedule(this::initializeDeviceStatusListener, Instant.now().plusSeconds(monitorProperties.getSubscriptionRetryDelay())); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 8492b16..0b355df 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -32,7 +32,6 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties { super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.: {}", monitorProperties); } @Override @@ -42,13 +41,10 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties protected void processPayload(@NonNull Map dataMap) { Object obj = dataMap.get("message"); - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: MAP: {}", dataMap); - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: OBJ: {}", obj); if (obj==null) { log.debug("DeviceMetricsMonitorService: Message does not contain device metrics (message field is null): {}", dataMap); return; } - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.processPayload: OBJ-CLASS: {}", obj.getClass()); if (obj instanceof Map infoMap) { if (infoMap.isEmpty()) log.debug("DeviceMetricsMonitorService: Device metrics map (message field) is empty: {}", dataMap); @@ -76,7 +72,6 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Get registered device using IP address Optional result = deviceManagementService.getByIpAddress(ipAddress); - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.updateDeviceMetrics: device: {}", result); if (result.isEmpty()) { log.debug("DeviceMetricsMonitorService: Device metrics IP address does not match any registered device: {}", infoMap); return; @@ -120,7 +115,6 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { deviceManagementService.update(device); log.debug("DeviceMetricsMonitorService: Device metrics updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), metrics); - log.warn("!!!!!!!!!!! DeviceMetricsMonitorService.updateDeviceMetrics: DONE: {}", device); } catch (Exception e) { log.warn("DeviceMetricsMonitorService: EXCEPTION while processing device metrics map: {}\n", infoMap, e); } From e3bd385283a175eb91178cf20789ff59d31da845 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 9 Apr 2024 11:56:40 +0300 Subject: [PATCH 084/132] RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) --- .../service/DeviceMetricsMonitorService.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java index 0b355df..d042696 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceMetricsMonitorService.java @@ -6,7 +6,6 @@ import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceMetrics; -import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -32,6 +31,7 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties { super("DeviceMetricsMonitorService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.deviceManagementService = deviceManagementService; + log.trace("DeviceMetricsMonitorService.: {}", monitorProperties); } @Override @@ -41,6 +41,7 @@ public DeviceMetricsMonitorService(ResourceDiscoveryProperties monitorProperties protected void processPayload(@NonNull Map dataMap) { Object obj = dataMap.get("message"); + log.trace("DeviceMetricsMonitorService: dataMap={}, message={}, message-class={}", dataMap, obj, obj!=null ? obj.getClass() : null); if (obj==null) { log.debug("DeviceMetricsMonitorService: Message does not contain device metrics (message field is null): {}", dataMap); return; @@ -63,6 +64,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { String clientId = stringValue(metricsMap.get("clientId")); String ipAddress = stringValue(metricsMap.get("ipAddress")); String timestampStr = stringValue(metricsMap.get("receivedAtServer")); + log.debug("DeviceMetricsMonitorService: client={}, ip={}, ts={}", clientId, ipAddress, timestampStr); if (clientId.isEmpty() || ipAddress.isEmpty() || timestampStr.isEmpty()) { log.warn("DeviceMetricsMonitorService: Device metrics received do not contain clientId or ipAddress or receivedAtServer. Ignoring them: {}", metricsMap); return; @@ -72,9 +74,18 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { // Get registered device using IP address Optional result = deviceManagementService.getByIpAddress(ipAddress); + log.debug("DeviceMetricsMonitorService: device-by-ip: {}", result); if (result.isEmpty()) { log.debug("DeviceMetricsMonitorService: Device metrics IP address does not match any registered device: {}", infoMap); - return; + + result = deviceManagementService.getAll().stream() + .filter(d->d.getStatusUpdate()!=null) + .filter(d->StringUtils.isNotBlank(d.getStatusUpdate().getClientId())) + .filter(d->StringUtils.equalsIgnoreCase(d.getStatusUpdate().getClientId(), clientId)) + .findAny(); + log.debug("DeviceMetricsMonitorService: device-by-clientId: {}", result); + if (result.isEmpty()) + return; } Device device = result.get(); @@ -115,6 +126,7 @@ private void updateDeviceMetrics(@NonNull Map infoMap) { deviceManagementService.update(device); log.debug("DeviceMetricsMonitorService: Device metrics updated for device: id={}, ip-address={}, update={}", device.getId(), device.getIpAddress(), metrics); + log.debug("DeviceMetricsMonitorService: Device statistics updated: {}", device); } catch (Exception e) { log.warn("DeviceMetricsMonitorService: EXCEPTION while processing device metrics map: {}\n", infoMap, e); } From 58ae0e1a8baa46ad2b58be2288fc2f1f0b90a2e1 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Mon, 29 Apr 2024 17:50:17 +0300 Subject: [PATCH 085/132] Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 --- .../broker_communication/SALCommunicator.java | 8 +++---- .../service/SALRegistrationService.java | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 10f28da..4b692aa 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -142,7 +142,7 @@ private static void register_devices(String request_body_file, String sessionID, } - public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password) { + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, double device_longitude, double device_latitude) { JSONObject root_json_object = new JSONObject(); JSONObject loginCredential = new JSONObject(); @@ -168,12 +168,12 @@ public static String get_device_registration_json(String internal_ip_address, St operatingSystem.put("operatingSystemFamily", "UBUNTU"); operatingSystem.put("operatingSystemArchitecture", "ARMv8"); - operatingSystem.put("operatingSystemVersion", 1804); + operatingSystem.put("operatingSystemVersion", 2204); geoLocation.put("city", city_name); geoLocation.put("country", country_name); - geoLocation.put("latitude", new Random().nextFloat(-90, 90)); - geoLocation.put("longitude", new Random().nextFloat(-90, 90)); + geoLocation.put("latitude", device_longitude); + geoLocation.put("longitude", device_latitude); nodeProperties.put("providerId", provider_id); nodeProperties.put("numberOfCores", cpu_cores); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1e0d08d..872f1d6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -58,6 +58,26 @@ public void register(Device device) { String device_password = new String(device.getPassword()); String device_pub_key = new String(device.getPublicKey()); //TODO get here private key instead and pass this to device registration //TODO implement provider here: String provider = device.getProvider(); + String provider_id = device.getOwner(); //TODO improve this + String city_name = ""; //TODO improve this + String country_name = ""; //TODO improve this + String internal_ip = ""; //TODO improve this + double device_longitude = 0,device_latitude =0; + if (device.getLocation()!=null){ + String location_name = ""; + if (device.getLocation().getName()!=null) { + location_name = device.getLocation().getName(); + } + if (device.getLocation().getCity()!=null) { + city_name = device.getLocation().getCity(); + } + if (device.getLocation().getCountry()!=null){ + country_name= device.getLocation().getCountry(); + } + device_longitude = device.getLocation().getLongitude(); + device_latitude = device.getLocation().getLatitude(); + } + //String network_rx =device_info.get("RX"); //String network_tx = device_info.get("TX"); @@ -67,7 +87,7 @@ public void register(Device device) { //register_device_message.put("device_name",device_name); //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json("10.100.100",external_ip_address,cores,ram_gb,disk_gb,device_name,"test_provider","Athens","Greece", device_username, device_password); + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,device_longitude,device_latitude); log.error("topic is "+get_registration_topic_name(application_name)); log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); log.error("broker port is "+processorProperties.getNebulous_broker_port()); From c03fcb288be8944af1fdab7eca2d897457d41e18 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 30 Apr 2024 18:29:15 +0300 Subject: [PATCH 086/132] Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 --- .../service/SALRegistrationService.java | 129 +++++++++++++----- 1 file changed, 94 insertions(+), 35 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 872f1d6..fcde53a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -3,28 +3,36 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; import eu.nebulous.resource.discovery.monitor.model.Device; +import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.json.simple.JSONObject; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; import java.time.Clock; -import java.util.ArrayList; import java.util.Collections; import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; -import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.*; +import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.get_device_registration_json; @Slf4j @Service +@RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { - @Autowired + private final DeviceManagementService deviceManagementService; private final ResourceDiscoveryProperties processorProperties; - - public SALRegistrationService(ResourceDiscoveryProperties processorProperties) { - this.processorProperties = processorProperties; + private final TaskExecutor taskExecutor; + private final LinkedBlockingDeque queue = new LinkedBlockingDeque<>(); + private Thread processQueueThread; + private long lastRegistrationStartTimestamp = -1L; + + public void queueForRegistration(@NonNull Device device) { + if (processorProperties.isSalRegistrationEnabled()) + queue.add(device); } public void register(Device device) { @@ -50,14 +58,16 @@ public void register(Device device) { */ String device_name = device.getName(); - Integer cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); - Integer ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; - Integer disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; + int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); + int ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; + int disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; String external_ip_address = device.getIpAddress(); String device_username = device.getUsername(); String device_password = new String(device.getPassword()); String device_pub_key = new String(device.getPublicKey()); //TODO get here private key instead and pass this to device registration //TODO implement provider here: String provider = device.getProvider(); + //String network_rx =device_info.get("RX"); + //String network_tx = device_info.get("TX"); String provider_id = device.getOwner(); //TODO improve this String city_name = ""; //TODO improve this String country_name = ""; //TODO improve this @@ -78,28 +88,25 @@ public void register(Device device) { device_latitude = device.getLocation().getLatitude(); } - //String network_rx =device_info.get("RX"); - //String network_tx = device_info.get("TX"); - Clock clock = Clock.systemUTC(); //JSONObject register_device_message = new JSONObject(); //register_device_message.put("device_name",device_name); //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,device_longitude,device_latitude); - log.error("topic is "+get_registration_topic_name(application_name)); - log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); - log.error("broker port is "+processorProperties.getNebulous_broker_port()); - log.error("username is "+processorProperties.getNebulous_broker_username()); - log.error("password is "+processorProperties.getNebulous_broker_password()); + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,device_longitude, device_latitude); + log.info("topic is {}", get_registration_topic_name(application_name)); + log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); + log.info("broker port is {}", processorProperties.getNebulous_broker_port()); + log.info("username is {}", processorProperties.getNebulous_broker_username()); + log.info("password is {}", StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ? "" : ""); //String sal_running_applications_reply = request_running_applications_AMQP(); //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { - SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); - //TODO handle the response here - Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to register device " + device_name); + SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + //TODO handle the response here + Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); + log.info("The response received while trying to register device " + device_name); //} /* This is some realtime information, could be retrieved with a different call to the EMS. @@ -116,22 +123,74 @@ public void register(Device device) { } private String get_registration_topic_name(String application_name) { - return "eu.nebulouscloud.exn.sal.node.create"; + return processorProperties.getRegistration_topic_name(); //return ("eu.nebulouscloud.exn.sal.edge." + application_name); } @Override - public void afterPropertiesSet() throws Exception { - if ( processorProperties.getNebulous_broker_password()!=null && - processorProperties.getNebulous_broker_username()!=null && - processorProperties.getNebulous_broker_ip_address()!=null - ){ + public void afterPropertiesSet() { + if (! processorProperties.isSalRegistrationEnabled()) { + log.info("SAL registration is disabled due to configuration"); + return; + } + + if ( StringUtils.isNotBlank(processorProperties.getNebulous_broker_ip_address()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_username()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ) + { log.info("Successful setting of properties for communication with SAL"); - }else{ - log.error("broker ip is "+processorProperties.getNebulous_broker_ip_address()); - log.error("username is "+processorProperties.getNebulous_broker_username()); - log.error("password is "+processorProperties.getNebulous_broker_password()); - throw new Exception("Required data is null - broker ip is "+processorProperties.getNebulous_broker_ip_address()+" username is "+processorProperties.getNebulous_broker_username()+" password is "+processorProperties.getNebulous_broker_password()); + taskExecutor.execute(this::processQueue); + taskExecutor.execute(this::checkProcessQueue); + } else { + String message = String.format("Nebulous broker configuration is missing: ip-address=%s, username=%s, password=%s", + processorProperties.getNebulous_broker_ip_address(), + processorProperties.getNebulous_broker_username(), + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ? "" : ""); + log.error(message); + throw new RuntimeException(message); + } + } + + public void processQueue() { + processQueueThread = Thread.currentThread(); + while (true) { + Device device = null; + try { + device = queue.take(); + log.warn("SALRegistrationService: processQueue(): Will register device: {}", device); + lastRegistrationStartTimestamp = System.currentTimeMillis(); + register(device); + lastRegistrationStartTimestamp = -1L; + device.setRegisteredToSAL(true); + deviceManagementService.update(device); + log.warn("SALRegistrationService: processQueue(): Device registered to SAL: {}", device); + } catch (InterruptedException e) { + log.warn("SALRegistrationService: processQueue(): Interrupted. Will not register device to SAL: {}", device); + lastRegistrationStartTimestamp = -1L; +// break; + } catch (Exception e) { + log.warn("SALRegistrationService: processQueue(): EXCEPTION caught. Will not register device to SAL: {}", device, e); + lastRegistrationStartTimestamp = -1L; + } + } + } + + public void checkProcessQueue() { + while (true) { + try { + Thread.sleep(1000); + if (processQueueThread!=null && lastRegistrationStartTimestamp > 0) { + long runningTime = System.currentTimeMillis() - lastRegistrationStartTimestamp; + if (runningTime > processorProperties.getSalRegistrationTimeout()) { + log.warn("SALRegistrationService: checkProcessQueue(): Method 'processQueue' is running for too log. Will attempt to interrupt it"); + processQueueThread.interrupt(); + lastRegistrationStartTimestamp = -1L; + } + } + } catch (Exception e) { + log.warn("SALRegistrationService: checkProcessQueue(): EXCEPTION caught: ", e); + } } } } + From 8e5ecdbfa7361a5adb9ce4ba1e99cdf3c1df3886 Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 1 May 2024 14:43:11 +0300 Subject: [PATCH 087/132] RD: Updated Dockerfile --- resource-discovery/Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index 93d40ff..f567a01 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -10,7 +10,6 @@ ENV BASEDIR /app WORKDIR ${BASEDIR} COPY src ${BASEDIR}/src COPY pom.xml ${BASEDIR}/ -COPY run.sh ${BASEDIR}/ RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install && \ java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract @@ -26,9 +25,14 @@ RUN wget --progress=dot:giga -O /usr/local/bin/dumb-init \ https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 && \ chmod +x /usr/local/bin/dumb-init -#RUN apt-get update && \ -# apt-get install -y netcat vim iputils-ping && \ -# rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + netcat=1.218-4ubuntu1 \ + && rm -rf /var/lib/apt/lists/* +# iputils-ping=3:20211215-1 \ +# telnet=0.17-44build1 \ +# vim=2:8.2.3995-1ubuntu2.16 \ +# net-tools=1.60+git20181103.0eebece-1ubuntu5 \ # Add RD user ARG RD_USER=rd @@ -46,8 +50,10 @@ COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/dependencies COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/spring-boot-loader ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/snapshot-dependencies ${BASEDIR} COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/application ${BASEDIR} -COPY --chown=${RD_USER}:${RD_USER} --from=rd-builder /app/run.sh ${BASEDIR} -RUN chmod +x run.sh + +COPY --chown=${RD_USER}:${RD_USER} run.sh ${BASEDIR} +COPY --chown=${RD_USER}:${RD_USER} wait_for_mongodb.sh ${BASEDIR} +RUN chmod +x run.sh wait_for_mongodb.sh EXPOSE 8080 From 5133aa6c18829088f004d39e4ca921406b6de69c Mon Sep 17 00:00:00 2001 From: ipatini Date: Wed, 1 May 2024 14:58:31 +0300 Subject: [PATCH 088/132] RD: Updated run.sh and added wait_for_mongodb.sh --- resource-discovery/run.sh | 3 +++ resource-discovery/wait_for_mongodb.sh | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 resource-discovery/wait_for_mongodb.sh diff --git a/resource-discovery/run.sh b/resource-discovery/run.sh index 54cd70f..8e74d5f 100644 --- a/resource-discovery/run.sh +++ b/resource-discovery/run.sh @@ -5,6 +5,9 @@ PREVWORKDIR=`pwd` BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) cd ${BASEDIR} +# Wait for MongoDB to start +./wait_for_mongodb.sh + # Read JASYPT password (decrypts encrypted configuration settings) #if [[ -z "$JASYPT_PASSWORD" ]]; then # printf "Configuration Password: " diff --git a/resource-discovery/wait_for_mongodb.sh b/resource-discovery/wait_for_mongodb.sh new file mode 100644 index 0000000..400ef54 --- /dev/null +++ b/resource-discovery/wait_for_mongodb.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Define MongoDB connection details +MONGODB_HOST="localhost" +MONGODB_PORT="27017" + +# Function to check if MongoDB is ready +wait_for_mongodb() { + echo "Waiting for MongoDB to be ready..." + until nc -z $MONGODB_HOST $MONGODB_PORT + do + sleep 1 + done + echo "MongoDB is up and running!" +} + +# Call the function to wait for MongoDB +wait_for_mongodb \ No newline at end of file From cceea796ecb3b5b000f0d1d49016100cab5944ef Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 9 May 2024 13:08:53 +0300 Subject: [PATCH 089/132] RD: Improved wait_for_mongodb.sh script --- resource-discovery/wait_for_mongodb.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-discovery/wait_for_mongodb.sh b/resource-discovery/wait_for_mongodb.sh index 400ef54..f61f994 100644 --- a/resource-discovery/wait_for_mongodb.sh +++ b/resource-discovery/wait_for_mongodb.sh @@ -1,12 +1,12 @@ #!/bin/bash # Define MongoDB connection details -MONGODB_HOST="localhost" -MONGODB_PORT="27017" +if [ -z "$MONGODB_HOST" ]; then MONGODB_HOST="localhost"; fi +if [ -z "$MONGODB_PORT" ]; then MONGODB_PORT="27017"; fi # Function to check if MongoDB is ready wait_for_mongodb() { - echo "Waiting for MongoDB to be ready..." + echo "Waiting for MongoDB ($MONGODB_HOST:$MONGODB_PORT) to be ready..." until nc -z $MONGODB_HOST $MONGODB_PORT do sleep 1 From c5d653e194d01360a425e53ab677c4b1f560cac8 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Mon, 13 May 2024 12:23:55 +0300 Subject: [PATCH 090/132] Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 --- .../broker_communication/SALCommunicator.java | 13 +++++++------ .../SynchronousBrokerPublisher.java | 10 +++++----- .../service/SALRegistrationService.java | 10 ++++++++-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 4b692aa..7d3f0af 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -142,7 +142,7 @@ private static void register_devices(String request_body_file, String sessionID, } - public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, double device_longitude, double device_latitude) { + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, int os_version, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { JSONObject root_json_object = new JSONObject(); JSONObject loginCredential = new JSONObject(); @@ -154,7 +154,7 @@ public static String get_device_registration_json(String internal_ip_address, St loginCredential.put("username", device_username); loginCredential.put("password", device_password); - loginCredential.put("privateKey", ""); + loginCredential.put("privateKey", private_key); ipAddress1.put("IpAddressType", "PUBLIC_IP"); @@ -166,9 +166,9 @@ public static String get_device_registration_json(String internal_ip_address, St ipAddress2.put("value", internal_ip_address); - operatingSystem.put("operatingSystemFamily", "UBUNTU"); - operatingSystem.put("operatingSystemArchitecture", "ARMv8"); - operatingSystem.put("operatingSystemVersion", 2204); + operatingSystem.put("operatingSystemFamily", os_family); + operatingSystem.put("operatingSystemArchitecture", os_architecture); + operatingSystem.put("operatingSystemVersion", os_version); geoLocation.put("city", city_name); geoLocation.put("country", country_name); @@ -184,6 +184,7 @@ public static String get_device_registration_json(String internal_ip_address, St root_json_object.put("name", device_name); root_json_object.put("loginCredential", loginCredential); + root_json_object.put("port", external_access_port); JSONArray ipAddresses = new JSONArray(); ipAddresses.add(ipAddress1); @@ -191,7 +192,7 @@ public static String get_device_registration_json(String internal_ip_address, St root_json_object.put("ipAddresses", ipAddresses); root_json_object.put("nodeProperties", nodeProperties); - root_json_object.put("systemArch", "ARMv8"); + root_json_object.put("systemArch", os_architecture); //TODO refine - for now assuming that the architecture of the device is the same as that of the OS installed on top of it, could be a wrong assumption root_json_object.put("scriptURL", "https://www.google.com"); root_json_object.put("jarURL", "https://www.activeeon.com/public_content/7cde3381417ff3784639dc41fa7e7cd0544a5234-morphemic-7bulls/node_13.1.0-SNAPSHOT_armv8.jar"); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 3958e17..5cdf4b9 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -35,12 +35,12 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por HashSet topics_to_publish_to = new HashSet<>(); topics_to_publish_to.add(topic); broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); - log.error("changed1"); + //log.error("changed1"); publisher_configuration_changed = true; }else{ if (!broker_and_topics_to_publish_to.get(broker_ip).contains(topic)){ broker_and_topics_to_publish_to.get(broker_ip).add(topic); - log.error("changed2"); + //log.error("changed2"); publisher_configuration_changed = true; } else{ @@ -48,9 +48,9 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } } - log.error("preliminary_outside"); + //log.error("preliminary_outside"); if (publisher_configuration_changed){ - log.error("preliminary_inside1"); + //log.error("preliminary_inside1"); // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { @@ -58,7 +58,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } publishers.clear(); for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ - log.error("preliminary_inside2"); + //log.error("preliminary_inside2"); //ArrayList publishers = new ArrayList<>(); SyncedPublisher publisher = new SyncedPublisher("resource_manager_"+broker_topic, broker_topic, true, true); publishers.add(publisher); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index fcde53a..607cb1f 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -72,6 +72,11 @@ public void register(Device device) { String city_name = ""; //TODO improve this String country_name = ""; //TODO improve this String internal_ip = ""; //TODO improve this + String os_family = "UBUNTU"; //TODO improve this + String os_architecture = "ARMv8"; //TODO improve this + int os_version = 2204; //TODO improve this + String private_key = ""; //TODO improve this + int external_access_port = device.getPort(); double device_longitude = 0,device_latitude =0; if (device.getLocation()!=null){ String location_name = ""; @@ -94,7 +99,8 @@ public void register(Device device) { //register_device_message.put("device_name",device_name); //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,device_longitude, device_latitude); + + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,os_version,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); log.info("topic is {}", get_registration_topic_name(application_name)); log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); log.info("broker port is {}", processorProperties.getNebulous_broker_port()); @@ -106,7 +112,7 @@ public void register(Device device) { SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to register device " + device_name); + log.info("The response received while trying to register device " + device_name + " is "+response.toString()); //} /* This is some realtime information, could be retrieved with a different call to the EMS. From baa6857dec64be78ddcb3baf9d89e28bcedabddc Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Wed, 15 May 2024 17:31:19 +0300 Subject: [PATCH 091/132] Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 --- .../broker_communication/BrokerPublisher.java | 13 +++++++++--- .../SynchronousBrokerPublisher.java | 13 +++++++++--- .../discovery/monitor/DeviceProcessor.java | 21 ++++++++++++++++++- .../service/SALRegistrationService.java | 15 +++++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 03e8665..852167c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -26,9 +26,13 @@ public class BrokerPublisher { private int broker_port; public BrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { + this(topic,broker_ip,broker_port,brokerUsername,brokerPassword,amqLibraryConfigurationLocation,false); + } + public BrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation, boolean hard_initialize_connector) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerPublisher){ + log.error("Could not initialize BrokerPublisher"); return; } boolean publisher_configuration_changed; @@ -48,7 +52,7 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b } - if (publisher_configuration_changed){ + if (publisher_configuration_changed || hard_initialize_connector){ // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { @@ -99,12 +103,15 @@ public void publish (String json_string_content, Collection application_ } catch (ParseException p) { log.warn( "Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); } - if (private_publisher_instance != null) { + if (!is_publisher_null()) { private_publisher_instance.send(json_object); log.info("Sent new message\n"+json_object.toJSONString()); } else { - log.error( "Could not send message to AMQP broker, as the broker ip to be used has not been specified"); + log.error( "Could not send message to AMQP broker, as the publisher instance is null"); } } } + public boolean is_publisher_null(){ + return (private_publisher_instance == null); + } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 5cdf4b9..3db8fa6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -22,12 +22,16 @@ public class SynchronousBrokerPublisher { private ExtendedConnector active_connector; private String topic; private String broker_ip; - public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { + this(topic, broker_ip, broker_port, brokerUsername, brokerPassword, amqLibraryConfigurationLocation,false); + } + + public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation,boolean hard_initialize_connector) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerPublisher){ + log.error("Unable to initialize SynchronousBrokerPublisher"); return; } boolean publisher_configuration_changed; @@ -49,7 +53,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } //log.error("preliminary_outside"); - if (publisher_configuration_changed){ + if (publisher_configuration_changed || hard_initialize_connector){ //log.error("preliminary_inside1"); // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); @@ -71,7 +75,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por } //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); - active_connector = new ExtendedConnector("resource_manager" + active_connector = new ExtendedConnector("resource_manager_synchronous" , new CustomConnectorHandler() {} , publishers , List.of(), @@ -146,4 +150,7 @@ public Map publish_for_response (String json_string_content, Collection } return reply; } + public boolean is_publisher_null(){ + return (private_publisher_instance == null); + } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 8fe87bb..4b8d893 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -138,7 +138,26 @@ private void processFailedDevices() { lost_device_message.put("device_name",device.getName()); Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); - BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(),processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + log.info("Creating new BrokerPublisher to publish device lost message"); + BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + int sending_attempt = 1; + while (device_lost_publisher.is_publisher_null()){ + + try { + log.info("Attempting to recreate new BrokerPublisher to publish the device lost message"); + log.info("The topic name is "+processorProperties.getLost_device_topic()+", the broker ip is "+ processorProperties.getNebulous_broker_ip_address()+", the broker port is "+ processorProperties.getNebulous_broker_port()+", the username is "+ processorProperties.getNebulous_broker_username()+", and the password is "+ processorProperties.getNebulous_broker_password()); + if (sending_attempt<=2) { + device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + }else{ + log.warn("Will now attempt to reset the BrokerPublisher connector"); + device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), "",true); + } + Thread.sleep(3000); + }catch (InterruptedException i){ + i.printStackTrace(); + } + sending_attempt++; + } device_lost_publisher.publish(lost_device_message.toJSONString(), Collections.singleton("")); log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 607cb1f..153ac91 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -110,6 +110,21 @@ public void register(Device device) { //ArrayList applications = get_running_applications(sal_running_applications_reply); //for (String application_name:applications) { SynchronousBrokerPublisher register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + int sending_attempt = 1; + while (register_device_publisher.is_publisher_null()){ + if (sending_attempt<=2) { + register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + }else{ + log.warn("Will now attempt to reset the Synchronous publisher connector"); + register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + } + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + sending_attempt++; + } //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.info("The response received while trying to register device " + device_name + " is "+response.toString()); From 644c98e6d12e16ce6dfb12f1a4fae108dae9b751 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 7 Oct 2024 14:16:39 +0300 Subject: [PATCH 092/132] Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' --- .../discovery/broker_communication/SALCommunicator.java | 4 ++-- .../resource/discovery/monitor/model/DeviceStatus.java | 2 +- .../monitor/service/DeviceLifeCycleRequestService.java | 6 ++++++ .../monitor/service/DeviceManagementService.java | 9 +++++++++ .../registration/service/SALRegistrationService.java | 8 +++++--- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 7d3f0af..494bc1c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -142,7 +142,7 @@ private static void register_devices(String request_body_file, String sessionID, } - public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, int os_version, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { JSONObject root_json_object = new JSONObject(); JSONObject loginCredential = new JSONObject(); @@ -194,7 +194,7 @@ public static String get_device_registration_json(String internal_ip_address, St root_json_object.put("nodeProperties", nodeProperties); root_json_object.put("systemArch", os_architecture); //TODO refine - for now assuming that the architecture of the device is the same as that of the OS installed on top of it, could be a wrong assumption root_json_object.put("scriptURL", "https://www.google.com"); - root_json_object.put("jarURL", "https://www.activeeon.com/public_content/7cde3381417ff3784639dc41fa7e7cd0544a5234-morphemic-7bulls/node_13.1.0-SNAPSHOT_armv8.jar"); + root_json_object.put("jarURL", jar_url); //JSONObject root_json_object = JsonFileParser.parse(request_body_file); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java index e124edf..467f2e1 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/DeviceStatus.java @@ -5,5 +5,5 @@ public enum DeviceStatus { ONBOARDING, ONBOARDED, ONBOARD_ERROR, HEALTHY, SUSPECT, FAILED, BUSY, IDLE, - OFFBOARDING, OFFBOARDED, OFFBOARD_ERROR + OFFBOARDING, OFFBOARDED, OFFBOARD_ERROR, COMPROMISED } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java index 160a321..5d40768 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceLifeCycleRequestService.java @@ -3,6 +3,7 @@ import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; import eu.nebulous.resource.discovery.common.BrokerUtil; import eu.nebulous.resource.discovery.common.REQUEST_TYPE; +import eu.nebulous.resource.discovery.monitor.model.ArchivedDevice; import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceException; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; @@ -41,6 +42,11 @@ public void reinstallRequest(String id) { // Send request brokerUtil.sendMessage(properties.getDeviceLifeCycleRequestsTopic(), onboardingRequest); + Optional archived_device = deviceManagementService.isCompromised(device.getId()); + if (archived_device.isPresent()){ + log.warn("reinstallRequest: Attempted to onboard a device which had been compromised in the past"); + return; + } device.setStatus(DeviceStatus.ONBOARDING); log.debug("reinstallRequest: Save updated device: id={}, device={}", device.getId(), device); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index bcf4a67..ac4ee52 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -46,6 +46,15 @@ public Optional getByIpAddress(@NonNull String ipAddress) { return deviceRepository.findByIpAddress(ipAddress); } + public void setCompromised(@NonNull String id){ + Optional device = getById(id); + device.ifPresent(value -> value.setStatus(DeviceStatus.COMPROMISED)); + } + + public Optional isCompromised(@NonNull String id){ + return archivedDeviceRepository.findById(id).filter(device -> device.getStatus()!= null && device.getStatus() == DeviceStatus.COMPROMISED ); + } + public boolean isIpAddressInUse(@NonNull String ipAddress) { return deviceRepository.findByIpAddress(ipAddress).isPresent(); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 153ac91..c042feb 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -37,7 +37,7 @@ public void queueForRegistration(@NonNull Device device) { public void register(Device device) { - String application_name = "default-application"; //TODO decide on this + String application_name = ""; //TODO decide on this Map device_info = device.getDeviceInfo(); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS @@ -73,7 +73,9 @@ public void register(Device device) { String country_name = ""; //TODO improve this String internal_ip = ""; //TODO improve this String os_family = "UBUNTU"; //TODO improve this - String os_architecture = "ARMv8"; //TODO improve this + //String os_architecture = "ARMv8"; //TODO improve this + String os_architecture = device_info.get("OS_ARCHITECTURE").equals("aarch64") ? "armv8" : device_info.get("OS_ARCHITECTURE").equals("armv8") ?"armv8": device_info.get("OS_ARCHITECTURE").equals("armv7l") ? "armv7l": "AMD"; + String jar_url = os_architecture.equals("armv8") ? "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_arm_v8.jar" : os_architecture.equals("armv7l") ? "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_arm_v7l.jar" : "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_amd.jar"; int os_version = 2204; //TODO improve this String private_key = ""; //TODO improve this int external_access_port = device.getPort(); @@ -100,7 +102,7 @@ public void register(Device device) { //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,os_version,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,jar_url,os_version,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); log.info("topic is {}", get_registration_topic_name(application_name)); log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); log.info("broker port is {}", processorProperties.getNebulous_broker_port()); From a635914ad41321703e43028b9edbd45f885131a3 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:03:43 +0300 Subject: [PATCH 093/132] RD: Added API Key authentication --- .../ResourceDiscoveryProperties.java | 4 + .../resource/discovery/SecurityConfig.java | 75 ++++++++++++++++++- .../src/main/resources/application.yml | 3 + 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 735eb4a..e8a8dd3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -89,6 +89,10 @@ public class ResourceDiscoveryProperties { // Users private List users; + // API-Key authentication fields + private boolean apiKeyAuthenticationEnabled; + private String apiKeyValue; + // Nebulous broker subscription details private String nebulous_broker_ip_address; private int nebulous_broker_port; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java index 399e40e..4849cb3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/SecurityConfig.java @@ -1,21 +1,31 @@ package eu.nebulous.resource.discovery; +import jakarta.servlet.Filter; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.userdetails.User; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.provisioning.InMemoryUserDetailsManager; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import java.security.SecureRandom; +import java.util.Collections; import static org.springframework.security.config.Customizer.withDefaults; @@ -25,6 +35,15 @@ @EnableMethodSecurity @RequiredArgsConstructor public class SecurityConfig { + private final static String USERNAME_REQUEST_HEADER = "X-SSO-USER"; + private final static String USERNAME_REQUEST_PARAM = "ssoUser"; + private final static String API_KEY_REQUEST_HEADER = "X-API-KEY"; + private final static String API_KEY_REQUEST_PARAM = "apiKey"; + + public final static String SSO_USER_DEFAULT = "anonymous"; + public final static String SSO_USER_PREFIX = "SSO "; + public final static String SSO_USER_ROLE = "ROLE_SSO_USER"; + private final ResourceDiscoveryProperties properties; @Bean @@ -34,7 +53,9 @@ public SecurityFilterChain securityFilterChain(HttpSecurity httpSecurity) throws .authorizeHttpRequests(authorize -> authorize.requestMatchers( "/discovery/**", "/*.html").authenticated()) .authorizeHttpRequests(authorize -> authorize.anyRequest().permitAll()) - .csrf(AbstractHttpConfigurer::disable); + .addFilterAfter(apiKeyAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class) + .csrf(AbstractHttpConfigurer::disable) + .sessionManagement(sm -> sm.sessionCreationPolicy(SessionCreationPolicy.ALWAYS)); return httpSecurity.build(); } @@ -76,4 +97,56 @@ public boolean matches(CharSequence rawPassword, String encodedPassword) { } }; }*/ + + public Filter apiKeyAuthenticationFilter() { + return (servletRequest, servletResponse, filterChain) -> { + if (properties.isApiKeyAuthenticationEnabled() && StringUtils.isNotBlank(properties.getApiKeyValue())) { + if (servletRequest instanceof HttpServletRequest request && servletResponse instanceof HttpServletResponse) { + + String apiKey = request.getHeader(API_KEY_REQUEST_HEADER); + if (StringUtils.isBlank(apiKey)) { + apiKey = request.getParameter(API_KEY_REQUEST_PARAM); + } + if (StringUtils.isNotBlank(apiKey)) { + log.debug("apiKeyAuthenticationFilter: API Key found"); + + if (properties.getApiKeyValue().equals(apiKey)) { + log.debug("apiKeyAuthenticationFilter: API Key is correct"); + try { + // Get SSO username if passed + String username = request.getHeader(USERNAME_REQUEST_HEADER); + if (StringUtils.isBlank(username)) { + username = request.getParameter(USERNAME_REQUEST_PARAM); + } + if (StringUtils.isBlank(username)) { + username = SSO_USER_DEFAULT; + } + username = SSO_USER_PREFIX + username; + + // construct one of Spring's auth tokens + UsernamePasswordAuthenticationToken authentication = + new UsernamePasswordAuthenticationToken(username, properties.getApiKeyValue(), + Collections.singletonList(new SimpleGrantedAuthority(SSO_USER_ROLE))); + // store completed authentication in security context + SecurityContextHolder.getContext().setAuthentication(authentication); + log.info("apiKeyAuthenticationFilter: Successful authentication with API Key. SSO user: {}", username); + } catch (Exception e) { + log.error("apiKeyAuthenticationFilter: EXCEPTION: ", e); + } + } else { + log.debug("apiKeyAuthenticationFilter: API Key is incorrect"); + } + } else { + log.debug("apiKeyAuthenticationFilter: No API Key found in request headers or parameters"); + } + } else { + throw new IllegalArgumentException("API Key Authentication filter does not support non-HTTP requests and responses. Req-class: " + +servletRequest.getClass().getName()+" Resp-class: "+servletResponse.getClass().getName()); + } + } + + // continue down the chain + filterChain.doFilter(servletRequest, servletResponse); + }; + } } diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 2d707bf..3c16b05 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -42,4 +42,7 @@ discovery: password: '$2a$10$I6GSOKiY5n4/Ql0LA7Js0.4HT4UXVCNaNpGv5UdZt/brEdv/F.ttG' # user1 (BCrypt; 10 iterations) roles: [ USER ] +# apiKeyAuthenticationEnabled: true +# apiKeyValue: 1234567890 + #logging.level.eu.nebulous.resource.discovery.registration.RegistrationRequestProcessor: TRACE \ No newline at end of file From 32dbaa7d7e3dc80f9ac0eb01a2eb2acf6fcd123d Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:06:04 +0300 Subject: [PATCH 094/132] RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. --- resource-discovery/Dockerfile | 15 +++++---------- resource-discovery/pom.xml | 10 ++++++---- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/resource-discovery/Dockerfile b/resource-discovery/Dockerfile index f567a01..9018e63 100644 --- a/resource-discovery/Dockerfile +++ b/resource-discovery/Dockerfile @@ -1,12 +1,7 @@ -ARG BUILDER_IMAGE=docker.io/library/maven -ARG BUILDER_IMAGE_TAG=3.9.6-eclipse-temurin-21 -ARG RUN_IMAGE=docker.io/library/eclipse-temurin -ARG RUN_IMAGE_TAG=21.0.1_12-jre - # ----------------- Builder image ----------------- -FROM $BUILDER_IMAGE:$BUILDER_IMAGE_TAG as rd-builder -ENV BASEDIR /app +FROM docker.io/library/maven:3.9.6-eclipse-temurin-21 AS rd-builder +ENV BASEDIR=/app WORKDIR ${BASEDIR} COPY src ${BASEDIR}/src COPY pom.xml ${BASEDIR}/ @@ -14,11 +9,11 @@ RUN mvn -f ${BASEDIR}/pom.xml -DskipTests clean install && \ java -Djarmode=layertools -jar ${BASEDIR}/target/resource-discovery-*.jar extract # ----------------- Runtime image ----------------- -FROM $RUN_IMAGE:$RUN_IMAGE_TAG +FROM docker.io/library/eclipse-temurin:21.0.1_12-jre # Setup environment -ENV BASEDIR /opt/resource-discovery -ENV RD_HOME ${BASEDIR} +ENV BASEDIR=/opt/resource-discovery +ENV RD_HOME=${BASEDIR} # Install required and optional packages RUN wget --progress=dot:giga -O /usr/local/bin/dumb-init \ diff --git a/resource-discovery/pom.xml b/resource-discovery/pom.xml index b0258b9..7630101 100644 --- a/resource-discovery/pom.xml +++ b/resource-discovery/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.2.4 + 3.2.10 @@ -19,6 +19,8 @@ 21 ${project.artifactId}:${project.version} + 21 + 21 @@ -59,12 +61,12 @@ org.projectlombok lombok - 1.18.30 + 1.18.34 org.apache.commons commons-lang3 - 3.14.0 + 3.15.0 @@ -96,7 +98,7 @@ commons-io commons-io - 2.15.1 + 2.16.1 From 24ac9445ab33c02db2363e0bae3e03af12bddfff Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:14:09 +0300 Subject: [PATCH 095/132] RD: Code improvements (esp. wrt @PreAuthorize annotations) --- .../ResourceDiscoveryProperties.java | 4 +++- .../broker_communication/BrokerPublisher.java | 5 +---- .../DeviceManagementController.java | 21 ++++++++++++------- .../RegistrationRequestController.java | 18 +++++++++------- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index e8a8dd3..9711036 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -12,8 +12,10 @@ @Slf4j @Data @Configuration -@ConfigurationProperties(prefix = "discovery") +@ConfigurationProperties(prefix = ResourceDiscoveryProperties.CONFIG_PREFIX) public class ResourceDiscoveryProperties { + public final static String CONFIG_PREFIX = "discovery"; + // Broker configuration private String brokerURL; private String brokerUsername; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 852167c..b5299af 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -1,14 +1,11 @@ package eu.nebulous.resource.discovery.broker_communication; -import eu.nebulouscloud.exn.Connector; import eu.nebulouscloud.exn.core.Publisher; -import eu.nebulouscloud.exn.handlers.ConnectorHandler; import eu.nebulouscloud.exn.settings.StaticExnConfig; +import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index d968f8e..2ace975 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.monitor.controller; +import eu.nebulous.resource.discovery.SecurityConfig; import eu.nebulous.resource.discovery.monitor.DeviceProcessor; import eu.nebulous.resource.discovery.monitor.model.ArchivedDevice; import eu.nebulous.resource.discovery.monitor.model.Device; @@ -27,8 +28,12 @@ @RestController @RequiredArgsConstructor @RequestMapping("/monitor") -@PreAuthorize("hasAuthority('ROLE_ADMIN')") +@PreAuthorize(DeviceManagementController.REQUIRES_ADMIN_ROLE) public class DeviceManagementController { + public final static String REQUIRES_ADMIN_ROLE = "hasAuthority('ROLE_ADMIN')"; + public final static String REQUIRES_ADMIN_OR_USER_ROLE = + "hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER') || hasAuthority('"+ SecurityConfig.SSO_USER_ROLE +"')"; + private final DeviceProcessor deviceProcessor; private final DeviceManagementService deviceService; private final DeviceLifeCycleRequestService deviceLifeCycleRequestService; @@ -47,7 +52,7 @@ private boolean isAdmin(Authentication authentication) { return false; } - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/device", produces = MediaType.APPLICATION_JSON_VALUE) public List listDevicesUser(Authentication authentication) { return isAuthenticated(authentication) @@ -65,7 +70,7 @@ public List listDevicesForOwner(@PathVariable String owner) { return deviceService.getByOwner(owner); } - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/device/{id}", produces = MediaType.APPLICATION_JSON_VALUE) public Device getDevice(@PathVariable String id, Authentication authentication) { Device device = deviceService.getById(id) @@ -105,19 +110,19 @@ public void deleteDevice(@PathVariable String id) { // ------------------------------------------------------------------------ - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/device/{id}/onboard") public void onboardDevice(@PathVariable String id) { deviceLifeCycleRequestService.reinstallRequest(id); } - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/device/{id}/offboard") public void offboardDevice(@PathVariable String id) { deviceLifeCycleRequestService.uninstallRequest(id); } - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/request-update") public String requestUpdate() { deviceLifeCycleRequestService.requestInfoUpdate(); @@ -145,7 +150,7 @@ public String unarchiveDevice(@PathVariable String id, @RequestBody Map listArchivedRequests(Authentication authentication) { return deviceService.getArchivedByOwner(authentication); @@ -156,7 +161,7 @@ public List listArchivedRequestsAdmin() { return deviceService.getArchivedAll(); } - @PreAuthorize("hasAuthority('ROLE_ADMIN') || hasAuthority('ROLE_USER')") + @PreAuthorize(REQUIRES_ADMIN_OR_USER_ROLE) @GetMapping(value = "/device/archived/{id}", produces = MediaType.APPLICATION_JSON_VALUE) public ArchivedDevice getArchivedRequest(@PathVariable String id, Authentication authentication) { return deviceService.getArchivedById(id, authentication) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java index 7085470..d0b2835 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java @@ -26,6 +26,8 @@ @RequiredArgsConstructor @RequestMapping("/discovery") public class RegistrationRequestController { + private final static String REQUIRES_ADMIN_ROLE = "hasAuthority('ROLE_ADMIN')"; + private final RegistrationRequestService registrationRequestService; private final IRegistrationRequestProcessor registrationRequestProcessor; @@ -44,14 +46,14 @@ public Map whoami(Authentication authentication) { ); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/process", produces = MediaType.APPLICATION_JSON_VALUE) public Map processRequests() throws ExecutionException, InterruptedException { Future future = registrationRequestProcessor.processRequests(); return Map.of("result", future.isDone() ? future.get() : "STARTED"); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/all", produces = MediaType.APPLICATION_JSON_VALUE) public List listRequestsAdmin(Authentication authentication) { return registrationRequestService.getAll(); @@ -89,19 +91,19 @@ public void deleteRequest(@PathVariable String id, Authentication authentication registrationRequestService.deleteByIdAsUser(id, authentication); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/{id}/authorize", produces = MediaType.APPLICATION_JSON_VALUE) public RegistrationRequest authorizeRequest(@PathVariable String id, Authentication authentication) { return registrationRequestService.authorizeRequest(id, true, authentication); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/{id}/reject", produces = MediaType.APPLICATION_JSON_VALUE) public RegistrationRequest rejectRequest(@PathVariable String id, Authentication authentication) { return registrationRequestService.authorizeRequest(id, false, authentication); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/{id}/status/{newStatus}", produces = MediaType.APPLICATION_JSON_VALUE) public RegistrationRequest setRequestStatus(@PathVariable String id, @PathVariable String newStatus) { RegistrationRequestStatus _newStatus = RegistrationRequestStatus.valueOf(newStatus); @@ -111,14 +113,14 @@ public RegistrationRequest setRequestStatus(@PathVariable String id, @PathVariab return registrationRequestService.update(request, false); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/{id}/archive", produces = MediaType.APPLICATION_JSON_VALUE) public String archiveRequest(@PathVariable String id, Authentication authentication) { registrationRequestService.archiveRequest(id, authentication); return "ARCHIVED"; } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @PostMapping(value = "/request/{id}/unarchive", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public RegistrationRequest unarchiveRequest(@PathVariable String id, @@ -135,7 +137,7 @@ public List listArchivedRequests(Authentication aut return registrationRequestService.getArchivedAllAsUser(authentication); } - @PreAuthorize("hasAuthority('ROLE_ADMIN')") + @PreAuthorize(REQUIRES_ADMIN_ROLE) @GetMapping(value = "/request/archived/all", produces = MediaType.APPLICATION_JSON_VALUE) public List listArchivedRequestsAdmin() { return registrationRequestService.getArchivedAll(); From ab99d9b7d5c214256485305799b0986625c60a34 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:15:52 +0300 Subject: [PATCH 096/132] RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. --- .../controller/DeviceManagementController.java | 5 +++-- .../service/UnknownDeviceRegistrationService.java | 11 +++++++---- .../registration/RegistrationRequestProcessor.java | 5 +++-- .../registration/service/SALRegistrationService.java | 2 ++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 2ace975..48dbc45 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -37,7 +38,7 @@ public class DeviceManagementController { private final DeviceProcessor deviceProcessor; private final DeviceManagementService deviceService; private final DeviceLifeCycleRequestService deviceLifeCycleRequestService; - private final SALRegistrationService salRegistrationService; + private final Optional salRegistrationService; private boolean isAuthenticated(Authentication authentication) { return authentication!=null && StringUtils.isNotBlank(authentication.getName()); @@ -91,7 +92,7 @@ public Device getDeviceByIpAddress(@PathVariable String ipAddress) { @PutMapping(value = "/device", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public Device createDevice(@RequestBody Device device) { - salRegistrationService.register(device); + salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.register(device)); return deviceService.save(device); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index 290fa3b..f079426 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -30,13 +30,14 @@ public class UnknownDeviceRegistrationService extends AbstractMonitorService { ); private final RegistrationRequestService registrationRequestService; private final DeviceManagementService deviceManagementService; - private final SALRegistrationService salRegistrationService; + private final Optional salRegistrationService; private final Map detectedDevices = Collections.synchronizedMap(new LinkedHashMap<>()); private final List deviceDetailsQueue = Collections.synchronizedList(new LinkedList<>()); public UnknownDeviceRegistrationService(ResourceDiscoveryProperties monitorProperties, TaskScheduler taskScheduler, ObjectMapper objectMapper, DeviceManagementService deviceManagementService, - RegistrationRequestService registrationRequestService, BrokerUtil brokerUtil, SALRegistrationService salRegistrationService) + RegistrationRequestService registrationRequestService, BrokerUtil brokerUtil, + Optional salRegistrationService) { super("UnknownDeviceRegistrationService", monitorProperties, taskScheduler, objectMapper, brokerUtil); this.registrationRequestService = registrationRequestService; @@ -247,8 +248,10 @@ private void processDeviceDetailsResponses() { newDevice = deviceManagementService.save(newDevice); log.info("UnknownDeviceRegistrationService: Registered device: {}", newDevice); - log.info("Registering the device {} to SAL...",newDevice); - salRegistrationService.register(newDevice); + if (salRegistrationService.isPresent()) { + log.info("Registering the device {} to SAL...", newDevice); + salRegistrationService.get().register(newDevice); + } } catch (Exception e) { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 90052d9..5efc3ab 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; @@ -51,7 +52,7 @@ public class RegistrationRequestProcessor implements IRegistrationRequestProcess private final ResourceDiscoveryProperties processorProperties; private final RegistrationRequestService registrationRequestService; private final DeviceManagementService deviceManagementService; - private final SALRegistrationService salRegistrationService; + private final Optional salRegistrationService; private final TaskScheduler taskScheduler; private final ObjectMapper objectMapper; private final BrokerUtil brokerUtil; @@ -371,6 +372,6 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { device.setNodeReference(registrationRequest.getNodeReference()); deviceManagementService.save(device); if (processorProperties.isSalRegistrationEnabled()) - salRegistrationService.queueForRegistration(device); + salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.queueForRegistration(device)); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 153ac91..ed9a7ee 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; @@ -21,6 +22,7 @@ @Slf4j @Service +@ConditionalOnProperty(name = ResourceDiscoveryProperties.CONFIG_PREFIX + ".sal-registration.enabled", havingValue = "true", matchIfMissing = true) @RequiredArgsConstructor public class SALRegistrationService implements InitializingBean { private final DeviceManagementService deviceManagementService; From a74a89ea3c1221e6558fceff1ccc2bdf464cb7f3 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:28:27 +0300 Subject: [PATCH 097/132] RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. --- .../RegistrationRequestProcessor.java | 12 ++--- .../RegistrationRequestController.java | 2 +- .../service/RegistrationRequestService.java | 54 ++++++++++++------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 5efc3ab..52d37f8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -124,12 +124,12 @@ private void processNewRequests() { registrationRequest.setStatus(RegistrationRequestStatus.DATA_COLLECTION_REQUESTED); log.debug("processNewRequests: Save updated request: id={}, request={}", registrationRequest.getId(), registrationRequest); - registrationRequestService.update(registrationRequest); + registrationRequestService.update(registrationRequest, null); log.debug("processNewRequests: Data collection request sent for request with Id: {}", registrationRequest.getId()); } catch (Exception e) { log.warn("processNewRequests: EXCEPTION while sending data collection request for request with Id: {}\n", registrationRequest.getId(), e); registrationRequest.setStatus(RegistrationRequestStatus.DATA_COLLECTION_ERROR); - registrationRequestService.update(registrationRequest); + registrationRequestService.update(registrationRequest, null); } } @@ -158,13 +158,13 @@ private void processOnboardingRequests() { registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_REQUESTED); log.debug("processOnboardingRequests: Save updated request: id={}, request={}", registrationRequest.getId(), registrationRequest); - registrationRequestService.update(registrationRequest, false); + registrationRequestService.update(registrationRequest, false, null); log.debug("processOnboardingRequests: Onboarding request sent for request with Id: {}", registrationRequest.getId()); } catch (Exception e) { log.warn("processOnboardingRequests: EXCEPTION while sending onboarding request for request with Id: {}\n", registrationRequest.getId(), e); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_ERROR); registrationRequest.getMessages().add("EXCEPTION "+e.getMessage()); - registrationRequestService.update(registrationRequest, false); + registrationRequestService.update(registrationRequest, false, null); } } @@ -282,7 +282,7 @@ private void processResponse(@NonNull Map response) { if (log.isDebugEnabled()) log.debug("processResponse: Save request with errors: id={}, errors={}, request={}", requestId, registrationRequest.getMessages(), registrationRequest); log.warn("processResponse: Save request with errors: id={}, errors={}", requestId, registrationRequest.getMessages()); - registrationRequestService.update(registrationRequest, false, true); + registrationRequestService.update(registrationRequest, false, true, null); return; } @@ -344,7 +344,7 @@ private void processResponse(@NonNull Map response) { // Store changes log.debug("processResponse: Save updated request: id={}, request={}", requestId, registrationRequest); - registrationRequestService.update(registrationRequest, false, true); + registrationRequestService.update(registrationRequest, false, true, null); // Archive success requests if (doArchive) { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java index d0b2835..55f084a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/controller/RegistrationRequestController.java @@ -110,7 +110,7 @@ public RegistrationRequest setRequestStatus(@PathVariable String id, @PathVariab RegistrationRequest request = registrationRequestService.getById(id) .orElseThrow(() -> new RegistrationRequestException("Not found registration request with id: " + id)); request.setStatus(_newStatus); - return registrationRequestService.update(request, false); + return registrationRequestService.update(request, false, null); } @PreAuthorize(REQUIRES_ADMIN_ROLE) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java index e775cb3..30664c9 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java @@ -54,7 +54,7 @@ public boolean isIpAddressInUse(@NonNull String ipAddress, String excludeId) { return result.stream().anyMatch(r -> !r.getId().equals(excludeId)); } - public @NonNull RegistrationRequest save(@NonNull RegistrationRequest registrationRequest) { + public @NonNull RegistrationRequest save(@NonNull RegistrationRequest registrationRequest, Authentication authentication) { RegistrationRequestStatus status = registrationRequest.getStatus(); if (status == null) { registrationRequest.setStatus(RegistrationRequestStatus.NEW_REQUEST); @@ -72,7 +72,7 @@ public boolean isIpAddressInUse(@NonNull String ipAddress, String excludeId) { throw new RegistrationRequestException( "A registration request with the same Id already exists in repository: "+registrationRequest.getId()); registrationRequest.setRequestDate(Instant.now()); - checkRegistrationRequest(registrationRequest); + checkRegistrationRequest(registrationRequest, null, authentication); // check IP address uniqueness checkIpAddressUniqueness(registrationRequest); @@ -81,20 +81,20 @@ public boolean isIpAddressInUse(@NonNull String ipAddress, String excludeId) { return registrationRequest; } - public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest) { - return update(registrationRequest, true); + public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest, Authentication authentication) { + return update(registrationRequest, true, authentication); } - public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest, boolean checkEditDel) { - return update(registrationRequest, checkEditDel, false); + public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest, boolean checkEditDel, Authentication authentication) { + return update(registrationRequest, checkEditDel, false, authentication); } - public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest, boolean checkEditDel, boolean skipUniqueIpAddressCheck) { + public RegistrationRequest update(@NonNull RegistrationRequest registrationRequest, boolean checkEditDel, boolean skipUniqueIpAddressCheck, Authentication authentication) { Optional result = getById(registrationRequest.getId()); if (result.isEmpty()) throw new RegistrationRequestException( "Registration request with the Id does not exists in repository: "+registrationRequest.getId()); - checkRegistrationRequest(registrationRequest); + checkRegistrationRequest(registrationRequest, result.get(), authentication); if (checkEditDel) canEditOrDelete(result.get()); @@ -137,7 +137,7 @@ private boolean isWhiteSpaceChar(char c) { return c==' ' || c=='\t' || c=='\r' || c=='\n'; } - private void checkRegistrationRequest(@NonNull RegistrationRequest registrationRequest) { + private void checkRegistrationRequest(@NonNull RegistrationRequest registrationRequest, RegistrationRequest storedRequest, Authentication authentication) { List errors = new ArrayList<>(); if (StringUtils.isBlank(registrationRequest.getId())) errors.add("Null or blank Id"); if (registrationRequest.getDevice()==null) errors.add("No Device specified"); @@ -149,11 +149,26 @@ private void checkRegistrationRequest(@NonNull RegistrationRequest registrationR String.format("Registration request has errors: %s\n%s", String.join(", ", errors), registrationRequest)); } - checkDevice(registrationRequest.getDevice()); + + Device storedDevice = storedRequest != null ? storedRequest.getDevice() : null; + checkDevice(registrationRequest.getDevice(), storedDevice, authentication, errors); + if (!errors.isEmpty()) { + throw new RegistrationRequestException( + String.format("Device data in Registration request has errors: %s\n%s", + String.join(", ", errors), registrationRequest)); + } } - private void checkDevice(@NonNull Device device) { - //XXX:TODO + private void checkDevice(@NonNull Device device, Device storedRequestDevice, Authentication authentication, List errors) { + //if (StringUtils.isBlank(device.getId())) errors.add("Null or blank device id"); + if (StringUtils.isBlank(device.getOwner())) errors.add("Null or blank device owner"); + if (authentication!=null && StringUtils.isNotBlank(authentication.getName())) + if (! isAdmin(authentication) && ! device.getOwner().equals(authentication.getName())) + errors.add("Device owner differs from authenticated user"); + if (storedRequestDevice!=null) { + if (! storedRequestDevice.getId().equals(device.getId())) errors.add("Device Id is different than stored"); + if (! storedRequestDevice.getOwner().equals(device.getOwner())) errors.add("Device owner is different than stored"); + } } private void checkIpAddressUniqueness(RegistrationRequest registrationRequest) { @@ -198,10 +213,13 @@ public Optional getByIdAsUser(@NonNull String id, Authentic return result; } - private void checkAdmin(@NonNull String requestId, Authentication authentication) { - boolean isAdmin = authentication.getAuthorities().stream() + private boolean isAdmin(Authentication authentication) { + return authentication.getAuthorities().stream() .map(GrantedAuthority::getAuthority).toList().contains("ROLE_ADMIN"); - if (! isAdmin) + } + + private void checkAdmin(@NonNull String requestId, Authentication authentication) { + if (! isAdmin(authentication)) throw new RegistrationRequestException( new IllegalAccessException("Operation requires ADMIN role. Cannot access request with Id: "+requestId)); } @@ -233,7 +251,7 @@ public List getAllAsUser(Authentication authentication) { public @NonNull RegistrationRequest saveAsUser(@NonNull RegistrationRequest registrationRequest, Authentication authentication) { registrationRequest.setRequester(authentication.getName()); checkRequester(registrationRequest, authentication); - return save(registrationRequest); + return save(registrationRequest, authentication); } public RegistrationRequest updateAsUser(@NonNull RegistrationRequest registrationRequest, Authentication authentication) { @@ -242,7 +260,7 @@ public RegistrationRequest updateAsUser(@NonNull RegistrationRequest registratio throw new RegistrationRequestException( "Registration request with the Id does not exists in repository: "+registrationRequest.getId()); checkRequester(result.get(), authentication); - return update(registrationRequest); + return update(registrationRequest, authentication); } public void deleteByIdAsUser(@NonNull String id, Authentication authentication) { @@ -271,7 +289,7 @@ public RegistrationRequest authorizeRequest(String id, boolean authorize, Authen result.get().setStatus( authorize ? RegistrationRequestStatus.PENDING_ONBOARDING : RegistrationRequestStatus.AUTHORIZATION_REJECT ); - return update(result.get()); + return update(result.get(), authentication); } // ------------------------------------------------------------------------ From e52d8e8f50a5457139b47f6a1334afafe016168e Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:30:28 +0300 Subject: [PATCH 098/132] RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). --- .../resource/discovery/monitor/model/Device.java | 1 + .../discovery/registration/model/Device.java | 1 + .../service/RegistrationRequestService.java | 12 ++++++++++++ 3 files changed, 14 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index 505c12b..5ad29d3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -17,6 +17,7 @@ @Document(collection = "device") public class Device { private String id; + private String ref; private String os; private String name; private String owner; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java index 34335b9..ae18e57 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java @@ -14,6 +14,7 @@ @NoArgsConstructor public class Device { private String id; + private String ref; private String os; private String name; private String owner; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java index 30664c9..8825a14 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java @@ -68,6 +68,16 @@ public boolean isIpAddressInUse(@NonNull String ipAddress, String excludeId) { throw new RegistrationRequestException( "New registration request already has an Id: " + registrationRequest.getId()); } + if (registrationRequest.getDevice()!=null && StringUtils.isBlank(registrationRequest.getDevice().getId())) { + String devId; + if (StringUtils.isNotBlank(registrationRequest.getDevice().getRef())) { + int p = registrationRequest.getDevice().getRef().lastIndexOf("|"); + devId = p < 0 ? registrationRequest.getDevice().getRef() : registrationRequest.getDevice().getRef().substring(p + 1); + } else { + devId = UUID.randomUUID().toString(); + } + registrationRequest.getDevice().setId(devId); + } if (getById(registrationRequest.getId()).isPresent()) throw new RegistrationRequestException( "A registration request with the same Id already exists in repository: "+registrationRequest.getId()); @@ -103,9 +113,11 @@ public RegistrationRequest update(@NonNull RegistrationRequest registrationReque checkIpAddressUniqueness(registrationRequest); // Copy submitted registration request data onto the retrieved request + String devOwner = result.get().getDevice().getOwner(); BeanUtils.copyProperties(registrationRequest, result.get(), "id", "device", "requester", "requestDate"); result.get().setLastUpdateDate(Instant.now()); + result.get().getDevice().setOwner(devOwner); // Check if device password/public key need update... List ignoreList = new ArrayList<>(); From 9356f4005a7a37f27b4552bfd5fa427113422fb5 Mon Sep 17 00:00:00 2001 From: ipatini Date: Tue, 8 Oct 2024 09:31:39 +0300 Subject: [PATCH 099/132] RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. --- .../archived-device-view.html | 10 +++++- .../archived-request-view.html | 10 +++++- .../static/freebees_webdesign_6/archived.html | 27 ++++++++++++---- .../freebees_webdesign_6/device-view.html | 10 +++++- .../static/freebees_webdesign_6/devices.html | 6 +++- .../freebees_webdesign_6/request-edit.html | 32 +++++++++++++++++-- .../static/freebees_webdesign_6/requests.html | 2 ++ 7 files changed, 84 insertions(+), 13 deletions(-) diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html index b21c45b..5e2d1ee 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html @@ -49,7 +49,8 @@ // console.log('refreshDeviceInfo: OK: ', data); var devId = data.id; if (devId!=='') - $('#page_title').html( 'Device '+devId ); + $('#page_title').html( 'Device '+devId + + '
' + data.ref + ''); else $('#page_title').html( $(`
Error: ${status}: ${error}
`) ); @@ -259,6 +260,13 @@
Device details
+ +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html index ea9e61d..8a602ab 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html @@ -49,7 +49,8 @@ // console.log('refreshRequestInfo: OK: ', data); var reqId = data.id; if (reqId!=='') - $('#page_title').html( 'Archived Registration Request '+reqId ); + $('#page_title').html( 'Archived Registration Request '+reqId + + '
' + data.device.ref + '' ); else $('#page_title').html( $(`
Error: ${status}: ${error}
`) ); @@ -272,6 +273,13 @@
Device details
+ +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html index a97df0b..91c6c4c 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html @@ -46,6 +46,7 @@ data.forEach(item => { var reqId = item.id; var requester = item.requester; + var devRef = item.device.ref; var devName = item.device.name; var ipAddress = item.device.ipAddress; var date = new Date( Date.parse( item.requestDate ) ); @@ -54,7 +55,7 @@ var status = item.status; var color = getRequestStatusColor(status); var adminActions = (isAdmin) ? ` - `: ''; @@ -65,6 +66,7 @@ ${requester} ${devName} +
${devRef} ${ipAddress} ${dateStr} @@ -94,9 +96,13 @@ return 'table-info'; } -function unarchiveRequest(reqId, ipAddress) { +function unarchiveRequest(reqId, ipAddress, devRef) { if (! confirm('Restore request?')) return; - showModal('Request: '+ipAddress+' ['+reqId+']', 'REQUEST', reqId); + //showModal('Request: '+ipAddress+' ['+reqId+']', 'REQUEST', reqId, devRef); + showModal(`Request Id: ${reqId}
+ Request Ref: ${devRef}
+ IP address: ${ipAddress}
+ `, 'REQUEST', reqId, devRef); } function _unarchiveRequest(reqId, credentials) { @@ -120,11 +126,12 @@ // ---------------------------------------------------------------------------- -function showModal(info, type, id) { +function showModal(info, type, id, ref) { // Set credentials modal data and clear form $('#credentialsModal_info').html(info); $(`[id="device#type"]`).val(type); $(`[id="device#id"]`).val(id); + $(`[id="device#ref"]`).val(ref); $(`[id="device#username"]`).val(''); $(`[id="device#password"]`).val(''); @@ -182,6 +189,7 @@ var ii = 0; data.forEach(item => { var devId = item.id; + var devRef = item.ref; var owner = item.owner; var devName = item.name; var ipAddress = item.ipAddress; @@ -191,7 +199,7 @@ var status = item.status; var color = getDeviceStatusColor(status); var adminActions = (isAdmin) ? ` - `: ''; @@ -202,6 +210,7 @@ ${owner} ${devName} +
${devRef} ${ipAddress} ${dateStr} @@ -234,9 +243,13 @@ return 'table-info'; } -function unarchiveDevice(devId, ipAddress) { +function unarchiveDevice(devId, ipAddress, devRef) { if (! confirm('Restore device?')) return; - showModal('Device: '+ipAddress+' ['+devId+']', 'DEVICE', devId); + //showModal('Device: '+ipAddress+' ['+devId+']', 'DEVICE', devId, devRef); + showModal(`Device Id: ${devId}
+ Device Ref: ${devRef}
+ IP Address: ${ipAddress}
+ `, 'DEVICE', devId, devRef); } function _unarchiveDevice(devId, credentials) { diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html index c39924e..498cbd2 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -74,7 +74,8 @@ // console.log('refreshDeviceInfo: OK: ', data); var devId = data.id; if (devId!=='') - $('#page_title').html( 'Device '+devId ); + $('#page_title').html( 'Device '+devId + + '
' + data.ref + '' ); else $('#page_title').html( $(`
Error: ${status}: ${error}
`) ); @@ -376,6 +377,13 @@
Device details
+ +
+ +
+ +
+
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html index 363dffe..d70e926 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html @@ -52,6 +52,7 @@ var ii = 0; data.forEach(item => { var devId = item.id; + var devRef = item.ref; var owner = item.owner; var devName = (item.name && item.name.trim()!=='') ? item.name.trim() : `(No name - Id ${devId})`; var ipAddress = item.ipAddress; @@ -82,7 +83,10 @@ ${ii} ${owner} - ${devName} + + ${devName} +
${devRef} + ${ipAddress} ${load} ${status} diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html index 20266ce..9cc7f2f 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -22,6 +22,7 @@ $(function() { const urlParams = new URLSearchParams(window.location.search); const reqId = urlParams.get('id') ?? ''; + const appId = urlParams.get('appId') ?? 'all-applications'; const readonly = urlParams.get('readonly') ?? ''; requestId = reqId; @@ -32,9 +33,23 @@ refreshRequestInfo(reqId); else isNew = true; + + if (appId!=='' || ! isNew) { + var devId = gen_uuid(); + var devRef = 'application_id|' + appId + '|' + devId; + var it = $(`[id="request#device#ref"]`); + it.val( devRef ); + } checkSameUser(); }); +function gen_uuid() { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8); + return v.toString(16); + }); +} + var isNew = false; var isAdmin = false; var isReadonly = false; @@ -77,6 +92,7 @@ var reqId = data.id; $('#page_title').html( reqId=='' ? 'New Registration Request' : 'Registration Request '+reqId + + '
' + data.device.ref + '' ); requestId = data.id; @@ -328,6 +344,11 @@ data.admin ? $('#whoami').html( $(`${data.user}`) ) : $('#whoami').html( data.user ); if (isAdmin) $('.adminOnly').toggleClass('d-none'); + if (isNew) { + var it = $(`[id="request#device#owner"]`); + it.val( username ); + } + checkSameUser(); }) .fail(function(xhr, status, error) { $('#whoami').html( $(`Error: ${status} ${JSON.stringify(error)}`) ); }); @@ -427,7 +448,14 @@
Device details
- + +
+
+ +
+ +
+
@@ -448,7 +476,7 @@
Device details
- +
diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html index f8203cb..95913d1 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html @@ -42,6 +42,7 @@ data.forEach(item => { var reqId = item.id; var requester = item.requester; + var devRef = item.device.ref; var devName = item.device.name; var ipAddress = item.device.ipAddress; var date = new Date( Date.parse( item.requestDate ) ); @@ -91,6 +92,7 @@ ${requester} ${devName} @ ${ipAddress} +
${devRef} ${ipAddress} ${dateStr} From 05c022ce81640a4244fc37dc1792daeca0d457e5 Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 8 Oct 2024 14:05:28 +0300 Subject: [PATCH 100/132] Use device reference instead of name to register to SAL --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index f411204..424c657 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -58,7 +58,7 @@ public void register(Device device) { echo OS_KERNEL=$TMP_KERNEL echo OS_KERNEL_RELEASE=$TMP_KERNEL_RELEASE */ - String device_name = device.getName(); + String device_name = device.getRef(); int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); int ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; From f39977b7b7468cef51ef0fed182e719e33ddf35f Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 15 Oct 2024 11:47:20 +0300 Subject: [PATCH 101/132] Improvements in device registration and component communication with the broker --- .../ExtendedConnector.java | 1 + .../broker_communication/SALCommunicator.java | 2 +- .../service/SALRegistrationService.java | 21 ++++++++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java index 1e09f3e..5773402 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java @@ -74,6 +74,7 @@ public void stop(ArrayList consumers, ArrayList publishers if (publishers.size()>0) { stop_publishers(publishers); } + this.stop(); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 494bc1c..47058b6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -142,7 +142,7 @@ private static void register_devices(String request_body_file, String sessionID, } - public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, int ram_gb, int disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, long ram_gb, long disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { JSONObject root_json_object = new JSONObject(); JSONObject loginCredential = new JSONObject(); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 424c657..8c499a8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -39,7 +39,17 @@ public void queueForRegistration(@NonNull Device device) { public void register(Device device) { - String application_name = ""; //TODO decide on this + + String application_name = device.getRef().split("\\|")[1]; + if (application_name.equals("all_applications")){ + application_name=""; + } + String public_ip = System.getenv("NEBULOUS_IP"); + if (public_ip.equals("")){ + log.warn("Using default IP address as the environmental variable was not set or found"); + public_ip = "158.39.201.36"; + } + Map device_info = device.getDeviceInfo(); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS @@ -61,8 +71,8 @@ public void register(Device device) { String device_name = device.getRef(); int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); - int ram_gb = Integer.parseInt(device_info.get("RAM_TOTAL_KB"))/1000000; - int disk_gb = Integer.parseInt(device_info.get("DISK_TOTAL_KB"))/1000000; + long ram_gb = Math.round(Integer.parseInt(device_info.get("RAM_TOTAL_KB"))*1.0/1000000); + long disk_gb = Math.round(Integer.parseInt(device_info.get("DISK_TOTAL_KB"))*1.0/1000000); String external_ip_address = device.getIpAddress(); String device_username = device.getUsername(); String device_password = new String(device.getPassword()); @@ -76,8 +86,9 @@ public void register(Device device) { String internal_ip = ""; //TODO improve this String os_family = "UBUNTU"; //TODO improve this //String os_architecture = "ARMv8"; //TODO improve this - String os_architecture = device_info.get("OS_ARCHITECTURE").equals("aarch64") ? "armv8" : device_info.get("OS_ARCHITECTURE").equals("armv8") ?"armv8": device_info.get("OS_ARCHITECTURE").equals("armv7l") ? "armv7l": "AMD"; - String jar_url = os_architecture.equals("armv8") ? "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_arm_v8.jar" : os_architecture.equals("armv7l") ? "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_arm_v7l.jar" : "https://www.activeeon.com/public_content/nebulous/node_14.1.0-SNAPSHOT_amd.jar"; + String os_architecture = device_info.get("OS_ARCHITECTURE").equals("aarch64") ? "ARMv8" : device_info.get("OS_ARCHITECTURE").equals("armv8") ?"ARMv8": device_info.get("OS_ARCHITECTURE").equals("armv7l") ? "ARMv7": "AMD"; + String jar_url = os_architecture.equals("ARMv8") ? "http://"+public_ip+":8880/rest/node-arm-v8.jar" : os_architecture.equals("ARMv7") ? "http://"+public_ip+":8880/rest/node-arm-v7.jar" : "http://"+public_ip+":8880/rest/node-amd-64.jar"; + int os_version = 2204; //TODO improve this String private_key = ""; //TODO improve this int external_access_port = device.getPort(); From 5f02a300dd37ff0e8478c09b47c369dcf8b038c6 Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 15 Oct 2024 12:57:54 +0300 Subject: [PATCH 102/132] Small improvements in device registration --- .../resource/discovery/ResourceDiscoveryProperties.java | 3 +++ .../registration/service/SALRegistrationService.java | 8 ++++---- resource-discovery/src/main/resources/application.yml | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 9711036..801bdb1 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -101,6 +101,9 @@ public class ResourceDiscoveryProperties { private String nebulous_broker_username; private String nebulous_broker_password; private String lost_device_topic; + + // Nebulous server data + private String nebulous_server_ip_address; @Data public static class UserData { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 8c499a8..ff4ca13 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -47,7 +47,7 @@ public void register(Device device) { String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip.equals("")){ log.warn("Using default IP address as the environmental variable was not set or found"); - public_ip = "158.39.201.36"; + public_ip = processorProperties.getNebulous_server_ip_address(); } Map device_info = device.getDeviceInfo(); @@ -71,8 +71,8 @@ public void register(Device device) { String device_name = device.getRef(); int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); - long ram_gb = Math.round(Integer.parseInt(device_info.get("RAM_TOTAL_KB"))*1.0/1000000); - long disk_gb = Math.round(Integer.parseInt(device_info.get("DISK_TOTAL_KB"))*1.0/1000000); + long ram_mb = Math.round(Integer.parseInt(device_info.get("RAM_TOTAL_KB"))*1.0/1000); + long disk_mb = Math.round(Integer.parseInt(device_info.get("DISK_TOTAL_KB"))*1.0/1000); String external_ip_address = device.getIpAddress(); String device_username = device.getUsername(); String device_password = new String(device.getPassword()); @@ -115,7 +115,7 @@ public void register(Device device) { //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,jar_url,os_version,cores,ram_gb,disk_gb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,jar_url,os_version,cores,ram_mb,disk_mb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); log.info("topic is {}", get_registration_topic_name(application_name)); log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); log.info("broker port is {}", processorProperties.getNebulous_broker_port()); diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 16f1359..404e88e 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -27,6 +27,7 @@ discovery: sal_host: "localhost" sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" + nebulous_server_ip_address: "158.39.201.36" # trustStoreFile: tests/config/broker-truststore.p12 # trustStorePassword: melodic # trustStoreType: PKCS12 From 9ac2a2bc39596c1289c78f95c5136fa9bc54da6c Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 15 Oct 2024 14:08:55 +0300 Subject: [PATCH 103/132] Small improvement in getting device registration details --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index ff4ca13..b8688ce 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -45,7 +45,7 @@ public void register(Device device) { application_name=""; } String public_ip = System.getenv("NEBULOUS_IP"); - if (public_ip.equals("")){ + if (public_ip!=null && public_ip.isEmpty()){ log.warn("Using default IP address as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); } From dd55eda0dcf66bdf10dc868cb4122143ccf9ca07 Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 15 Oct 2024 14:41:02 +0300 Subject: [PATCH 104/132] Small improvement in getting device registration details --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index b8688ce..6fc32b5 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -45,7 +45,7 @@ public void register(Device device) { application_name=""; } String public_ip = System.getenv("NEBULOUS_IP"); - if (public_ip!=null && public_ip.isEmpty()){ + if (public_ip==null || public_ip.isEmpty()){ log.warn("Using default IP address as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); } From bc28d48de1f8ea2e0809049d87d2455366f98fd4 Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 17 Oct 2024 11:01:40 +0300 Subject: [PATCH 105/132] RD: Fix in SALRegistrationService class --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index b8688ce..6fc32b5 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -45,7 +45,7 @@ public void register(Device device) { application_name=""; } String public_ip = System.getenv("NEBULOUS_IP"); - if (public_ip!=null && public_ip.isEmpty()){ + if (public_ip==null || public_ip.isEmpty()){ log.warn("Using default IP address as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); } From 9346f010bc043b1b550fb9573c2fb24f952a107f Mon Sep 17 00:00:00 2001 From: ipatini Date: Thu, 17 Oct 2024 12:56:03 +0300 Subject: [PATCH 106/132] RD: Fixed fontawesome cdn url --- .../static/freebees_webdesign_6/archived-device-view.html | 2 +- .../static/freebees_webdesign_6/archived-request-view.html | 2 +- .../main/resources/static/freebees_webdesign_6/archived.html | 2 +- .../main/resources/static/freebees_webdesign_6/device-view.html | 2 +- .../src/main/resources/static/freebees_webdesign_6/devices.html | 2 +- .../src/main/resources/static/freebees_webdesign_6/index.html | 2 +- .../resources/static/freebees_webdesign_6/request-edit.html | 2 +- .../main/resources/static/freebees_webdesign_6/requests.html | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html index 5e2d1ee..282dd76 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-device-view.html @@ -10,10 +10,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html index 8a602ab..9f81a4d 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived-request-view.html @@ -10,10 +10,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html index 91c6c4c..573f93a 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/archived.html @@ -9,10 +9,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html index 498cbd2..34f36ae 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/device-view.html @@ -10,10 +10,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html index d70e926..9cc7ea5 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/devices.html @@ -9,10 +9,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html index 8f90e19..7078b78 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/index.html @@ -9,10 +9,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html index 9cc7f2f..819f661 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -10,10 +10,10 @@ + - diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html index 95913d1..f4ba810 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/requests.html @@ -9,10 +9,10 @@ + - From 919e11dffe4cbf23b6006a1f7ed362a95e0359ec Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 21 Oct 2024 14:34:24 +0300 Subject: [PATCH 107/132] Initial deregistration support --- .../ResourceDiscoveryProperties.java | 1 + .../BrokerSubscriber.java | 27 ++- .../broker_communication/SALCommunicator.java | 4 + .../discovery/monitor/DeviceProcessor.java | 6 + .../service/SALDeregistrationService.java | 205 ++++++++++++++++++ .../service/SALRegistrationService.java | 2 +- .../src/main/resources/application.yml | 1 + 7 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 801bdb1..fc9bbad 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -101,6 +101,7 @@ public class ResourceDiscoveryProperties { private String nebulous_broker_username; private String nebulous_broker_password; private String lost_device_topic; + private String compromised_device_topic; // Nebulous server data private String nebulous_server_ip_address; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java index 7ff11d5..844fc0b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java @@ -17,6 +17,8 @@ @Slf4j public class BrokerSubscriber { + private AtomicBoolean stop_signal = new AtomicBoolean(); + private class MessageProcessingHandler extends Handler { private BrokerSubscriptionDetails broker_details; private static final BiFunction temporary_function = (Object o, Object o2) -> { @@ -54,6 +56,7 @@ public void setProcessing_function(BiFunction processing_function) { private static HashMap> broker_and_topics_to_subscribe_to = new HashMap<>(); private static HashMap> active_consumers_per_topic_per_broker_ip = new HashMap<>(); private static HashMap current_connectors = new HashMap<>(); + ArrayList consumers = new ArrayList<>(); private String topic; private String broker_ip; private int broker_port; @@ -62,6 +65,7 @@ public void setProcessing_function(BiFunction processing_function) { BrokerSubscriptionDetails broker_details; public BrokerSubscriber(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation, String application_name) { + stop_signal.set(false); boolean able_to_initialize_BrokerSubscriber = topic != null && broker_ip != null && brokerUsername != null && brokerPassword != null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerSubscriber) { @@ -120,10 +124,20 @@ public BrokerSubscriber(String topic, String broker_ip, int broker_port, String */ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { if (current_connectors.get(broker_ip) != null) { - current_connectors.get(broker_ip).add_consumer(new_consumer); - } else { - ArrayList consumers = new ArrayList<>(); + current_connectors.get(broker_ip).stop(consumers,new ArrayList<>()); + } + if (consumers.isEmpty()){ + consumers = new ArrayList<>(); + } + boolean do_not_add_new_consumer = false; + for (Consumer consumer : consumers) { + if (Objects.equals(consumer.linkAddress, new_consumer.linkAddress)) { + do_not_add_new_consumer = true; + } + } + if(!do_not_add_new_consumer) { consumers.add(new_consumer); + } ExtendedConnector extended_connector = new ExtendedConnector("resource_manager", new CustomConnectorHandler() { }, @@ -142,7 +156,6 @@ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { ); extended_connector.start(); current_connectors.put(broker_ip, extended_connector); - } } private void remove_topic_from_broker_connector(String topic_key) { @@ -150,7 +163,11 @@ private void remove_topic_from_broker_connector(String topic_key) { current_connectors.get(broker_ip).remove_consumer_with_key(topic_key); } } - + + public int subscribe (BiFunction function, String application_name) { + return subscribe(function,application_name,stop_signal); + } + public int subscribe(BiFunction function, String application_name, AtomicBoolean stop_signal) { int exit_status = -1; log.info("ESTABLISHING SUBSCRIPTION for " + topic); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 47058b6..0303b3a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.broker_communication; +import eu.nebulous.resource.discovery.monitor.model.Device; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; @@ -141,6 +142,9 @@ private static void register_devices(String request_body_file, String sessionID, } } + public static String get_device_deregistration_json(Device device){ + return ""; + } public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, long ram_gb, long disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 4b8d893..d4668b6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -7,6 +7,7 @@ import eu.nebulous.resource.discovery.monitor.model.Device; import eu.nebulous.resource.discovery.monitor.model.DeviceStatus; import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; +import eu.nebulous.resource.discovery.registration.service.SALDeregistrationService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; @@ -22,6 +23,7 @@ import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,6 +48,8 @@ public class DeviceProcessor implements InitializingBean { private final DeviceManagementService deviceManagementService; private final TaskScheduler taskScheduler; private final AtomicBoolean isRunning = new AtomicBoolean(false); + private final Optional salDeregistrationService; + @Override public void afterPropertiesSet() throws Exception { @@ -134,6 +138,8 @@ private void processFailedDevices() { && device.getCreationDate().isBefore(failedDeviceThreshold) ) { device.setStatus(DeviceStatus.FAILED); + log.info("processFailedDevices: Deregistering device with Id: {}", device.getId()); + salDeregistrationService.ifPresent(deregistrationService -> deregistrationService.deregister(device)); JSONObject lost_device_message = new JSONObject(); lost_device_message.put("device_name",device.getName()); Clock clock = Clock.systemUTC(); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java new file mode 100644 index 0000000..2354c66 --- /dev/null +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -0,0 +1,205 @@ +package eu.nebulous.resource.discovery.registration.service; + + + +import eu.nebulous.resource.discovery.ResourceDiscoveryProperties; +import eu.nebulous.resource.discovery.broker_communication.BrokerSubscriber; +import eu.nebulous.resource.discovery.broker_communication.BrokerSubscriptionDetails; +import eu.nebulous.resource.discovery.broker_communication.SynchronousBrokerPublisher; +import eu.nebulous.resource.discovery.monitor.model.Device; +import eu.nebulous.resource.discovery.monitor.service.DeviceManagementService; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.task.TaskExecutor; +import org.springframework.stereotype.Service; + +import java.time.Clock; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.function.BiFunction; + +import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.get_device_deregistration_json; +import static eu.nebulous.resource.discovery.broker_communication.SALCommunicator.get_device_registration_json; + +@Slf4j +@Service +@ConditionalOnProperty(name = ResourceDiscoveryProperties.CONFIG_PREFIX + ".sal-registration.enabled", havingValue = "true", matchIfMissing = true) +@RequiredArgsConstructor +public class SALDeregistrationService implements InitializingBean { + + private final DeviceManagementService deviceManagementService; + private final ResourceDiscoveryProperties processorProperties; + private final TaskExecutor taskExecutor; + private final LinkedBlockingDeque queue = new LinkedBlockingDeque<>(); + private Thread processQueueThread; + private long lastDeregistrationStartTimestamp = -1L; + + public void queueForRegistration(@NonNull Device device) { + if (processorProperties.isSalRegistrationEnabled()) //If registration is enabled, so should be deregistration as well + queue.add(device); + } + + public void deregister(Device device) { + + + String application_name = device.getRef().split("\\|")[1]; + if (application_name.equals("all_applications")) { + application_name = ""; + } + + + + String deregister_device_message_string = get_device_deregistration_json(device); + + SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + int sending_attempt = 1; + while (deregister_device_publisher.is_publisher_null()) { + if (sending_attempt <= 2) { + deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + } else { + log.warn("Will now attempt to reset the Synchronous publisher connector to deregister"); + deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), "",true); + } + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + sending_attempt++; + } + //TODO handle the response here + Map response = deregister_device_publisher.publish_for_response(deregister_device_message_string, Collections.singleton(application_name)); + log.info("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); + //} + + /* This is some realtime information, could be retrieved with a different call to the EMS. + CurrDateTime: 1709207141 + UpDateTime: 1709186638 + Uptime: 20503 + CPU: 0 + RAM: 31.4725 + DISK: 10.3586 + RX: 0 + TX: 0 + */ + + } + + private String get_deregistration_topic_name(String application_name) { + return processorProperties.getRegistration_topic_name(); + //return ("eu.nebulouscloud.exn.sal.edge." + application_name); + } + + @Override + public void afterPropertiesSet() { + if (!processorProperties.isSalRegistrationEnabled()) { + log.info("SAL (de)registration is disabled due to configuration"); + return; + } + + if (StringUtils.isNotBlank(processorProperties.getNebulous_broker_ip_address()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_username()) && + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password())) { + log.info("Successful setting of properties for communication with SAL"); + taskExecutor.execute(this::processQueue); + taskExecutor.execute(this::checkProcessQueue); + } else { + String message = String.format("Nebulous broker configuration is missing: ip-address=%s, username=%s, password=%s", + processorProperties.getNebulous_broker_ip_address(), + processorProperties.getNebulous_broker_username(), + StringUtils.isNotBlank(processorProperties.getNebulous_broker_password()) ? "" : ""); + log.error(message); + throw new RuntimeException(message); + } + } + + public void subscribeCompromisedDevices() { + + BiFunction compromised_device_handling = (broker_subscription_details, message) -> { + + JSONParser parser = new JSONParser(); + JSONObject jsonObject = null; + try { + jsonObject = (JSONObject) parser.parse(message); + } catch (ParseException e) { + throw new RuntimeException(e); + } + + // Assuming the device name is inside a "device" object + String device_id = (String) jsonObject.get("device_id"); + + Optional compromisedDevice = deviceManagementService.getById(device_id); + if (compromisedDevice.isPresent()){ + queue.add(compromisedDevice.get()); + return compromisedDevice.get(); + }else{ + return null; + } + }; + + String topicName = get_device_compromised_topic_name(); + log.info("Subscribing to topic: {}", topicName); + try { + BrokerSubscriber subscriber = new BrokerSubscriber(topicName,processorProperties.getNebulous_broker_ip_address(),processorProperties.getNebulous_broker_port(),processorProperties.getBrokerUsername(),processorProperties.getBrokerPassword(),"",""); + subscriber.subscribe(compromised_device_handling,""); + } catch (Exception e) { + log.error("Error while subscribing to topic: " + topicName, e); + } + } + + + public void processQueue() { + processQueueThread = Thread.currentThread(); + while (true) { + Device device = null; + try { + device = queue.take(); + log.warn("SALRegistrationService: processQueue(): Will deregister device: {}", device); + lastDeregistrationStartTimestamp = System.currentTimeMillis(); + deregister(device); + lastDeregistrationStartTimestamp = -1L; + device.setRegisteredToSAL(false); + deviceManagementService.update(device); + log.warn("SALRegistrationService: processQueue(): Device deregistered from SAL: {}", device); + } catch (InterruptedException e) { + log.warn("SALRegistrationService: processQueue(): Interrupted. Will not deregister device from SAL: {}", device); + lastDeregistrationStartTimestamp = -1L; +// break; + } catch (Exception e) { + log.warn("SALRegistrationService: processQueue(): EXCEPTION caught. Will not deregister device from SAL: {}", device, e); + lastDeregistrationStartTimestamp = -1L; + } + } + } + + public void checkProcessQueue() { + while (true) { + try { + Thread.sleep(1000); + if (processQueueThread != null && lastDeregistrationStartTimestamp > 0) { + long runningTime = System.currentTimeMillis() - lastDeregistrationStartTimestamp; + if (runningTime > processorProperties.getSalRegistrationTimeout()) { + log.warn("SALDeregistrationService: checkProcessQueue(): Method 'processQueue' is running for too log. Will attempt to interrupt it"); + processQueueThread.interrupt(); + lastDeregistrationStartTimestamp = -1L; + } + } + } catch (Exception e) { + log.warn("SALDeregistrationService: checkProcessQueue(): EXCEPTION caught: ", e); + } + } + } + + private String get_device_compromised_topic_name(){ + return processorProperties.getCompromised_device_topic(); + } +} \ No newline at end of file diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 6fc32b5..1cd659a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -46,7 +46,7 @@ public void register(Device device) { } String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip==null || public_ip.isEmpty()){ - log.warn("Using default IP address as the environmental variable was not set or found"); + log.warn("Using default IP address ("+processorProperties.getNebulous_server_ip_address()+") to fetch Proactive client jar files from, as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); } diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 404e88e..be6c227 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -27,6 +27,7 @@ discovery: sal_host: "localhost" sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" + compromised_device_topic: "eu.nebulouscloud.monitoring.device_compromised" nebulous_server_ip_address: "158.39.201.36" # trustStoreFile: tests/config/broker-truststore.p12 # trustStorePassword: melodic From ebbbcb264bb256dc5b533add01c0bbe4ea769ac4 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 21 Oct 2024 18:39:04 +0300 Subject: [PATCH 108/132] RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) --- .../discovery/ResourceDiscoveryProperties.java | 9 +++++++++ .../registration/RegistrationRequestProcessor.java | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 801bdb1..c5ab492 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -53,6 +53,9 @@ public class ResourceDiscoveryProperties { private String dataCollectionResponseTopic = "ems.client.installation.reports"; private List allowedDeviceInfoKeys = new ArrayList<>(List.of("*")); + // Device authorization settings + private AUTHORIZATION_TYPE authorizationType = AUTHORIZATION_TYPE.ALWAYS_AUTHORIZE; + // Device monitoring settings private String deviceStatusMonitorTopic = "_ui_instance_info"; //XXX:TODO: Change Topic name. Also update EMS config. private String deviceMetricsMonitorTopic = "_client_metrics"; //XXX:TODO: Change Topic name. Also update EMS config. @@ -105,6 +108,12 @@ public class ResourceDiscoveryProperties { // Nebulous server data private String nebulous_server_ip_address; + public enum AUTHORIZATION_TYPE { + NONE, MANUAL, + ALWAYS_AUTHORIZE, ALWAYS_REJECT + //, CALL_REST_API, ASK_ABAC_SERVER + } + @Data public static class UserData { private final String username; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 52d37f8..faec07d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -320,7 +320,7 @@ private void processResponse(@NonNull Map response) { // Set new status if (currStatus==RegistrationRequestStatus.DATA_COLLECTION_REQUESTED) - registrationRequest.setStatus(RegistrationRequestStatus.PENDING_AUTHORIZATION); + registrationRequest.setStatus( getNextStatus(currStatus) ); if (currStatus==RegistrationRequestStatus.ONBOARDING_REQUESTED) { registrationRequest.setStatus(RegistrationRequestStatus.SUCCESS); doArchive = processorProperties.isImmediatelyArchiveSuccessRequests(); @@ -355,6 +355,14 @@ private void processResponse(@NonNull Map response) { } } + private RegistrationRequestStatus getNextStatus(RegistrationRequestStatus currStatus) { + return switch (processorProperties.getAuthorizationType()) { + case MANUAL -> RegistrationRequestStatus.PENDING_AUTHORIZATION; + case NONE, ALWAYS_AUTHORIZE -> RegistrationRequestStatus.PENDING_ONBOARDING; + case ALWAYS_REJECT -> RegistrationRequestStatus.AUTHORIZATION_REJECT; + }; + } + private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { Device device = objectMapper.convertValue(registrationRequest.getDevice(), Device.class); // override values From ebc2a2700427fdc9670e2472a330d339ea299402 Mon Sep 17 00:00:00 2001 From: ipatini Date: Mon, 21 Oct 2024 20:45:16 +0300 Subject: [PATCH 109/132] RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS --- .../RegistrationRequestProcessor.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index faec07d..df5e583 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -153,8 +153,8 @@ private void processOnboardingRequests() { deviceManagementService.checkDevice(deviceForMonitoring, true); log.debug("processOnboardingRequests: Requesting device onboarding for request with Id: {}", registrationRequest.getId()); - Map dataCollectionRequest = prepareRequestPayload(REQUEST_TYPE.INSTALL, registrationRequest); - brokerUtil.sendMessage(processorProperties.getDataCollectionRequestTopic(), dataCollectionRequest); + Map deviceOnboardingRequest = prepareRequestPayload(REQUEST_TYPE.INSTALL, registrationRequest); + brokerUtil.sendMessage(processorProperties.getDataCollectionRequestTopic(), deviceOnboardingRequest); registrationRequest.setStatus(RegistrationRequestStatus.ONBOARDING_REQUESTED); log.debug("processOnboardingRequests: Save updated request: id={}, request={}", registrationRequest.getId(), registrationRequest); @@ -173,18 +173,18 @@ private void processOnboardingRequests() { private static Map prepareRequestPayload(@NonNull REQUEST_TYPE requestType, RegistrationRequest registrationRequest) { try { - Map payload = new LinkedHashMap<>(Map.of( - "requestId", registrationRequest.getId(), - "requestType", requestType.name(), - "deviceId", registrationRequest.getDevice().getId(), - "deviceOs", registrationRequest.getDevice().getOs(), - "deviceName", registrationRequest.getDevice().getName(), - "deviceIpAddress", registrationRequest.getDevice().getIpAddress(), - "devicePort", Integer.toString( registrationRequest.getDevice().getPort() ), - "deviceUsername", registrationRequest.getDevice().getUsername(), - "devicePassword", new String(registrationRequest.getDevice().getPassword()), - "devicePublicKey", new String(registrationRequest.getDevice().getPublicKey()) - )); + Map payload = new LinkedHashMap<>(); + payload.put("requestId", registrationRequest.getId()); + payload.put("requestType", requestType.name()); + payload.put("deviceId", registrationRequest.getDevice().getId()); + payload.put("deviceRef", registrationRequest.getDevice().getRef()); + payload.put("deviceOs", registrationRequest.getDevice().getOs()); + payload.put("deviceName", registrationRequest.getDevice().getName()); + payload.put("deviceIpAddress", registrationRequest.getDevice().getIpAddress()); + payload.put("devicePort", Integer.toString( registrationRequest.getDevice().getPort() )); + payload.put("deviceUsername", registrationRequest.getDevice().getUsername()); + payload.put("devicePassword", new String(registrationRequest.getDevice().getPassword())); + payload.put("devicePublicKey", new String(registrationRequest.getDevice().getPublicKey())); payload.put("timestamp", Long.toString(Instant.now().toEpochMilli())); payload.put("priority", Double.toString(1.0)); payload.put("retry", Integer.toString(1)); From d47887e0f2edd5137b994bf85f395df316745c4a Mon Sep 17 00:00:00 2001 From: atsag Date: Wed, 6 Nov 2024 14:03:23 +0200 Subject: [PATCH 110/132] Improvements in the handling of AMQP connections --- .../broker_communication/BrokerPublisher.java | 53 +++++++++---------- .../ExtendedConnector.java | 9 +++- .../SynchronousBrokerPublisher.java | 40 ++++++-------- .../discovery/monitor/DeviceProcessor.java | 11 ++-- .../service/SALRegistrationService.java | 2 +- 5 files changed, 54 insertions(+), 61 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index b5299af..766effa 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -13,7 +13,7 @@ @Slf4j public class BrokerPublisher { public static String EMPTY=""; - private static HashMap> broker_and_topics_to_publish_to = new HashMap<>(); + //private HashMap> broker_and_topics_to_publish_to = new HashMap<>(); private Publisher private_publisher_instance; private ArrayList publishers = new ArrayList<>(); @@ -22,10 +22,8 @@ public class BrokerPublisher { private String broker_ip; private int broker_port; + public BrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { - this(topic,broker_ip,broker_port,brokerUsername,brokerPassword,amqLibraryConfigurationLocation,false); - } - public BrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation, boolean hard_initialize_connector) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); if (!able_to_initialize_BrokerPublisher){ @@ -33,40 +31,41 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b return; } boolean publisher_configuration_changed; - if (!broker_and_topics_to_publish_to.containsKey(broker_ip)){ - HashSet topics_to_publish_to = new HashSet<>(); - topics_to_publish_to.add(topic); - broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); - publisher_configuration_changed = true; - }else{ - if (!broker_and_topics_to_publish_to.get(broker_ip).contains(topic)){ - broker_and_topics_to_publish_to.get(broker_ip).add(topic); - publisher_configuration_changed = true; - } - else{ - publisher_configuration_changed = false; - } - } +// if (!broker_and_topics_to_publish_to.containsKey(broker_ip)){ +// HashSet topics_to_publish_to = new HashSet<>(); +// topics_to_publish_to.add(topic); +// broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); +// publisher_configuration_changed = true; +// }else{ +// if (!broker_and_topics_to_publish_to.get(broker_ip).contains(topic)){ +// broker_and_topics_to_publish_to.get(broker_ip).add(topic); +// publisher_configuration_changed = true; +// } +// else{ +// publisher_configuration_changed = false; +// } +// } - if (publisher_configuration_changed || hard_initialize_connector){ + //if (publisher_configuration_changed || hard_initialize_connector){ // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { active_connector.stop(new ArrayList<>(), publishers); } publishers.clear(); - for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ + //for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ //ArrayList publishers = new ArrayList<>(); - Publisher publisher = new Publisher("resource_manager_"+broker_topic, broker_topic, true, true); + Publisher publisher = new Publisher("resource_manager_"+topic, topic, true, true); publishers.add(publisher); - if (broker_topic.equals(topic)){ - this.private_publisher_instance = publishers.get(publishers.size()-1); - this.topic = broker_topic; + //if (broker_topic.equals(topic)){ + this.private_publisher_instance = publisher; + //this.topic = broker_topic; + this.topic = topic; this.broker_ip = broker_ip; this.broker_port = broker_port; - } - } + //} + //} //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); active_connector = new ExtendedConnector("resource_manager" @@ -86,7 +85,7 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b ); active_connector.start(); - } + //} } //TODO The methods below assume that the only content to be sent is json-like diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java index 5773402..b35e855 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java @@ -74,7 +74,14 @@ public void stop(ArrayList consumers, ArrayList publishers if (publishers.size()>0) { stop_publishers(publishers); } - this.stop(); + Context context = ((CustomConnectorHandler)handler).getContext(); + try { + context.stop(); + this.stop(); + log.info("Successfully stopped the ExtendedConnector"); + }catch (Exception e){ + log.warn("There was an issue while trying to stop an ExtendedConnector"); + } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 3db8fa6..341cadc 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -15,7 +15,7 @@ @Slf4j public class SynchronousBrokerPublisher { public static String EMPTY=""; - private static HashMap> broker_and_topics_to_publish_to = new HashMap<>(); + private HashMap> broker_and_topics_to_publish_to = new HashMap<>(); private SyncedPublisher private_publisher_instance; private ArrayList publishers = new ArrayList<>(); @@ -35,25 +35,13 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por return; } boolean publisher_configuration_changed; - if (!broker_and_topics_to_publish_to.containsKey(broker_ip)){ - HashSet topics_to_publish_to = new HashSet<>(); - topics_to_publish_to.add(topic); - broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); - //log.error("changed1"); - publisher_configuration_changed = true; - }else{ - if (!broker_and_topics_to_publish_to.get(broker_ip).contains(topic)){ - broker_and_topics_to_publish_to.get(broker_ip).add(topic); - //log.error("changed2"); - publisher_configuration_changed = true; - } - else{ - publisher_configuration_changed = false; - } - } + HashSet topics_to_publish_to = new HashSet<>(); + topics_to_publish_to.add(topic); + broker_and_topics_to_publish_to.put(broker_ip,topics_to_publish_to); + //log.error("changed1"); //log.error("preliminary_outside"); - if (publisher_configuration_changed || hard_initialize_connector){ + //if (publisher_configuration_changed || hard_initialize_connector){ //log.error("preliminary_inside1"); // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); @@ -61,18 +49,20 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por active_connector.stop(new ArrayList<>(), publishers); } publishers.clear(); - for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ + //for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ //log.error("preliminary_inside2"); //ArrayList publishers = new ArrayList<>(); - SyncedPublisher publisher = new SyncedPublisher("resource_manager_"+broker_topic, broker_topic, true, true); + //SyncedPublisher publisher = new SyncedPublisher("resource_manager_"+broker_topic, broker_topic, true, true); + SyncedPublisher publisher = new SyncedPublisher("resource_manager_"+topic, topic, true, true); publishers.add(publisher); - if (broker_topic.equals(topic)){ + // if (broker_topic.equals(topic)){ log.error("inside_assignment_to_private_publisher_instance"); this.private_publisher_instance = (SyncedPublisher) publishers.get(publishers.size()-1); - this.topic = broker_topic; + //this.topic = broker_topic; + this.topic = topic; this.broker_ip = broker_ip; - } - } + // } + //} //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); active_connector = new ExtendedConnector("resource_manager_synchronous" @@ -92,7 +82,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por ); active_connector.start(); - } + //} } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index d4668b6..e65bc22 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -150,14 +150,11 @@ private void processFailedDevices() { while (device_lost_publisher.is_publisher_null()){ try { - log.info("Attempting to recreate new BrokerPublisher to publish the device lost message"); + log.warn("Will now make attempt No "+sending_attempt+" to recreate the BrokerPublisher connector for the lost device topic"); log.info("The topic name is "+processorProperties.getLost_device_topic()+", the broker ip is "+ processorProperties.getNebulous_broker_ip_address()+", the broker port is "+ processorProperties.getNebulous_broker_port()+", the username is "+ processorProperties.getNebulous_broker_username()+", and the password is "+ processorProperties.getNebulous_broker_password()); - if (sending_attempt<=2) { - device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); - }else{ - log.warn("Will now attempt to reset the BrokerPublisher connector"); - device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), "",true); - } + + device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + Thread.sleep(3000); }catch (InterruptedException i){ i.printStackTrace(); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 1cd659a..870fa67 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -130,7 +130,7 @@ public void register(Device device) { if (sending_attempt<=2) { register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); }else{ - log.warn("Will now attempt to reset the Synchronous publisher connector"); + log.warn("Will now attempt to reset the Synchronous publisher connector to register"); register_device_publisher = new SynchronousBrokerPublisher(get_registration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); } try { From 93a147985e87ae4bc88522914d8c46d94ecb1ed8 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 12:38:10 +0200 Subject: [PATCH 111/132] Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher --- .../broker_communication/SynchronousBrokerPublisher.java | 4 ---- .../nebulous/resource/discovery/monitor/model/Device.java | 1 + .../discovery/monitor/service/DeviceManagementService.java | 1 + .../monitor/service/UnknownDeviceRegistrationService.java | 2 ++ .../RegistrationRequestService_SampleDataCreator.java | 1 + .../resource/discovery/registration/model/Device.java | 1 + .../registration/service/RegistrationRequestService.java | 3 ++- .../registration/service/SALDeregistrationService.java | 2 +- .../registration/service/SALRegistrationService.java | 2 +- .../static/freebees_webdesign_6/request-edit.html | 7 +++++++ 10 files changed, 17 insertions(+), 7 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 341cadc..4f18320 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -23,10 +23,6 @@ public class SynchronousBrokerPublisher { private String topic; private String broker_ip; public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { - this(topic, broker_ip, broker_port, brokerUsername, brokerPassword, amqLibraryConfigurationLocation,false); - } - - public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation,boolean hard_initialize_connector) { boolean able_to_initialize_BrokerPublisher = topic!=null && broker_ip!=null && brokerUsername!=null && brokerPassword!=null && !topic.equals(EMPTY) && !broker_ip.equals(EMPTY) && !brokerUsername.equals(EMPTY) && !brokerPassword.equals(EMPTY); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index 5ad29d3..2a771e0 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -21,6 +21,7 @@ public class Device { private String os; private String name; private String owner; + private String provider; private String ipAddress; private int port; private DeviceLocation location; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index ac4ee52..b6ed8dc 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -107,6 +107,7 @@ public void checkDevice(@NonNull Device device, boolean dryRun) { if (!dryRun && StringUtils.isBlank(device.getId())) errors.add("Null or blank Id"); if (StringUtils.isBlank(device.getOs())) errors.add("Null or blank OS"); if (StringUtils.isBlank(device.getOwner())) errors.add("Null or blank Owner"); + if (StringUtils.isBlank(device.getProvider())) errors.add("Null or blank Provider"); if (StringUtils.isBlank(device.getIpAddress())) errors.add("Null or blank IP address"); if (!dryRun && StringUtils.isBlank(device.getNodeReference())) errors.add("Null or blank Node reference"); if (!dryRun && device.getCreationDate()==null) errors.add("Null Creation date"); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index f079426..87442c8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -191,6 +191,7 @@ private void processDeviceDetailsResponses() { String os = map.getOrDefault("os", null).toString(); String name = map.getOrDefault("name", null).toString(); String owner = "-EMS-"; + String provider = map.getOrDefault("provider", null).toString(); String ipAddress = map.getOrDefault("deviceIpAddress", null).toString(); String username = map.getOrDefault("username", null).toString(); char[] password = map.getOrDefault("password", "").toString().toCharArray(); @@ -235,6 +236,7 @@ private void processDeviceDetailsResponses() { Device newDevice = Device.builder() .name(name) .owner("--EMS--") + .provider(provider) .requestId(requestId) .ipAddress(ipAddress) .nodeReference(nodeReference) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java index ec09d79..f9dbf11 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestService_SampleDataCreator.java @@ -63,6 +63,7 @@ private Device createDevice(int pos, String owner) { .name( "Device name #"+pos ) .os("LINUX") .owner(owner) + .provider("default-provider") .ipAddress("10.10.0."+(100+pos)) .username("ubuntu_"+pos) .password("password".toCharArray()) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java index ae18e57..3dc295b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java @@ -18,6 +18,7 @@ public class Device { private String os; private String name; private String owner; + private String provider; private String ipAddress; private int port; private DeviceLocation location; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java index 8825a14..97e59fb 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/RegistrationRequestService.java @@ -103,7 +103,7 @@ public RegistrationRequest update(@NonNull RegistrationRequest registrationReque Optional result = getById(registrationRequest.getId()); if (result.isEmpty()) throw new RegistrationRequestException( - "Registration request with the Id does not exists in repository: "+registrationRequest.getId()); + "Registration request with Id: "+registrationRequest.getId()+" does not exist in repository"); checkRegistrationRequest(registrationRequest, result.get(), authentication); if (checkEditDel) canEditOrDelete(result.get()); @@ -174,6 +174,7 @@ private void checkRegistrationRequest(@NonNull RegistrationRequest registrationR private void checkDevice(@NonNull Device device, Device storedRequestDevice, Authentication authentication, List errors) { //if (StringUtils.isBlank(device.getId())) errors.add("Null or blank device id"); if (StringUtils.isBlank(device.getOwner())) errors.add("Null or blank device owner"); + if (StringUtils.isBlank(device.getProvider())) errors.add("Null or blank device provider"); if (authentication!=null && StringUtils.isNotBlank(authentication.getName())) if (! isAdmin(authentication) && ! device.getOwner().equals(authentication.getName())) errors.add("Device owner differs from authenticated user"); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java index 2354c66..ed020b0 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -67,7 +67,7 @@ public void deregister(Device device) { deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); } else { log.warn("Will now attempt to reset the Synchronous publisher connector to deregister"); - deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), "",true); + deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); } try { Thread.sleep(3000); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 870fa67..47b7c9b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -80,7 +80,7 @@ public void register(Device device) { //TODO implement provider here: String provider = device.getProvider(); //String network_rx =device_info.get("RX"); //String network_tx = device_info.get("TX"); - String provider_id = device.getOwner(); //TODO improve this + String provider_id = device.getProvider(); String city_name = ""; //TODO improve this String country_name = ""; //TODO improve this String internal_ip = ""; //TODO improve this diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html index 819f661..b3b7550 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -479,6 +479,12 @@
Device details
+
+ +
+ +
+
@@ -576,6 +582,7 @@
Device details
You are free to: Share and Adapt. You must give appropriate credit, you may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. -->
+
From 90c18b02f95d7e9c671cabd5aa12fda2999016dd Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 15:38:15 +0200 Subject: [PATCH 112/132] Minor logging improvements --- .../discovery/broker_communication/BrokerPublisher.java | 2 +- .../broker_communication/SynchronousBrokerPublisher.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 766effa..1cf0ceb 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -97,7 +97,7 @@ public void publish (String json_string_content, Collection application_ try { json_object = (JSONObject) parser.parse(json_string_content); } catch (ParseException p) { - log.warn( "Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); + log.warn( "publish: Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); } if (!is_publisher_null()) { private_publisher_instance.send(json_object); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index 4f18320..a37223f 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -98,7 +98,7 @@ public Map publish_for_response (String json_string_content, Collection json_object = (JSONObject) parser.parse(json_string_content); successful_json_parsing = true; } catch (ParseException p) { - log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); + log.warn("publish_for_response-if: Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); } metadata.put("jobId",application_name); payload.put("metaData",metadata); @@ -124,7 +124,7 @@ public Map publish_for_response (String json_string_content, Collection json_object = (JSONObject) parser.parse(json_string_content); } catch (ParseException p) { - log.warn("Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); + log.warn("publish_for_response-else: Could not parse the string content to be published to the broker as json, which is the following: " + json_string_content); } if (private_publisher_instance != null) { log.info("Sending new synchronous message\n"+json_object.toJSONString()); From aecb6b58d9bb7a968e11849b480545334c16b371 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 16:36:01 +0200 Subject: [PATCH 113/132] Deregistration process improvement --- .../discovery/ResourceDiscoveryProperties.java | 1 + .../broker_communication/SALCommunicator.java | 4 +++- .../service/SALDeregistrationService.java | 12 +++++++++--- .../src/main/resources/application.yml | 1 + 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 8a0c638..7ff3345 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -103,6 +103,7 @@ public class ResourceDiscoveryProperties { private int nebulous_broker_port; private String nebulous_broker_username; private String nebulous_broker_password; + private boolean deregistration_emulated; private String lost_device_topic; private String compromised_device_topic; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index 0303b3a..cc78d2f 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -143,7 +143,9 @@ private static void register_devices(String request_body_file, String sessionID, } public static String get_device_deregistration_json(Device device){ - return ""; + JSONObject root_json_object = new JSONObject(); + root_json_object.put("name",device.getRef()); + return root_json_object.toJSONString(); } public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, long ram_gb, long disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java index ed020b0..e642227 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -59,7 +59,9 @@ public void deregister(Device device) { String deregister_device_message_string = get_device_deregistration_json(device); - + if (processorProperties.isDeregistration_emulated()){ + return; + } SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (deregister_device_publisher.is_publisher_null()) { @@ -77,8 +79,12 @@ public void deregister(Device device) { sending_attempt++; } //TODO handle the response here - Map response = deregister_device_publisher.publish_for_response(deregister_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); + if (deregister_device_message_string!=null && !deregister_device_message_string.isEmpty()) { + Map response = deregister_device_publisher.publish_for_response(deregister_device_message_string, Collections.singleton(application_name)); + log.info("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); + }else{ + log.warn("Deregistration was to be initiated with an empty deregistration payload"); + } //} /* This is some realtime information, could be retrieved with a different call to the EMS. diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index be6c227..835793f 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -24,6 +24,7 @@ discovery: nebulous_broker_port: 5672 nebulous_broker_username: "admin" nebulous_broker_password: "admin" + deregistration_emulated: true sal_host: "localhost" sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" From 91c62fbdb66177657d336a72630940535550b839 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 17:49:40 +0200 Subject: [PATCH 114/132] Stopping the device lost publisher --- .../discovery/broker_communication/BrokerPublisher.java | 6 ++++++ .../resource/discovery/monitor/DeviceProcessor.java | 1 + .../discovery/monitor/service/DeviceManagementService.java | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 1cf0ceb..04daf94 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -110,4 +110,10 @@ public void publish (String json_string_content, Collection application_ public boolean is_publisher_null(){ return (private_publisher_instance == null); } + + public void stop(){ + if (active_connector!=null) { + active_connector.stop(new ArrayList<>(), publishers); + } + } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index e65bc22..2ea34c3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -163,6 +163,7 @@ private void processFailedDevices() { } device_lost_publisher.publish(lost_device_message.toJSONString(), Collections.singleton("")); log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); + device_lost_publisher.stop(); } deviceManagementService.update(device); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java index b6ed8dc..3058cc6 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/DeviceManagementService.java @@ -92,7 +92,7 @@ public Device update(@NonNull Device device) { Optional result = getById(device.getId()); if (result.isEmpty()) throw new DeviceException( - "Device with the Id does not exists in repository: "+device.getId()); + "Device with the Id does not exist in repository: "+device.getId()); checkDevice(device, false); device.setLastUpdateDate(Instant.now()); From 0bde76bb084e69fe9f21b8fe07c2bcd1dda36ce8 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 20:45:13 +0200 Subject: [PATCH 115/132] Refactoring to use the original Connector class instead of ExtendedConnector --- .../broker_communication/BrokerPublisher.java | 11 +++++++---- .../BrokerSubscriber.java | 19 ++++++++++++------- .../ExtendedConnector.java | 2 +- .../SynchronousBrokerPublisher.java | 8 +++++--- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 04daf94..6dc1ffe 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.broker_communication; +import eu.nebulouscloud.exn.Connector; import eu.nebulouscloud.exn.core.Publisher; import eu.nebulouscloud.exn.settings.StaticExnConfig; import lombok.extern.slf4j.Slf4j; @@ -17,7 +18,7 @@ public class BrokerPublisher { private Publisher private_publisher_instance; private ArrayList publishers = new ArrayList<>(); - private ExtendedConnector active_connector; + private Connector active_connector; private String topic; private String broker_ip; private int broker_port; @@ -51,7 +52,8 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { - active_connector.stop(new ArrayList<>(), publishers); + //active_connector.stop(new ArrayList<>(), publishers); + active_connector.stop(); } publishers.clear(); //for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ @@ -68,7 +70,7 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b //} //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); - active_connector = new ExtendedConnector("resource_manager" + active_connector = new Connector("resource_manager" , new CustomConnectorHandler() {} , publishers , List.of(), @@ -113,7 +115,8 @@ public boolean is_publisher_null(){ public void stop(){ if (active_connector!=null) { - active_connector.stop(new ArrayList<>(), publishers); + //active_connector.stop(new ArrayList<>(), publishers); + active_connector.stop(); } } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java index 844fc0b..8963f9d 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerSubscriber.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.broker_communication; +import eu.nebulouscloud.exn.Connector; import eu.nebulouscloud.exn.core.Consumer; import eu.nebulouscloud.exn.core.Context; import eu.nebulouscloud.exn.core.Handler; @@ -55,7 +56,7 @@ public void setProcessing_function(BiFunction processing_function) { private static HashMap> broker_and_topics_to_subscribe_to = new HashMap<>(); private static HashMap> active_consumers_per_topic_per_broker_ip = new HashMap<>(); - private static HashMap current_connectors = new HashMap<>(); + private static HashMap current_connectors = new HashMap<>(); ArrayList consumers = new ArrayList<>(); private String topic; private String broker_ip; @@ -124,7 +125,8 @@ public BrokerSubscriber(String topic, String broker_ip, int broker_port, String */ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { if (current_connectors.get(broker_ip) != null) { - current_connectors.get(broker_ip).stop(consumers,new ArrayList<>()); + //current_connectors.get(broker_ip).stop(consumers,new ArrayList<>()); + current_connectors.get(broker_ip).stop(); } if (consumers.isEmpty()){ consumers = new ArrayList<>(); @@ -138,7 +140,7 @@ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { if(!do_not_add_new_consumer) { consumers.add(new_consumer); } - ExtendedConnector extended_connector = new ExtendedConnector("resource_manager", + Connector connector = new Connector("resource_manager", new CustomConnectorHandler() { }, List.of(), @@ -154,13 +156,13 @@ private void add_topic_consumer_to_broker_connector(Consumer new_consumer) { EMPTY ) ); - extended_connector.start(); - current_connectors.put(broker_ip, extended_connector); + connector.start(); + current_connectors.put(broker_ip, connector); } private void remove_topic_from_broker_connector(String topic_key) { if (current_connectors.get(broker_ip) != null) { - current_connectors.get(broker_ip).remove_consumer_with_key(topic_key); + //current_connectors.get(broker_ip).remove_consumer_with_key(topic_key); } } @@ -171,6 +173,7 @@ public int subscribe (BiFunction function, String application_name) { public int subscribe(BiFunction function, String application_name, AtomicBoolean stop_signal) { int exit_status = -1; log.info("ESTABLISHING SUBSCRIPTION for " + topic); + /* //First remove any leftover consumer if (active_consumers_per_topic_per_broker_ip.containsKey(broker_ip)) { active_consumers_per_topic_per_broker_ip.get(broker_ip).remove(topic); @@ -178,6 +181,7 @@ public int subscribe(BiFunction function, String application_name, AtomicBoolean } else { active_consumers_per_topic_per_broker_ip.put(broker_ip, new HashMap<>()); } + */ //Then add the new consumer Consumer new_consumer; if (application_name != null && !application_name.equals(EMPTY)) { @@ -204,7 +208,8 @@ public int subscribe(BiFunction function, String application_name, AtomicBoolean stop_signal.set(false); } active_consumers_per_topic_per_broker_ip.get(broker_ip).remove(topic); - remove_topic_from_broker_connector(topic); + //remove_topic_from_broker_connector(topic); + current_connectors.get(broker_ip).stop(); exit_status = 0; return exit_status; } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java index b35e855..3c84c15 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/ExtendedConnector.java @@ -76,7 +76,7 @@ public void stop(ArrayList consumers, ArrayList publishers } Context context = ((CustomConnectorHandler)handler).getContext(); try { - context.stop(); + //context.stop(); this.stop(); log.info("Successfully stopped the ExtendedConnector"); }catch (Exception e){ diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java index a37223f..ff156fc 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SynchronousBrokerPublisher.java @@ -1,5 +1,6 @@ package eu.nebulous.resource.discovery.broker_communication; +import eu.nebulouscloud.exn.Connector; import eu.nebulouscloud.exn.core.Publisher; import eu.nebulouscloud.exn.core.SyncedPublisher; import eu.nebulouscloud.exn.settings.StaticExnConfig; @@ -19,7 +20,7 @@ public class SynchronousBrokerPublisher { private SyncedPublisher private_publisher_instance; private ArrayList publishers = new ArrayList<>(); - private ExtendedConnector active_connector; + private Connector active_connector; private String topic; private String broker_ip; public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_port, String brokerUsername, String brokerPassword, String amqLibraryConfigurationLocation) { @@ -42,7 +43,8 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { - active_connector.stop(new ArrayList<>(), publishers); + //active_connector.stop(new ArrayList<>(), publishers); + active_connector.stop(); } publishers.clear(); //for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ @@ -61,7 +63,7 @@ public SynchronousBrokerPublisher(String topic, String broker_ip, int broker_por //} //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); - active_connector = new ExtendedConnector("resource_manager_synchronous" + active_connector = new Connector("resource_manager_synchronous" , new CustomConnectorHandler() {} , publishers , List.of(), From 96c195176f7b0049b14870ba56b81454d99b9755 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 11 Nov 2024 20:54:31 +0200 Subject: [PATCH 116/132] Minor logging improvement --- .../discovery/broker_communication/BrokerPublisher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 6dc1ffe..ec13586 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -50,7 +50,7 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b //if (publisher_configuration_changed || hard_initialize_connector){ // for (String current_broker_ip : broker_and_topics_to_publish_to.keySet()){ - log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); + log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { //active_connector.stop(new ArrayList<>(), publishers); active_connector.stop(); From 3abb2c2db4aca3b8c9e29a3d976e18e4d2d98f7a Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 12 Nov 2024 14:40:56 +0200 Subject: [PATCH 117/132] Only try to stop the connector if it has been previously initialized --- .../broker_communication/BrokerPublisher.java | 18 +++++++++++++++--- .../CustomConnectorHandler.java | 11 +++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index ec13586..c990d97 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -2,6 +2,7 @@ import eu.nebulouscloud.exn.Connector; import eu.nebulouscloud.exn.core.Publisher; +import eu.nebulouscloud.exn.handlers.ConnectorHandler; import eu.nebulouscloud.exn.settings.StaticExnConfig; import lombok.extern.slf4j.Slf4j; import org.json.simple.JSONObject; @@ -19,6 +20,7 @@ public class BrokerPublisher { private ArrayList publishers = new ArrayList<>(); private Connector active_connector; + private CustomConnectorHandler active_connector_handler; private String topic; private String broker_ip; private int broker_port; @@ -53,7 +55,17 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b log.info("Publisher configuration changed, creating new connector at "+broker_ip+" for topic "+topic); if (active_connector!=null) { //active_connector.stop(new ArrayList<>(), publishers); - active_connector.stop(); + synchronized (active_connector_handler.getReady()){ + while (!active_connector_handler.getReady().get()) { + try { + active_connector_handler.wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + active_connector.stop(); + } + } publishers.clear(); //for (String broker_topic : broker_and_topics_to_publish_to.get(broker_ip)){ @@ -69,9 +81,9 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b //} //} //CustomConnectorHandler custom_handler = new CustomConnectorHandler(); - + active_connector_handler = new CustomConnectorHandler() {}; active_connector = new Connector("resource_manager" - , new CustomConnectorHandler() {} + , active_connector_handler , publishers , List.of(), false, diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java index 6959a21..ff4b604 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java @@ -3,12 +3,19 @@ import eu.nebulouscloud.exn.core.Context; import eu.nebulouscloud.exn.handlers.ConnectorHandler; +import java.util.concurrent.atomic.AtomicBoolean; + public class CustomConnectorHandler extends ConnectorHandler { private Context context; + private final AtomicBoolean ready = new AtomicBoolean(false); @Override public void onReady(Context context) { this.context = context; + synchronized (ready) { + this.ready.set(true); + this.ready.notify(); + } } public void remove_consumer_with_key(String key){ context.unregisterConsumer(key); @@ -21,4 +28,8 @@ public Context getContext() { public void setContext(Context context) { this.context = context; } + + public AtomicBoolean getReady() { + return ready; + } } From e40c231e2aeb4e134a76d623e14a9cae1fbe1e12 Mon Sep 17 00:00:00 2001 From: atsag Date: Mon, 18 Nov 2024 17:40:00 +0200 Subject: [PATCH 118/132] Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized --- .../broker_communication/BrokerPublisher.java | 10 ++++++++++ .../broker_communication/CustomConnectorHandler.java | 2 +- .../broker_communication/SALCommunicator.java | 3 ++- .../resource/discovery/monitor/model/Device.java | 1 + .../service/UnknownDeviceRegistrationService.java | 2 ++ .../resource/discovery/registration/model/Device.java | 1 + .../registration/service/SALDeregistrationService.java | 2 +- .../registration/service/SALRegistrationService.java | 6 ++++-- .../static/freebees_webdesign_6/request-edit.html | 6 ++++++ 9 files changed, 28 insertions(+), 5 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index c990d97..6863080 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -128,6 +128,16 @@ public boolean is_publisher_null(){ public void stop(){ if (active_connector!=null) { //active_connector.stop(new ArrayList<>(), publishers); + synchronized (active_connector_handler.getReady()){ + while (!active_connector_handler.getReady().get()) { + try { + active_connector_handler.getReady().wait(); + } catch (InterruptedException e) { + + } + } + } + active_connector.stop(); } } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java index ff4b604..93c9cdd 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/CustomConnectorHandler.java @@ -14,7 +14,7 @@ public void onReady(Context context) { this.context = context; synchronized (ready) { this.ready.set(true); - this.ready.notify(); + this.ready.notifyAll(); } } public void remove_consumer_with_key(String key){ diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index cc78d2f..b7a598e 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -148,7 +148,7 @@ public static String get_device_deregistration_json(Device device){ return root_json_object.toJSONString(); } - public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, long ram_gb, long disk_gb, String device_name,String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { + public static String get_device_registration_json(String internal_ip_address, String external_ip_address, int external_access_port, String os_family, String os_architecture, String jar_url, int os_version, int cpu_cores, long ram_gb, long disk_gb, String device_name,Double price, String provider_id, String city_name, String country_name, String device_username, String device_password, String private_key, double device_longitude, double device_latitude) { JSONObject root_json_object = new JSONObject(); JSONObject loginCredential = new JSONObject(); @@ -185,6 +185,7 @@ public static String get_device_registration_json(String internal_ip_address, St nodeProperties.put("numberOfCores", cpu_cores); nodeProperties.put("memory", ram_gb); nodeProperties.put("disk", disk_gb); + nodeProperties.put("price", price); nodeProperties.put("operatingSystem", operatingSystem); nodeProperties.put("geoLocation", geoLocation); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index 2a771e0..f3df47e 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -18,6 +18,7 @@ public class Device { private String id; private String ref; + private Double price; private String os; private String name; private String owner; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index 87442c8..b544fcc 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -192,6 +192,7 @@ private void processDeviceDetailsResponses() { String name = map.getOrDefault("name", null).toString(); String owner = "-EMS-"; String provider = map.getOrDefault("provider", null).toString(); + Double price = (Double) map.getOrDefault("price", null); String ipAddress = map.getOrDefault("deviceIpAddress", null).toString(); String username = map.getOrDefault("username", null).toString(); char[] password = map.getOrDefault("password", "").toString().toCharArray(); @@ -237,6 +238,7 @@ private void processDeviceDetailsResponses() { .name(name) .owner("--EMS--") .provider(provider) + .price(price) .requestId(requestId) .ipAddress(ipAddress) .nodeReference(nodeReference) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java index 3dc295b..efc5b55 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/model/Device.java @@ -15,6 +15,7 @@ public class Device { private String id; private String ref; + private Double price; private String os; private String name; private String owner; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java index e642227..95099a2 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -43,7 +43,7 @@ public class SALDeregistrationService implements InitializingBean { private Thread processQueueThread; private long lastDeregistrationStartTimestamp = -1L; - public void queueForRegistration(@NonNull Device device) { + public void queueForDeregistration(@NonNull Device device) { if (processorProperties.isSalRegistrationEnabled()) //If registration is enabled, so should be deregistration as well queue.add(device); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 47b7c9b..a5b769a 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -69,13 +69,15 @@ public void register(Device device) { echo OS_KERNEL_RELEASE=$TMP_KERNEL_RELEASE */ String device_name = device.getRef(); + Double price = device.getPrice(); int cores = Integer.parseInt(device_info.get("CPU_PROCESSORS")); long ram_mb = Math.round(Integer.parseInt(device_info.get("RAM_TOTAL_KB"))*1.0/1000); long disk_mb = Math.round(Integer.parseInt(device_info.get("DISK_TOTAL_KB"))*1.0/1000); String external_ip_address = device.getIpAddress(); String device_username = device.getUsername(); - String device_password = new String(device.getPassword()); + String device_password = new String + (device.getPassword()); String device_pub_key = new String(device.getPublicKey()); //TODO get here private key instead and pass this to device registration //TODO implement provider here: String provider = device.getProvider(); //String network_rx =device_info.get("RX"); @@ -115,7 +117,7 @@ public void register(Device device) { //register_device_message.put("timestamp",(int)(clock.millis()/1000)); - String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,jar_url,os_version,cores,ram_mb,disk_mb,device_name,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); + String register_device_message_string = get_device_registration_json(internal_ip,external_ip_address,external_access_port,os_family,os_architecture,jar_url,os_version,cores,ram_mb,disk_mb,device_name,price,provider_id,city_name,country_name, device_username, device_password,private_key,device_longitude, device_latitude); log.info("topic is {}", get_registration_topic_name(application_name)); log.info("broker ip is {}", processorProperties.getNebulous_broker_ip_address()); log.info("broker port is {}", processorProperties.getNebulous_broker_port()); diff --git a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html index b3b7550..56f3409 100644 --- a/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html +++ b/resource-discovery/src/main/resources/static/freebees_webdesign_6/request-edit.html @@ -479,6 +479,12 @@
Device details
+
+ +
+ +
+
From c29abe4fee2e4120153f5d9a30e9647f543d732a Mon Sep 17 00:00:00 2001 From: atsag Date: Tue, 19 Nov 2024 18:25:22 +0200 Subject: [PATCH 119/132] Password improvements Removal of plaintext passwords and replacement with environmental variable values --- resource-discovery/src/main/resources/application.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-discovery/src/main/resources/application.yml b/resource-discovery/src/main/resources/application.yml index 835793f..dea2804 100644 --- a/resource-discovery/src/main/resources/application.yml +++ b/resource-discovery/src/main/resources/application.yml @@ -29,7 +29,7 @@ discovery: sal_port: 8080 lost_device_topic: "eu.nebulouscloud.monitoring.device_lost" compromised_device_topic: "eu.nebulouscloud.monitoring.device_compromised" - nebulous_server_ip_address: "158.39.201.36" + nebulous_server_ip_address: ${NEBULOUS_SERVER_IP_ADDRESS} # trustStoreFile: tests/config/broker-truststore.p12 # trustStorePassword: melodic # trustStoreType: PKCS12 @@ -39,10 +39,10 @@ discovery: # To generate BCrypt encrypted passwords you can use: https://bcrypt-generator.com/ users: - username: admin - password: '$2a$10$5jzrhbVKq.W2J1PMGYeHyeydQtlw71PoVgryzDP0VZ.88FsPlq1ne' # admin1 (BCrypt; 10 iterations) + password: ${RM_ADMIN_PASSWD} roles: [ ADMIN ] - username: user - password: '$2a$10$I6GSOKiY5n4/Ql0LA7Js0.4HT4UXVCNaNpGv5UdZt/brEdv/F.ttG' # user1 (BCrypt; 10 iterations) + password: ${RM_USER_PASSWD} roles: [ USER ] # apiKeyAuthenticationEnabled: true From 982b8a6b2cb08d3a4b1f1e97275ef642289bed11 Mon Sep 17 00:00:00 2001 From: atsag Date: Wed, 20 Nov 2024 13:47:29 +0200 Subject: [PATCH 120/132] Debugging message commit --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index a5b769a..3113101 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -144,7 +144,7 @@ public void register(Device device) { } //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to register device " + device_name + " is "+response.toString()); + log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); //} /* This is some realtime information, could be retrieved with a different call to the EMS. From f88412290ff970c089ba04ee8207cd2a034c56ce Mon Sep 17 00:00:00 2001 From: atsag Date: Wed, 20 Nov 2024 13:48:05 +0200 Subject: [PATCH 121/132] Debugging message (#32) * RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. * RD: Moved 'metrics' section side-by-side to 'device info' section in device-view.html, for better viewing * RD: changed mongo db name to 'resource_discovery' in application.yml * RD: Minor GUI improvements * RD: Added DeviceMonitorService to watch the status updates from EMS. Added DeviceStatusUpdate class used in DeviceMonitorService, and added field statusUpdate in Device class. Restructured and improved ResourceDiscoveryProperties. * RD: Created AbstractMonitorService (to factor out common topic monitoring code). Renamed DeviceMonitorService to DeviceStatusMonitorService and made it subclass of AbstractMonitorService. Changed DeviceStatusUpdate class to ignore unknown properties during deserialization. * RD: Added banner * RD: Added DeviceMetricsMonitorService and DeviceMetrics classes. Updated Device class to store device metrics received from EMS * RD: Updated application.yml * RD: Added device metrics display in device-view.html. Added (EMS) device status in devices.html * RD: Updated device-view.html to display device metrics * RD: Modified Device in monitoring subsystem to not have its credentials serialized (and returned to browser). Also updated the affected code * RD: Added check for a request's device IP address being in use (in another request or a registered device) * RD: various minor code improvements * RD: Added better error reporting in request-edit.html * RD: Various small GUI improvements and a fix * RD: Fixed registration Device in order toString() not to print credentials. Fixed RegistrationRequest in order to have its 'messages' and 'history' fields copied by BeanUtils. Fixed 2 bugs in RegistrationRequestProcessor and added a few more logs. * RD: Minor change in device-view.html to move 'count-' metrics to the end of the device metrics list * RD: Changed monitoring Device to not serialize credentials * RD: Code cleanup in RegistrationRequestProcessor * RD: Improved GUI (added page shortcuts at top-right) * RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] * RD: Added DeviceLifeCycleRequestService and DeviceLifeCycleResponseService classes for handling reinstall and uninstall actions (via EMS broker). Moved LC request event sending from DeviceManagementService to DeviceLifeCycleRequestService. Updated ResourceDiscoveryProperties with settings for DeviceLifeCycleRequest/ResponseService's. * RD: Added device archiving and unarchiving in GUI * RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. * RD: Changed devices.html to always display 'Archive' button to Admin users * RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html * RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. * RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. * RD: Added shortcuts (top-right corner) to all detail pages * RD: Minor change in index.html (grayed out settings image) * RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. * RD: Minor code (mostly logging) and GUI improvements * RD: Added 'UnknownDeviceRegistrationService' class to monitor for unknown devices (based on IP address and reference in messages received from EMS), and registering them to RD. Added NODE_DETAILS in REQUEST_TYPE enum for messages sent to/received from EMS about acquiring detailed node info (including credentials). Added 'deviceInfoRequestsTopic' and 'deviceInfoReportsTopic' settings in ResourceDiscoveryProperties. * RD: Added prompting admin to provide device credentials when un-archiving a previously archived request or device. * RD: Moved REQUEST_TYPE enum to 'common' package and updated code * RD: Changed authentication to use BCrypt-encrypted passwords * RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. * RD: Minor code tidy up * RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml * RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does * RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version * RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. * RD: Added StatusController to report current application status (currently returns 'OK') * RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly * RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version * RD: Renamed 'management' module to 'resource-discovery' * RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore * RD: Modified base images in Dockerfile * RD: Modified Dockerfile in order to both compile the app and build the Docker image * RD: Improved Dockerfile * RD: Improved Dockerfile * RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. * RD: Fixed Dockerfile * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * EMS: Added K8sNetdataCollector [WIP] * RD: Added 'port' in forms and models * RD: Added two TODOs * RD: Deactivated UnknownDeviceRegistrationService * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 * Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 * Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 * Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 * Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 * Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml * RD: Fixed banner.txt * RD: Modified application.yml * RD: Added temp. debug messages * RD: Added temp. debug messages 2 * Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. * Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. * RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. * RD: Added temp. log messages * Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. * RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) * Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 * Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Updated Dockerfile * RD: Updated run.sh and added wait_for_mongodb.sh * RD: Improved wait_for_mongodb.sh script * Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 * Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 * Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' * RD: Added API Key authentication * RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. * RD: Code improvements (esp. wrt @PreAuthorize annotations) * RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. * RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. * RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). * RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. * Use device reference instead of name to register to SAL * Improvements in device registration and component communication with the broker * Small improvements in device registration * Small improvement in getting device registration details * Small improvement in getting device registration details * RD: Fix in SALRegistrationService class * RD: Fixed fontawesome cdn url * Initial deregistration support * RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) * RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS * Improvements in the handling of AMQP connections * Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher * Minor logging improvements * Deregistration process improvement * Stopping the device lost publisher * Refactoring to use the original Connector class instead of ExtendedConnector * Minor logging improvement * Only try to stop the connector if it has been previously initialized * Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized * Password improvements Removal of plaintext passwords and replacement with environmental variable values * Debugging message commit --------- Co-authored-by: ipatini Co-authored-by: Andreas Tsagkaropoulos --- .../discovery/registration/service/SALRegistrationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index a5b769a..3113101 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -144,7 +144,7 @@ public void register(Device device) { } //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to register device " + device_name + " is "+response.toString()); + log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); //} /* This is some realtime information, could be retrieved with a different call to the EMS. From bb4bdc27ab76de135bb9e922d819fd81533b01a3 Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 12:31:03 +0200 Subject: [PATCH 122/132] Work on Deregistration support --- .../discovery/ResourceDiscoveryProperties.java | 3 ++- .../broker_communication/SALCommunicator.java | 2 +- .../resource/discovery/monitor/model/Device.java | 5 +++-- .../registration/service/SALDeregistrationService.java | 9 ++++----- .../registration/service/SALRegistrationService.java | 10 ++++++++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 7ff3345..0702c7b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -67,7 +67,8 @@ public class ResourceDiscoveryProperties { private boolean salRegistrationEnabled = true; private long salRegistrationTimeout = 60*1000; private String registration_topic_name = "eu.nebulouscloud.exn.sal.node.create"; - + private String deregistration_topic_prefix = "eu.nebulouscloud.exn.sal.node.delete"; + // Failed devices detection private boolean automaticFailedDetection = true; private long suspectDeviceThreshold = 5; // in minutes diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index b7a598e..6787185 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -144,7 +144,7 @@ private static void register_devices(String request_body_file, String sessionID, public static String get_device_deregistration_json(Device device){ JSONObject root_json_object = new JSONObject(); - root_json_object.put("name",device.getRef()); + //root_json_object.put("name",device.getRef()); return root_json_object.toJSONString(); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index f3df47e..b1eafa8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -16,8 +16,9 @@ @NoArgsConstructor @Document(collection = "device") public class Device { - private String id; - private String ref; + private String id; //This is the internal id of the device in the Resource Discovery component + private String sal_id; //This identifier is used by SAL, and is used for deregistration + private String ref; //This identifier is used by the Cloud Fog Service Broker, to ascertain which devices are available for which applications private Double price; private String os; private String name; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java index 95099a2..cbad55c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -62,7 +62,7 @@ public void deregister(Device device) { if (processorProperties.isDeregistration_emulated()){ return; } - SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(device.getSal_id()), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (deregister_device_publisher.is_publisher_null()) { if (sending_attempt <= 2) { @@ -81,7 +81,7 @@ public void deregister(Device device) { //TODO handle the response here if (deregister_device_message_string!=null && !deregister_device_message_string.isEmpty()) { Map response = deregister_device_publisher.publish_for_response(deregister_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); + log.warn("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); }else{ log.warn("Deregistration was to be initiated with an empty deregistration payload"); } @@ -100,9 +100,8 @@ public void deregister(Device device) { } - private String get_deregistration_topic_name(String application_name) { - return processorProperties.getRegistration_topic_name(); - //return ("eu.nebulouscloud.exn.sal.edge." + application_name); + private String get_deregistration_topic_name(String sal_id) { + return processorProperties.getDeregistration_topic_prefix()+"."+sal_id; } @Override diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 3113101..23bfd2c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -8,6 +8,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONObject; import org.springframework.beans.factory.InitializingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.task.TaskExecutor; @@ -37,7 +38,7 @@ public void queueForRegistration(@NonNull Device device) { queue.add(device); } - public void register(Device device) { + public String register(Device device) { String application_name = device.getRef().split("\\|")[1]; @@ -145,6 +146,10 @@ public void register(Device device) { //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); + JSONObject response_json = new JSONObject(response); + JSONObject response_json_body = (JSONObject) response_json.get("body"); + String device_id = (String) response_json_body.get("id"); + return device_id; //} /* This is some realtime information, could be retrieved with a different call to the EMS. @@ -197,7 +202,8 @@ public void processQueue() { device = queue.take(); log.warn("SALRegistrationService: processQueue(): Will register device: {}", device); lastRegistrationStartTimestamp = System.currentTimeMillis(); - register(device); + String device_sal_id = register(device); + device.setSal_id(device_sal_id); lastRegistrationStartTimestamp = -1L; device.setRegisteredToSAL(true); deviceManagementService.update(device); From 0663266d64e93991f02149573a2b740c76bd7057 Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 14:56:01 +0200 Subject: [PATCH 123/132] Added more logging statements to debug device registration (work by ipatini) --- .../monitor/controller/DeviceManagementController.java | 1 + .../monitor/service/UnknownDeviceRegistrationService.java | 2 +- .../registration/RegistrationRequestProcessor.java | 7 +++++++ .../registration/service/SALRegistrationService.java | 5 +++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 48dbc45..8755e78 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -91,6 +91,7 @@ public Device getDeviceByIpAddress(@PathVariable String ipAddress) { @PutMapping(value = "/device", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public Device createDevice(@RequestBody Device device) { + log.warn("DeviceManagementController: createDevice: device: {}", device); salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.register(device)); return deviceService.save(device); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index b544fcc..65fe627 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -253,7 +253,7 @@ private void processDeviceDetailsResponses() { log.info("UnknownDeviceRegistrationService: Registered device: {}", newDevice); if (salRegistrationService.isPresent()) { - log.info("Registering the device {} to SAL...", newDevice); + log.info("Registering the device to SAL: {}", newDevice); salRegistrationService.get().register(newDevice); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index df5e583..680059c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -240,6 +240,7 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); + log.warn("RegistrationRequestProcessor: processResponse: request: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -288,6 +289,7 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {} {}", obj==null?null:obj.getClass().getTypeName(), obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -297,12 +299,14 @@ private void processResponse(@NonNull Map response) { boolean allowAllKeys = processorProperties.getAllowedDeviceInfoKeys().contains("*"); final Map processedDevInfo = new LinkedHashMap<>(); devInfo.forEach((key, value) -> { + log.warn("RegistrationRequestProcessor: processResponse: Dev-info pair: {} = {}", key, value); if (key!=null && value!=null) { String k = key.toString().trim(); String v = value.toString().trim(); if (StringUtils.isNotBlank(k) && StringUtils.isNotBlank(v)) { if (allowAllKeys || processorProperties.getAllowedDeviceInfoKeys().contains(k.toUpperCase())) { processedDevInfo.put(k, v); + log.warn("RegistrationRequestProcessor: processResponse: Dev-info pair ADDED: {} = {}", key, value); } else { log.debug("processResponse: Not allowed device info key for request: id={}, key={}", requestId, k); } @@ -364,6 +368,8 @@ private RegistrationRequestStatus getNextStatus(RegistrationRequestStatus currSt } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: BEGIN: request: {}", registrationRequest); + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: request-DEVICE: {}", registrationRequest.getDevice()); Device device = objectMapper.convertValue(registrationRequest.getDevice(), Device.class); // override values device.setId(null); @@ -379,6 +385,7 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { device.setRequestId(registrationRequest.getId()); device.setNodeReference(registrationRequest.getNodeReference()); deviceManagementService.save(device); + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: COPIED-DEVICE: {}", device); if (processorProperties.isSalRegistrationEnabled()) salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.queueForRegistration(device)); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 23bfd2c..c5b5f91 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -39,7 +39,7 @@ public void queueForRegistration(@NonNull Device device) { } public String register(Device device) { - + log.warn("SALRegistrationService: register: BEGIN: device: {}", device); String application_name = device.getRef().split("\\|")[1]; if (application_name.equals("all_applications")){ @@ -47,11 +47,12 @@ public String register(Device device) { } String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip==null || public_ip.isEmpty()){ - log.warn("Using default IP address ("+processorProperties.getNebulous_server_ip_address()+") to fetch Proactive client jar files from, as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); + log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip); } Map device_info = device.getDeviceInfo(); + log.warn("SALRegistrationService: register: DEVICE-INFO: {}", device_info); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS echo CPU_CORES=$TMP_NUM_CORES From 4f0932026a47882709d96e3c14d1f9976f7cc01e Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 14:58:06 +0200 Subject: [PATCH 124/132] Logging improvements (#33) * RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. * RD: Moved 'metrics' section side-by-side to 'device info' section in device-view.html, for better viewing * RD: changed mongo db name to 'resource_discovery' in application.yml * RD: Minor GUI improvements * RD: Added DeviceMonitorService to watch the status updates from EMS. Added DeviceStatusUpdate class used in DeviceMonitorService, and added field statusUpdate in Device class. Restructured and improved ResourceDiscoveryProperties. * RD: Created AbstractMonitorService (to factor out common topic monitoring code). Renamed DeviceMonitorService to DeviceStatusMonitorService and made it subclass of AbstractMonitorService. Changed DeviceStatusUpdate class to ignore unknown properties during deserialization. * RD: Added banner * RD: Added DeviceMetricsMonitorService and DeviceMetrics classes. Updated Device class to store device metrics received from EMS * RD: Updated application.yml * RD: Added device metrics display in device-view.html. Added (EMS) device status in devices.html * RD: Updated device-view.html to display device metrics * RD: Modified Device in monitoring subsystem to not have its credentials serialized (and returned to browser). Also updated the affected code * RD: Added check for a request's device IP address being in use (in another request or a registered device) * RD: various minor code improvements * RD: Added better error reporting in request-edit.html * RD: Various small GUI improvements and a fix * RD: Fixed registration Device in order toString() not to print credentials. Fixed RegistrationRequest in order to have its 'messages' and 'history' fields copied by BeanUtils. Fixed 2 bugs in RegistrationRequestProcessor and added a few more logs. * RD: Minor change in device-view.html to move 'count-' metrics to the end of the device metrics list * RD: Changed monitoring Device to not serialize credentials * RD: Code cleanup in RegistrationRequestProcessor * RD: Improved GUI (added page shortcuts at top-right) * RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] * RD: Added DeviceLifeCycleRequestService and DeviceLifeCycleResponseService classes for handling reinstall and uninstall actions (via EMS broker). Moved LC request event sending from DeviceManagementService to DeviceLifeCycleRequestService. Updated ResourceDiscoveryProperties with settings for DeviceLifeCycleRequest/ResponseService's. * RD: Added device archiving and unarchiving in GUI * RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. * RD: Changed devices.html to always display 'Archive' button to Admin users * RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html * RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. * RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. * RD: Added shortcuts (top-right corner) to all detail pages * RD: Minor change in index.html (grayed out settings image) * RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. * RD: Minor code (mostly logging) and GUI improvements * RD: Added 'UnknownDeviceRegistrationService' class to monitor for unknown devices (based on IP address and reference in messages received from EMS), and registering them to RD. Added NODE_DETAILS in REQUEST_TYPE enum for messages sent to/received from EMS about acquiring detailed node info (including credentials). Added 'deviceInfoRequestsTopic' and 'deviceInfoReportsTopic' settings in ResourceDiscoveryProperties. * RD: Added prompting admin to provide device credentials when un-archiving a previously archived request or device. * RD: Moved REQUEST_TYPE enum to 'common' package and updated code * RD: Changed authentication to use BCrypt-encrypted passwords * RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. * RD: Minor code tidy up * RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml * RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does * RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version * RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. * RD: Added StatusController to report current application status (currently returns 'OK') * RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly * RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version * RD: Renamed 'management' module to 'resource-discovery' * RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore * RD: Modified base images in Dockerfile * RD: Modified Dockerfile in order to both compile the app and build the Docker image * RD: Improved Dockerfile * RD: Improved Dockerfile * RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. * RD: Fixed Dockerfile * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * EMS: Added K8sNetdataCollector [WIP] * RD: Added 'port' in forms and models * RD: Added two TODOs * RD: Deactivated UnknownDeviceRegistrationService * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 * Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 * Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 * Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 * Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 * Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml * RD: Fixed banner.txt * RD: Modified application.yml * RD: Added temp. debug messages * RD: Added temp. debug messages 2 * Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. * Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. * RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. * RD: Added temp. log messages * Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. * RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) * Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 * Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Updated Dockerfile * RD: Updated run.sh and added wait_for_mongodb.sh * RD: Improved wait_for_mongodb.sh script * Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 * Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 * Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' * RD: Added API Key authentication * RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. * RD: Code improvements (esp. wrt @PreAuthorize annotations) * RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. * RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. * RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). * RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. * Use device reference instead of name to register to SAL * Improvements in device registration and component communication with the broker * Small improvements in device registration * Small improvement in getting device registration details * Small improvement in getting device registration details * RD: Fix in SALRegistrationService class * RD: Fixed fontawesome cdn url * Initial deregistration support * RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) * RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS * Improvements in the handling of AMQP connections * Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher * Minor logging improvements * Deregistration process improvement * Stopping the device lost publisher * Refactoring to use the original Connector class instead of ExtendedConnector * Minor logging improvement * Only try to stop the connector if it has been previously initialized * Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized * Password improvements Removal of plaintext passwords and replacement with environmental variable values * Debugging message commit * Work on Deregistration support * Added more logging statements to debug device registration (work by ipatini) --------- Co-authored-by: ipatini Co-authored-by: Andreas Tsagkaropoulos --- .../discovery/ResourceDiscoveryProperties.java | 3 ++- .../broker_communication/SALCommunicator.java | 2 +- .../controller/DeviceManagementController.java | 1 + .../resource/discovery/monitor/model/Device.java | 5 +++-- .../service/UnknownDeviceRegistrationService.java | 2 +- .../RegistrationRequestProcessor.java | 7 +++++++ .../service/SALDeregistrationService.java | 9 ++++----- .../service/SALRegistrationService.java | 15 +++++++++++---- 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java index 7ff3345..0702c7b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/ResourceDiscoveryProperties.java @@ -67,7 +67,8 @@ public class ResourceDiscoveryProperties { private boolean salRegistrationEnabled = true; private long salRegistrationTimeout = 60*1000; private String registration_topic_name = "eu.nebulouscloud.exn.sal.node.create"; - + private String deregistration_topic_prefix = "eu.nebulouscloud.exn.sal.node.delete"; + // Failed devices detection private boolean automaticFailedDetection = true; private long suspectDeviceThreshold = 5; // in minutes diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java index b7a598e..6787185 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/SALCommunicator.java @@ -144,7 +144,7 @@ private static void register_devices(String request_body_file, String sessionID, public static String get_device_deregistration_json(Device device){ JSONObject root_json_object = new JSONObject(); - root_json_object.put("name",device.getRef()); + //root_json_object.put("name",device.getRef()); return root_json_object.toJSONString(); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java index 48dbc45..8755e78 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/controller/DeviceManagementController.java @@ -91,6 +91,7 @@ public Device getDeviceByIpAddress(@PathVariable String ipAddress) { @PutMapping(value = "/device", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public Device createDevice(@RequestBody Device device) { + log.warn("DeviceManagementController: createDevice: device: {}", device); salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.register(device)); return deviceService.save(device); diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java index f3df47e..b1eafa8 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/model/Device.java @@ -16,8 +16,9 @@ @NoArgsConstructor @Document(collection = "device") public class Device { - private String id; - private String ref; + private String id; //This is the internal id of the device in the Resource Discovery component + private String sal_id; //This identifier is used by SAL, and is used for deregistration + private String ref; //This identifier is used by the Cloud Fog Service Broker, to ascertain which devices are available for which applications private Double price; private String os; private String name; diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java index b544fcc..65fe627 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/service/UnknownDeviceRegistrationService.java @@ -253,7 +253,7 @@ private void processDeviceDetailsResponses() { log.info("UnknownDeviceRegistrationService: Registered device: {}", newDevice); if (salRegistrationService.isPresent()) { - log.info("Registering the device {} to SAL...", newDevice); + log.info("Registering the device to SAL: {}", newDevice); salRegistrationService.get().register(newDevice); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index df5e583..680059c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -240,6 +240,7 @@ private void processResponse(@NonNull Map response) { long timestamp = Long.parseLong(response.getOrDefault("timestamp", "-1").toString().trim()); RegistrationRequest registrationRequest = registrationRequestService.getById(requestId).orElse(null); + log.warn("RegistrationRequestProcessor: processResponse: request: {}", registrationRequest); if (registrationRequest!=null) { RegistrationRequestStatus currStatus = registrationRequest.getStatus(); RegistrationRequestStatus newStatus = switch (currStatus) { @@ -288,6 +289,7 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {} {}", obj==null?null:obj.getClass().getTypeName(), obj); if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); @@ -297,12 +299,14 @@ private void processResponse(@NonNull Map response) { boolean allowAllKeys = processorProperties.getAllowedDeviceInfoKeys().contains("*"); final Map processedDevInfo = new LinkedHashMap<>(); devInfo.forEach((key, value) -> { + log.warn("RegistrationRequestProcessor: processResponse: Dev-info pair: {} = {}", key, value); if (key!=null && value!=null) { String k = key.toString().trim(); String v = value.toString().trim(); if (StringUtils.isNotBlank(k) && StringUtils.isNotBlank(v)) { if (allowAllKeys || processorProperties.getAllowedDeviceInfoKeys().contains(k.toUpperCase())) { processedDevInfo.put(k, v); + log.warn("RegistrationRequestProcessor: processResponse: Dev-info pair ADDED: {} = {}", key, value); } else { log.debug("processResponse: Not allowed device info key for request: id={}, key={}", requestId, k); } @@ -364,6 +368,8 @@ private RegistrationRequestStatus getNextStatus(RegistrationRequestStatus currSt } private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: BEGIN: request: {}", registrationRequest); + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: request-DEVICE: {}", registrationRequest.getDevice()); Device device = objectMapper.convertValue(registrationRequest.getDevice(), Device.class); // override values device.setId(null); @@ -379,6 +385,7 @@ private void copyDeviceToMonitoring(RegistrationRequest registrationRequest) { device.setRequestId(registrationRequest.getId()); device.setNodeReference(registrationRequest.getNodeReference()); deviceManagementService.save(device); + log.warn("RegistrationRequestProcessor: copyDeviceToMonitoring: COPIED-DEVICE: {}", device); if (processorProperties.isSalRegistrationEnabled()) salRegistrationService.ifPresent(salRegistrationService -> salRegistrationService.queueForRegistration(device)); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java index 95099a2..cbad55c 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALDeregistrationService.java @@ -62,7 +62,7 @@ public void deregister(Device device) { if (processorProperties.isDeregistration_emulated()){ return; } - SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(application_name), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + SynchronousBrokerPublisher deregister_device_publisher = new SynchronousBrokerPublisher(get_deregistration_topic_name(device.getSal_id()), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (deregister_device_publisher.is_publisher_null()) { if (sending_attempt <= 2) { @@ -81,7 +81,7 @@ public void deregister(Device device) { //TODO handle the response here if (deregister_device_message_string!=null && !deregister_device_message_string.isEmpty()) { Map response = deregister_device_publisher.publish_for_response(deregister_device_message_string, Collections.singleton(application_name)); - log.info("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); + log.warn("The response received while trying to deregister device " + device.getRef() + " is " + response.toString()); }else{ log.warn("Deregistration was to be initiated with an empty deregistration payload"); } @@ -100,9 +100,8 @@ public void deregister(Device device) { } - private String get_deregistration_topic_name(String application_name) { - return processorProperties.getRegistration_topic_name(); - //return ("eu.nebulouscloud.exn.sal.edge." + application_name); + private String get_deregistration_topic_name(String sal_id) { + return processorProperties.getDeregistration_topic_prefix()+"."+sal_id; } @Override diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index 3113101..c5b5f91 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -8,6 +8,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONObject; import org.springframework.beans.factory.InitializingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.task.TaskExecutor; @@ -37,8 +38,8 @@ public void queueForRegistration(@NonNull Device device) { queue.add(device); } - public void register(Device device) { - + public String register(Device device) { + log.warn("SALRegistrationService: register: BEGIN: device: {}", device); String application_name = device.getRef().split("\\|")[1]; if (application_name.equals("all_applications")){ @@ -46,11 +47,12 @@ public void register(Device device) { } String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip==null || public_ip.isEmpty()){ - log.warn("Using default IP address ("+processorProperties.getNebulous_server_ip_address()+") to fetch Proactive client jar files from, as the environmental variable was not set or found"); public_ip = processorProperties.getNebulous_server_ip_address(); + log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip); } Map device_info = device.getDeviceInfo(); + log.warn("SALRegistrationService: register: DEVICE-INFO: {}", device_info); /* Information available from the EMS, based on https://gitlab.com/nebulous-project/ems-main/-/blob/master/ems-core/bin/detect.sh?ref_type=heads echo CPU_SOCKETS=$TMP_NUM_CPUS echo CPU_CORES=$TMP_NUM_CORES @@ -145,6 +147,10 @@ public void register(Device device) { //TODO handle the response here Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); + JSONObject response_json = new JSONObject(response); + JSONObject response_json_body = (JSONObject) response_json.get("body"); + String device_id = (String) response_json_body.get("id"); + return device_id; //} /* This is some realtime information, could be retrieved with a different call to the EMS. @@ -197,7 +203,8 @@ public void processQueue() { device = queue.take(); log.warn("SALRegistrationService: processQueue(): Will register device: {}", device); lastRegistrationStartTimestamp = System.currentTimeMillis(); - register(device); + String device_sal_id = register(device); + device.setSal_id(device_sal_id); lastRegistrationStartTimestamp = -1L; device.setRegisteredToSAL(true); deviceManagementService.update(device); From 57ca53b7384b44abf82fbe2423d127800c5f4f6f Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 15:17:29 +0200 Subject: [PATCH 125/132] Fix parsing the json response of SAL --- .../registration/service/SALRegistrationService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index c5b5f91..a35912b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -9,6 +9,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; import org.springframework.beans.factory.InitializingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.task.TaskExecutor; @@ -148,7 +150,13 @@ public String register(Device device) { Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); JSONObject response_json = new JSONObject(response); - JSONObject response_json_body = (JSONObject) response_json.get("body"); + JSONParser json_parser = new JSONParser(); + JSONObject response_json_body = null; + try { + response_json_body = (JSONObject) json_parser.parse(String.valueOf(response_json.get("body"))); + } catch (ParseException e) { + throw new RuntimeException(e); + } String device_id = (String) response_json_body.get("id"); return device_id; //} From d1de53300717796141ef32ed67612890fc30ee06 Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 15:22:07 +0200 Subject: [PATCH 126/132] Sal response parsing improvement (#34) * RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. * RD: Moved 'metrics' section side-by-side to 'device info' section in device-view.html, for better viewing * RD: changed mongo db name to 'resource_discovery' in application.yml * RD: Minor GUI improvements * RD: Added DeviceMonitorService to watch the status updates from EMS. Added DeviceStatusUpdate class used in DeviceMonitorService, and added field statusUpdate in Device class. Restructured and improved ResourceDiscoveryProperties. * RD: Created AbstractMonitorService (to factor out common topic monitoring code). Renamed DeviceMonitorService to DeviceStatusMonitorService and made it subclass of AbstractMonitorService. Changed DeviceStatusUpdate class to ignore unknown properties during deserialization. * RD: Added banner * RD: Added DeviceMetricsMonitorService and DeviceMetrics classes. Updated Device class to store device metrics received from EMS * RD: Updated application.yml * RD: Added device metrics display in device-view.html. Added (EMS) device status in devices.html * RD: Updated device-view.html to display device metrics * RD: Modified Device in monitoring subsystem to not have its credentials serialized (and returned to browser). Also updated the affected code * RD: Added check for a request's device IP address being in use (in another request or a registered device) * RD: various minor code improvements * RD: Added better error reporting in request-edit.html * RD: Various small GUI improvements and a fix * RD: Fixed registration Device in order toString() not to print credentials. Fixed RegistrationRequest in order to have its 'messages' and 'history' fields copied by BeanUtils. Fixed 2 bugs in RegistrationRequestProcessor and added a few more logs. * RD: Minor change in device-view.html to move 'count-' metrics to the end of the device metrics list * RD: Changed monitoring Device to not serialize credentials * RD: Code cleanup in RegistrationRequestProcessor * RD: Improved GUI (added page shortcuts at top-right) * RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] * RD: Added DeviceLifeCycleRequestService and DeviceLifeCycleResponseService classes for handling reinstall and uninstall actions (via EMS broker). Moved LC request event sending from DeviceManagementService to DeviceLifeCycleRequestService. Updated ResourceDiscoveryProperties with settings for DeviceLifeCycleRequest/ResponseService's. * RD: Added device archiving and unarchiving in GUI * RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. * RD: Changed devices.html to always display 'Archive' button to Admin users * RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html * RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. * RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. * RD: Added shortcuts (top-right corner) to all detail pages * RD: Minor change in index.html (grayed out settings image) * RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. * RD: Minor code (mostly logging) and GUI improvements * RD: Added 'UnknownDeviceRegistrationService' class to monitor for unknown devices (based on IP address and reference in messages received from EMS), and registering them to RD. Added NODE_DETAILS in REQUEST_TYPE enum for messages sent to/received from EMS about acquiring detailed node info (including credentials). Added 'deviceInfoRequestsTopic' and 'deviceInfoReportsTopic' settings in ResourceDiscoveryProperties. * RD: Added prompting admin to provide device credentials when un-archiving a previously archived request or device. * RD: Moved REQUEST_TYPE enum to 'common' package and updated code * RD: Changed authentication to use BCrypt-encrypted passwords * RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. * RD: Minor code tidy up * RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml * RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does * RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version * RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. * RD: Added StatusController to report current application status (currently returns 'OK') * RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly * RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version * RD: Renamed 'management' module to 'resource-discovery' * RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore * RD: Modified base images in Dockerfile * RD: Modified Dockerfile in order to both compile the app and build the Docker image * RD: Improved Dockerfile * RD: Improved Dockerfile * RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. * RD: Fixed Dockerfile * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * EMS: Added K8sNetdataCollector [WIP] * RD: Added 'port' in forms and models * RD: Added two TODOs * RD: Deactivated UnknownDeviceRegistrationService * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 * Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 * Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 * Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 * Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 * Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml * RD: Fixed banner.txt * RD: Modified application.yml * RD: Added temp. debug messages * RD: Added temp. debug messages 2 * Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. * Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. * RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. * RD: Added temp. log messages * Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. * RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) * Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 * Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Updated Dockerfile * RD: Updated run.sh and added wait_for_mongodb.sh * RD: Improved wait_for_mongodb.sh script * Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 * Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 * Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' * RD: Added API Key authentication * RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. * RD: Code improvements (esp. wrt @PreAuthorize annotations) * RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. * RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. * RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). * RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. * Use device reference instead of name to register to SAL * Improvements in device registration and component communication with the broker * Small improvements in device registration * Small improvement in getting device registration details * Small improvement in getting device registration details * RD: Fix in SALRegistrationService class * RD: Fixed fontawesome cdn url * Initial deregistration support * RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) * RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS * Improvements in the handling of AMQP connections * Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher * Minor logging improvements * Deregistration process improvement * Stopping the device lost publisher * Refactoring to use the original Connector class instead of ExtendedConnector * Minor logging improvement * Only try to stop the connector if it has been previously initialized * Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized * Password improvements Removal of plaintext passwords and replacement with environmental variable values * Debugging message commit * Work on Deregistration support * Added more logging statements to debug device registration (work by ipatini) * Fix parsing the json response of SAL --------- Co-authored-by: ipatini Co-authored-by: Andreas Tsagkaropoulos --- .../registration/service/SALRegistrationService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index c5b5f91..a35912b 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -9,6 +9,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; import org.springframework.beans.factory.InitializingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.task.TaskExecutor; @@ -148,7 +150,13 @@ public String register(Device device) { Map response = register_device_publisher.publish_for_response(register_device_message_string, Collections.singleton(application_name)); log.warn("The response received while trying to register device " + device_name + " is "+response.toString()); JSONObject response_json = new JSONObject(response); - JSONObject response_json_body = (JSONObject) response_json.get("body"); + JSONParser json_parser = new JSONParser(); + JSONObject response_json_body = null; + try { + response_json_body = (JSONObject) json_parser.parse(String.valueOf(response_json.get("body"))); + } catch (ParseException e) { + throw new RuntimeException(e); + } String device_id = (String) response_json_body.get("id"); return device_id; //} From 2c3acd7c2ec285c16e73e984a1624b3eb243879f Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 18:09:35 +0200 Subject: [PATCH 127/132] Changes to avoid stopping the connector --- .../broker_communication/BrokerPublisher.java | 38 +++++++++++-------- .../discovery/monitor/DeviceProcessor.java | 5 ++- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 6863080..42a59ee 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -63,7 +63,11 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b throw new RuntimeException(e); } } - active_connector.stop(); + try { + active_connector.stop(); //TODO reassure expected stop() functionality is working here when this is necessary + }catch (Exception e){ + e.printStackTrace(); + } } } @@ -104,23 +108,27 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b //TODO The methods below assume that the only content to be sent is json-like public void publish (String json_string_content, Collection application_names){ - for (String application_name : application_names) { - JSONParser parser = new JSONParser(); - JSONObject json_object = new JSONObject(); - try { - json_object = (JSONObject) parser.parse(json_string_content); - } catch (ParseException p) { - log.warn( "publish: Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); - } - if (!is_publisher_null()) { - private_publisher_instance.send(json_object); - log.info("Sent new message\n"+json_object.toJSONString()); - } else { - log.error( "Could not send message to AMQP broker, as the publisher instance is null"); - } + publish(json_string_content); } } + + public void publish (String json_string_content){ + JSONParser parser = new JSONParser(); + JSONObject json_object = new JSONObject(); + try { + json_object = (JSONObject) parser.parse(json_string_content); + } catch (ParseException p) { + log.warn( "publish: Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); + } + if (!is_publisher_null()) { + private_publisher_instance.send(json_object); + log.info("Sent new message\n"+json_object.toJSONString()); + } else { + log.error( "Could not send message to AMQP broker, as the publisher instance is null"); + } + } + public boolean is_publisher_null(){ return (private_publisher_instance == null); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 2ea34c3..5fbdcb3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -49,6 +49,7 @@ public class DeviceProcessor implements InitializingBean { private final TaskScheduler taskScheduler; private final AtomicBoolean isRunning = new AtomicBoolean(false); private final Optional salDeregistrationService; + private BrokerPublisher device_lost_publisher; @Override @@ -145,7 +146,7 @@ private void processFailedDevices() { Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); log.info("Creating new BrokerPublisher to publish device lost message"); - BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (device_lost_publisher.is_publisher_null()){ @@ -163,7 +164,7 @@ private void processFailedDevices() { } device_lost_publisher.publish(lost_device_message.toJSONString(), Collections.singleton("")); log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); - device_lost_publisher.stop(); + //device_lost_publisher.stop(); } deviceManagementService.update(device); From 77625ca8b5fddaf6b34fce2f007c7d049388ce1d Mon Sep 17 00:00:00 2001 From: atsag Date: Thu, 21 Nov 2024 18:10:53 +0200 Subject: [PATCH 128/132] Avoid publish connector stop (#36) * RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. * RD: Moved 'metrics' section side-by-side to 'device info' section in device-view.html, for better viewing * RD: changed mongo db name to 'resource_discovery' in application.yml * RD: Minor GUI improvements * RD: Added DeviceMonitorService to watch the status updates from EMS. Added DeviceStatusUpdate class used in DeviceMonitorService, and added field statusUpdate in Device class. Restructured and improved ResourceDiscoveryProperties. * RD: Created AbstractMonitorService (to factor out common topic monitoring code). Renamed DeviceMonitorService to DeviceStatusMonitorService and made it subclass of AbstractMonitorService. Changed DeviceStatusUpdate class to ignore unknown properties during deserialization. * RD: Added banner * RD: Added DeviceMetricsMonitorService and DeviceMetrics classes. Updated Device class to store device metrics received from EMS * RD: Updated application.yml * RD: Added device metrics display in device-view.html. Added (EMS) device status in devices.html * RD: Updated device-view.html to display device metrics * RD: Modified Device in monitoring subsystem to not have its credentials serialized (and returned to browser). Also updated the affected code * RD: Added check for a request's device IP address being in use (in another request or a registered device) * RD: various minor code improvements * RD: Added better error reporting in request-edit.html * RD: Various small GUI improvements and a fix * RD: Fixed registration Device in order toString() not to print credentials. Fixed RegistrationRequest in order to have its 'messages' and 'history' fields copied by BeanUtils. Fixed 2 bugs in RegistrationRequestProcessor and added a few more logs. * RD: Minor change in device-view.html to move 'count-' metrics to the end of the device metrics list * RD: Changed monitoring Device to not serialize credentials * RD: Code cleanup in RegistrationRequestProcessor * RD: Improved GUI (added page shortcuts at top-right) * RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] * RD: Added DeviceLifeCycleRequestService and DeviceLifeCycleResponseService classes for handling reinstall and uninstall actions (via EMS broker). Moved LC request event sending from DeviceManagementService to DeviceLifeCycleRequestService. Updated ResourceDiscoveryProperties with settings for DeviceLifeCycleRequest/ResponseService's. * RD: Added device archiving and unarchiving in GUI * RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. * RD: Changed devices.html to always display 'Archive' button to Admin users * RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html * RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. * RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. * RD: Added shortcuts (top-right corner) to all detail pages * RD: Minor change in index.html (grayed out settings image) * RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. * RD: Minor code (mostly logging) and GUI improvements * RD: Added 'UnknownDeviceRegistrationService' class to monitor for unknown devices (based on IP address and reference in messages received from EMS), and registering them to RD. Added NODE_DETAILS in REQUEST_TYPE enum for messages sent to/received from EMS about acquiring detailed node info (including credentials). Added 'deviceInfoRequestsTopic' and 'deviceInfoReportsTopic' settings in ResourceDiscoveryProperties. * RD: Added prompting admin to provide device credentials when un-archiving a previously archived request or device. * RD: Moved REQUEST_TYPE enum to 'common' package and updated code * RD: Changed authentication to use BCrypt-encrypted passwords * RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. * RD: Minor code tidy up * RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml * RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does * RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version * RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. * RD: Added StatusController to report current application status (currently returns 'OK') * RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly * RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version * RD: Renamed 'management' module to 'resource-discovery' * RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore * RD: Modified base images in Dockerfile * RD: Modified Dockerfile in order to both compile the app and build the Docker image * RD: Improved Dockerfile * RD: Improved Dockerfile * RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. * RD: Fixed Dockerfile * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * EMS: Added K8sNetdataCollector [WIP] * RD: Added 'port' in forms and models * RD: Added two TODOs * RD: Deactivated UnknownDeviceRegistrationService * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 * Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 * Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 * Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 * Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 * Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml * RD: Fixed banner.txt * RD: Modified application.yml * RD: Added temp. debug messages * RD: Added temp. debug messages 2 * Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. * Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. * RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. * RD: Added temp. log messages * Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. * RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) * Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 * Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Updated Dockerfile * RD: Updated run.sh and added wait_for_mongodb.sh * RD: Improved wait_for_mongodb.sh script * Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 * Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 * Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' * RD: Added API Key authentication * RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. * RD: Code improvements (esp. wrt @PreAuthorize annotations) * RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. * RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. * RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). * RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. * Use device reference instead of name to register to SAL * Improvements in device registration and component communication with the broker * Small improvements in device registration * Small improvement in getting device registration details * Small improvement in getting device registration details * RD: Fix in SALRegistrationService class * RD: Fixed fontawesome cdn url * Initial deregistration support * RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) * RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS * Improvements in the handling of AMQP connections * Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher * Minor logging improvements * Deregistration process improvement * Stopping the device lost publisher * Refactoring to use the original Connector class instead of ExtendedConnector * Minor logging improvement * Only try to stop the connector if it has been previously initialized * Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized * Password improvements Removal of plaintext passwords and replacement with environmental variable values * Debugging message commit * Work on Deregistration support * Added more logging statements to debug device registration (work by ipatini) * Fix parsing the json response of SAL * Changes to avoid stopping the connector --------- Co-authored-by: ipatini Co-authored-by: Andreas Tsagkaropoulos --- .../broker_communication/BrokerPublisher.java | 38 +++++++++++-------- .../discovery/monitor/DeviceProcessor.java | 5 ++- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java index 6863080..42a59ee 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/broker_communication/BrokerPublisher.java @@ -63,7 +63,11 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b throw new RuntimeException(e); } } - active_connector.stop(); + try { + active_connector.stop(); //TODO reassure expected stop() functionality is working here when this is necessary + }catch (Exception e){ + e.printStackTrace(); + } } } @@ -104,23 +108,27 @@ public BrokerPublisher(String topic, String broker_ip, int broker_port, String b //TODO The methods below assume that the only content to be sent is json-like public void publish (String json_string_content, Collection application_names){ - for (String application_name : application_names) { - JSONParser parser = new JSONParser(); - JSONObject json_object = new JSONObject(); - try { - json_object = (JSONObject) parser.parse(json_string_content); - } catch (ParseException p) { - log.warn( "publish: Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); - } - if (!is_publisher_null()) { - private_publisher_instance.send(json_object); - log.info("Sent new message\n"+json_object.toJSONString()); - } else { - log.error( "Could not send message to AMQP broker, as the publisher instance is null"); - } + publish(json_string_content); } } + + public void publish (String json_string_content){ + JSONParser parser = new JSONParser(); + JSONObject json_object = new JSONObject(); + try { + json_object = (JSONObject) parser.parse(json_string_content); + } catch (ParseException p) { + log.warn( "publish: Could not parse the string content to be published to the broker as json, which is the following: "+json_string_content); + } + if (!is_publisher_null()) { + private_publisher_instance.send(json_object); + log.info("Sent new message\n"+json_object.toJSONString()); + } else { + log.error( "Could not send message to AMQP broker, as the publisher instance is null"); + } + } + public boolean is_publisher_null(){ return (private_publisher_instance == null); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 2ea34c3..5fbdcb3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -49,6 +49,7 @@ public class DeviceProcessor implements InitializingBean { private final TaskScheduler taskScheduler; private final AtomicBoolean isRunning = new AtomicBoolean(false); private final Optional salDeregistrationService; + private BrokerPublisher device_lost_publisher; @Override @@ -145,7 +146,7 @@ private void processFailedDevices() { Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); log.info("Creating new BrokerPublisher to publish device lost message"); - BrokerPublisher device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (device_lost_publisher.is_publisher_null()){ @@ -163,7 +164,7 @@ private void processFailedDevices() { } device_lost_publisher.publish(lost_device_message.toJSONString(), Collections.singleton("")); log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); - device_lost_publisher.stop(); + //device_lost_publisher.stop(); } deviceManagementService.update(device); From cae220b100374739731533efbc5bf3009d5564e7 Mon Sep 17 00:00:00 2001 From: atsag Date: Fri, 22 Nov 2024 11:55:27 +0200 Subject: [PATCH 129/132] Avoid starting new connectors, mark device as offboarded When a device has failed and it has been processed (i.e a message has been sent to the device lost topic) then it should be marked as offboarded to be archived Do not create a new connector unless this is required Do not use a redundant environmental variable --- .../resource/discovery/monitor/DeviceProcessor.java | 6 +++--- .../registration/service/SALRegistrationService.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 5fbdcb3..3dd0618 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -145,8 +145,8 @@ private void processFailedDevices() { lost_device_message.put("device_name",device.getName()); Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); - log.info("Creating new BrokerPublisher to publish device lost message"); - device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + log.info("Trying to use existing BrokerPublisher to publish device lost message"); + //device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (device_lost_publisher.is_publisher_null()){ @@ -166,7 +166,7 @@ private void processFailedDevices() { log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); //device_lost_publisher.stop(); } - + device.setStatus(DeviceStatus.OFFBOARDED); deviceManagementService.update(device); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index a35912b..ef28ba3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -47,11 +47,11 @@ public String register(Device device) { if (application_name.equals("all_applications")){ application_name=""; } - String public_ip = System.getenv("NEBULOUS_IP"); + /*String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip==null || public_ip.isEmpty()){ public_ip = processorProperties.getNebulous_server_ip_address(); - log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip); - } + log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip);}*/ + String public_ip = processorProperties.getNebulous_server_ip_address(); Map device_info = device.getDeviceInfo(); log.warn("SALRegistrationService: register: DEVICE-INFO: {}", device_info); From 1a02765fd9039884e8b5c69d2e3a3dc6135f5d6b Mon Sep 17 00:00:00 2001 From: atsag Date: Fri, 22 Nov 2024 11:58:26 +0200 Subject: [PATCH 130/132] Miscellaneous improvements (#37) * RD: Added device-view.html for viewing and editing device details. Fixed and improved devices.html. Modified DeviceManagementController so that the '/monitor/device' endpoint returns the devices of the current user, and also plain users can retrieve info for the devices they own. * RD: Moved 'metrics' section side-by-side to 'device info' section in device-view.html, for better viewing * RD: changed mongo db name to 'resource_discovery' in application.yml * RD: Minor GUI improvements * RD: Added DeviceMonitorService to watch the status updates from EMS. Added DeviceStatusUpdate class used in DeviceMonitorService, and added field statusUpdate in Device class. Restructured and improved ResourceDiscoveryProperties. * RD: Created AbstractMonitorService (to factor out common topic monitoring code). Renamed DeviceMonitorService to DeviceStatusMonitorService and made it subclass of AbstractMonitorService. Changed DeviceStatusUpdate class to ignore unknown properties during deserialization. * RD: Added banner * RD: Added DeviceMetricsMonitorService and DeviceMetrics classes. Updated Device class to store device metrics received from EMS * RD: Updated application.yml * RD: Added device metrics display in device-view.html. Added (EMS) device status in devices.html * RD: Updated device-view.html to display device metrics * RD: Modified Device in monitoring subsystem to not have its credentials serialized (and returned to browser). Also updated the affected code * RD: Added check for a request's device IP address being in use (in another request or a registered device) * RD: various minor code improvements * RD: Added better error reporting in request-edit.html * RD: Various small GUI improvements and a fix * RD: Fixed registration Device in order toString() not to print credentials. Fixed RegistrationRequest in order to have its 'messages' and 'history' fields copied by BeanUtils. Fixed 2 bugs in RegistrationRequestProcessor and added a few more logs. * RD: Minor change in device-view.html to move 'count-' metrics to the end of the device metrics list * RD: Changed monitoring Device to not serialize credentials * RD: Code cleanup in RegistrationRequestProcessor * RD: Improved GUI (added page shortcuts at top-right) * RD: Implementing re-onboarding, off-boarding, and request-for-info (device status and metrics) [WIP] * RD: Added DeviceLifeCycleRequestService and DeviceLifeCycleResponseService classes for handling reinstall and uninstall actions (via EMS broker). Moved LC request event sending from DeviceManagementService to DeviceLifeCycleRequestService. Updated ResourceDiscoveryProperties with settings for DeviceLifeCycleRequest/ResponseService's. * RD: Added device archiving and unarchiving in GUI * RD: Implemented DeviceProcessor to periodically archive off-boarded devices. Added configurable immediate archiving of Success registration requests, and off-boarded devices, and added relevant settings in ResourceDiscoveryProperties and in GUI. * RD: Changed devices.html to always display 'Archive' button to Admin users * RD: Added archived-device-view.html, renamed archived-view.html to archived-request-view.html. Updated archived.html * RD: Fixed DeviceManagementController to allow plain users (device owners) to access their device info (and archived devices), and control re-installing and off-boarding. Added device colouring based on their status. * RD: Extended DeviceProcessor to check for suspect and failed devices. Added SUSPECT and FAILED statues in DeviceStatus (and in devices.html and archived.html for colouring). Added relevant settings in ResourceDiscoveryProperties. Fixed a few naming errors. * RD: Added shortcuts (top-right corner) to all detail pages * RD: Minor change in index.html (grayed out settings image) * RD: Improved AbstractMonitorService (reuse of single connection, better logs). Updated affected code. * RD: Minor code (mostly logging) and GUI improvements * RD: Added 'UnknownDeviceRegistrationService' class to monitor for unknown devices (based on IP address and reference in messages received from EMS), and registering them to RD. Added NODE_DETAILS in REQUEST_TYPE enum for messages sent to/received from EMS about acquiring detailed node info (including credentials). Added 'deviceInfoRequestsTopic' and 'deviceInfoReportsTopic' settings in ResourceDiscoveryProperties. * RD: Added prompting admin to provide device credentials when un-archiving a previously archived request or device. * RD: Moved REQUEST_TYPE enum to 'common' package and updated code * RD: Changed authentication to use BCrypt-encrypted passwords * RD: Moved message broker communication code to BrokerUtil class, and updated using classes accordingly. Implemented EncryptionUtil for encrypting and decrypting broker messages [TO BE TESTED]. Few minor code improvements. * RD: Minor code tidy up * RD: Added support for SSL connections to ActiveMQ broker in BrokerUtil. Added the needed settings in ResourceDiscoveryProperties and application.yml * RD: Fixed issue where UnknownDeviceRegistrationService registers a device before RegistrationRequestProcessor does * RD: Modified pom.xml to build an image named 'eu.nebulous.resource-discovery' and tagged with current project version * RD: Added 'DeviceLocation' class and 'location' field in both 'Device' classes. Also updated all 4 details pages to include Location (name, lat, lon). Fixed the Restore button of the modal dialog in archived.html to display 'Restore Request' or 'Restore Device' depending on the restore type. * RD: Added StatusController to report current application status (currently returns 'OK') * RD: Fixed a few minor issues: * Fixed 'spring.web.resources.static-locations' * Changed Docker image name to 'resource-discovery' * Set BPE_LANG value to C.UTF-8 in order to render banner correctly * RD: Changes in pom.xml and application.yml: * Changed groupId in pom.xml to 'eu.nebulous.resource-management', and artifactId to 'resource-discovery' * Corrected service name and description * Set 'imageName' property to derive from artifactId * Fixed application.yml to use '@project.version@' placeholder for app version * RD: Renamed 'management' module to 'resource-discovery' * RD: * Commented out Buildpacks related settings in pom.xml * Added Dockerfile, run.sh (entrypoint script) and .dockerignore * RD: Modified base images in Dockerfile * RD: Modified Dockerfile in order to both compile the app and build the Docker image * RD: Improved Dockerfile * RD: Improved Dockerfile * RD: Upgraded SB version to SB 3.2.1 and JRE to version 21. Updated dependency to their latest versions. Modified model classes to avoid Lombok SuperBuilder warnings. * RD: Fixed Dockerfile * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * EMS: Added K8sNetdataCollector [WIP] * RD: Added 'port' in forms and models * RD: Added two TODOs * RD: Deactivated UnknownDeviceRegistrationService * Initial changes to support registration and deregistration of edge devices to SAL Introduction of Broker publisher and subscriber utilities Introduction of new device registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Improvement in the initialization of SALRegistrationService I94a6fdb4612de192c24511445f1236cdce94b000 * Increased logging to debug setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Modified logging to debug the setting of processorProperties I94a6fdb4612de192c24511445f1236cdce94b000 * Correction of syntactic error I94a6fdb4612de192c24511445f1236cdce94b000 * Addition of needed configuration properties I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Testing of alternative initialization of SAL registration service I94a6fdb4612de192c24511445f1236cdce94b000 * Updates on the topic and the payload format used to communicate with SAL for the registration of a device I94a6fdb4612de192c24511445f1236cdce94b000 * Allow setting and using a custom broker port I94a6fdb4612de192c24511445f1236cdce94b000 * Pass a default port to the configuration of the Resource discovery server I94a6fdb4612de192c24511445f1236cdce94b000 * Log debugging information related to the port of the NebulOuS broker I94a6fdb4612de192c24511445f1236cdce94b000 * Publishing of the appropriate message to the broker I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Removed truststore settings from application.yml. Upgraded to SB 3.2.4 and fixed pom.xml * RD: Fixed banner.txt * RD: Modified application.yml * RD: Added temp. debug messages * RD: Added temp. debug messages 2 * Revert "RD: Added temp. debug messages 2" This reverts commit 738f6048a8b88b4cfc7410db2146c7d7855c03bd. * Revert "RD: Added temp. debug messages" This reverts commit 31b103c207dd900689c94fe5b608a0a03db8fc80. * RD: Modified SALRegistrationService to run SAL registration in dedicated worker thread, and interrupt registration if it takes too long. Added related settings and assigned defaults. * RD: Added temp. log messages * Revert "RD: Added temp. log messages" This reverts commit 2dae6582de0359a4f33b0d8c066b1b55ecad2227. * RD: Updated DeviceMetricsMonitorService to match metric events to device using first the IP address and the client Id (stored in Device.StatusUpdate) * Improvements on edge device data propagation I94a6fdb4612de192c24511445f1236cdce94b000 * Integration of changes performed in commit e3bd3852 but accidentally overriden I94a6fdb4612de192c24511445f1236cdce94b000 * RD: Updated Dockerfile * RD: Updated run.sh and added wait_for_mongodb.sh * RD: Improved wait_for_mongodb.sh script * Various Improvements Addition of the port attribute in the registration of an edge device Modification of the edge device registration to allow for more dynamic registration json field population I94a6fdb4612de192c24511445f1236cdce94b000 * Attempt to fix a problem when publishing to the broker to register information to SAL or notify the SLO Violation detector I94a6fdb4612de192c24511445f1236cdce94b000 * Miscellaneous improvements Updated the registration of architecture/jar files for each edge device registered Preliminary work to support 'Compromised state' * RD: Added API Key authentication * RD: Upgraded SB to 3.2.10, and Lombok, commons-lang3 dependencies to their latest versions. Improved Dockerfile. * RD: Code improvements (esp. wrt @PreAuthorize annotations) * RD: Made SALRegistrationService service conditionally enabled (by 'discovery.sal-registration.enabled' property). Its uses were updated accordingly. * RD: Updated RegistrationRequestService (and its uses) to take authenticated user into consideration during checks of requests and devices sent from GUI. Also added checks for the device data provided. * RD: Added 'Device.ref' field and added its initialization. Ref field will be populated with a unique device reference following the Nebulous naming convention (including app id). * RD: Updated GUI pages to include the new 'ref' field. A few more improvements were introduced. * Use device reference instead of name to register to SAL * Improvements in device registration and component communication with the broker * Small improvements in device registration * Small improvement in getting device registration details * Small improvement in getting device registration details * RD: Fix in SALRegistrationService class * RD: Fixed fontawesome cdn url * Initial deregistration support * RD: Made device on-boarding authorization configurable (MANUAL, ALWAYS_AUTHORIZE, and ALWAYS_REJECT) * RD: Updated RegistrationRequestProcessor to include Device Ref in onboarding request to EMS * Improvements in the handling of AMQP connections * Addition of provider field, miscellaneous improvements - Addition of provider field in device registration - Small improvements the Synchronous Broker publisher * Minor logging improvements * Deregistration process improvement * Stopping the device lost publisher * Refactoring to use the original Connector class instead of ExtendedConnector * Minor logging improvement * Only try to stop the connector if it has been previously initialized * Miscellaneous improvements Implementation of edge device/byon hourly cost (price) field Only try to stop the connector if it has been previously initialized * Password improvements Removal of plaintext passwords and replacement with environmental variable values * Debugging message commit * Work on Deregistration support * Added more logging statements to debug device registration (work by ipatini) * Fix parsing the json response of SAL * Changes to avoid stopping the connector * Avoid starting new connectors, mark device as offboarded When a device has failed and it has been processed (i.e a message has been sent to the device lost topic) then it should be marked as offboarded to be archived Do not create a new connector unless this is required Do not use a redundant environmental variable --------- Co-authored-by: ipatini Co-authored-by: Andreas Tsagkaropoulos --- .../resource/discovery/monitor/DeviceProcessor.java | 6 +++--- .../registration/service/SALRegistrationService.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java index 5fbdcb3..3dd0618 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/monitor/DeviceProcessor.java @@ -145,8 +145,8 @@ private void processFailedDevices() { lost_device_message.put("device_name",device.getName()); Clock clock = Clock.systemUTC(); lost_device_message.put("timestamp",(int)(clock.millis()/1000)); - log.info("Creating new BrokerPublisher to publish device lost message"); - device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); + log.info("Trying to use existing BrokerPublisher to publish device lost message"); + //device_lost_publisher = new BrokerPublisher(processorProperties.getLost_device_topic(), processorProperties.getNebulous_broker_ip_address(), processorProperties.getNebulous_broker_port(), processorProperties.getNebulous_broker_username(), processorProperties.getNebulous_broker_password(), ""); int sending_attempt = 1; while (device_lost_publisher.is_publisher_null()){ @@ -166,7 +166,7 @@ private void processFailedDevices() { log.warn("processFailedDevices: Marked as FAILED device with Id: {}", device.getId()); //device_lost_publisher.stop(); } - + device.setStatus(DeviceStatus.OFFBOARDED); deviceManagementService.update(device); } diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java index a35912b..ef28ba3 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/service/SALRegistrationService.java @@ -47,11 +47,11 @@ public String register(Device device) { if (application_name.equals("all_applications")){ application_name=""; } - String public_ip = System.getenv("NEBULOUS_IP"); + /*String public_ip = System.getenv("NEBULOUS_IP"); if (public_ip==null || public_ip.isEmpty()){ public_ip = processorProperties.getNebulous_server_ip_address(); - log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip); - } + log.warn("Using default IP address ({}) to fetch Proactive client jar files from, as the environmental variable was not set or found", public_ip);}*/ + String public_ip = processorProperties.getNebulous_server_ip_address(); Map device_info = device.getDeviceInfo(); log.warn("SALRegistrationService: register: DEVICE-INFO: {}", device_info); From d79877b325592d549e3cb83f8b125ed329a9cab5 Mon Sep 17 00:00:00 2001 From: ipatini <3739531+ipatini@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:12:36 +0200 Subject: [PATCH 131/132] Fix for missing device info. Also additional log messages (#38) --- .../registration/RegistrationRequestProcessor.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 680059c..ce67109 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -290,6 +290,20 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {} {}", obj==null?null:obj.getClass().getTypeName(), obj); + + // If device info are missing copy them from the registration request + if (obj==null || obj instanceof Map && ((Map) obj).isEmpty()) { + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** NO DEVICE INFO IN RESPONSE **"); + if (! registrationRequest.getDevice().getDeviceInfo().isEmpty()) { + obj = registrationRequest.getDevice().getDeviceInfo(); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** DEVICE INFO COPIED FROM REGISTRATION REQUEST **"); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {}", obj); + } else { + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** PROBLEM: REGISTRATION REQUEST DOES NOT CONTAIN DEVICE INFO EITHER **"); + } + } + log.warn("RegistrationRequestProcessor: processResponse: CHECK: ** NOT A REAL EXCEPTION **\n", new RuntimeException("NOT A REAL EXCEPTION")); + if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp)); From eda6077d83cb5cbc1d8307390a86bd55434bd1a2 Mon Sep 17 00:00:00 2001 From: ipatini Date: Fri, 22 Nov 2024 12:22:16 +0200 Subject: [PATCH 132/132] RD: Fix for missing device info --- .../registration/RegistrationRequestProcessor.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java index 680059c..ce67109 100644 --- a/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java +++ b/resource-discovery/src/main/java/eu/nebulous/resource/discovery/registration/RegistrationRequestProcessor.java @@ -290,6 +290,20 @@ private void processResponse(@NonNull Map response) { boolean doArchive = false; Object obj = response.get("nodeInfo"); log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {} {}", obj==null?null:obj.getClass().getTypeName(), obj); + + // If device info are missing copy them from the registration request + if (obj==null || obj instanceof Map && ((Map) obj).isEmpty()) { + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** NO DEVICE INFO IN RESPONSE **"); + if (! registrationRequest.getDevice().getDeviceInfo().isEmpty()) { + obj = registrationRequest.getDevice().getDeviceInfo(); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** DEVICE INFO COPIED FROM REGISTRATION REQUEST **"); + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: {}", obj); + } else { + log.warn("RegistrationRequestProcessor: processResponse: nodeInfo: ** PROBLEM: REGISTRATION REQUEST DOES NOT CONTAIN DEVICE INFO EITHER **"); + } + } + log.warn("RegistrationRequestProcessor: processResponse: CHECK: ** NOT A REAL EXCEPTION **\n", new RuntimeException("NOT A REAL EXCEPTION")); + if (obj instanceof Map devInfo) { // Update request info registrationRequest.setLastUpdateDate(Instant.ofEpochMilli(timestamp));