diff --git a/enexa-module-dir-claim-persistentvolume.yaml b/enexa-module-dir-claim-persistentvolume.yaml new file mode 100644 index 0000000..7603527 --- /dev/null +++ b/enexa-module-dir-claim-persistentvolume.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: enexa-module-dir-claim + name: enexa-module-dir-claim +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Mi diff --git a/enexa-role.yml b/enexa-role.yml new file mode 100644 index 0000000..274d1d7 --- /dev/null +++ b/enexa-role.yml @@ -0,0 +1,9 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: pod-creator +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "update", "delete"] diff --git a/enexa-service-deployment.yaml b/enexa-service-deployment.yaml index a2e140e..4808540 100644 --- a/enexa-service-deployment.yaml +++ b/enexa-service-deployment.yaml @@ -27,10 +27,10 @@ spec: containers: - name: enexa-service imagePullPolicy: IfNotPresent - image: enexa-service:0.0.1 + image: hub.cs.upb.de/enexa/images/enexa-service-dev:0.0.11 env: - name: ENEXA_MODULE_DIRECTORY - value: /modules + value: enexa-module-dir-claim - name: ENEXA_SERVICE_URL value: http://enexa-service:8080 - name: ENEXA_SHARED_DIRECTORY @@ -38,7 +38,7 @@ spec: - name: ENEXA_SHARED_VOLUME value: enexa-shared-dir-claim - name: ENEXA_META_DATA_ENDPOINT - value: http://localhost:3030/enexa/ + value: http://fuseki-devwd-service:3030/mydataset/ - name: ENEXA_META_DATA_GRAPH value: http://example.org/meta-data - name: ENEXA_RESOURCE_NAMESPACE @@ -50,16 +50,15 @@ spec: - mountPath: /enexa name: enexa-shared-dir-claim - mountPath: /modules - name: enexa-module-dir + name: enexa-module-dir-claim restartPolicy: Always volumes: - name: enexa-shared-dir-claim persistentVolumeClaim: claimName: enexa-shared-dir-claim - - name: enexa-module-dir - hostPath: - # directory location on host - path: /home/micha/data/enexa/modules - # this field is optional - type: DirectoryOrCreate + - name: enexa-module-dir-claim + persistentVolumeClaim: + claimName: enexa-module-dir-claim + nodeSelector: + kubernetes.io/hostname: minikube status: {} diff --git a/enexa-service-role.yaml b/enexa-service-role.yaml new file mode 100644 index 0000000..c0fa1b0 --- /dev/null +++ b/enexa-service-role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-creator-binding + namespace: default +subjects: + - kind: ServiceAccount + name: default # Specify the service account name here + namespace: default +roleRef: + kind: Role + name: pod-creator + apiGroup: rbac.authorization.k8s.io diff --git a/enexa-service-service.yaml b/enexa-service-service.yaml index b9bede5..78efba9 100644 --- a/enexa-service-service.yaml +++ b/enexa-service-service.yaml @@ -1,19 +1,40 @@ +#apiVersion: v1 +#kind: Service +#metadata: +# annotations: +# kompose.cmd: kompose convert +# kompose.version: 1.26.0 (40646f47) +# creationTimestamp: null +# labels: +# io.kompose.service: enexa-service +# name: enexa-service +#spec: +# ports: +# - name: "8081" +# port: 8081 +# targetPort: 8081 +# selector: +# io.kompose.service: enexa-service +#status: +# loadBalancer: {} + apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: enexa-service - name: enexa-service + annotations: + kompose.cmd: kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: enexa-service + name: enexa-service spec: - ports: - - name: "8080" - port: 8080 - targetPort: 8080 - selector: - io.kompose.service: enexa-service + ports: + - name: "8081" # Exposed port (can be any open port) + port: 8081 # Target port (port the pod listens on) + targetPort: 8081 + selector: + io.kompose.service: enexa-service + type: NodePort # Exposes the pod on a NodePort status: - loadBalancer: {} + loadBalancer: {} diff --git a/enexa-shared-dir-claim-persistentvolumeclaim.yaml b/enexa-shared-dir-claim-persistentvolumeclaim.yaml index e7f8df5..dec02e3 100644 --- a/enexa-shared-dir-claim-persistentvolumeclaim.yaml +++ b/enexa-shared-dir-claim-persistentvolumeclaim.yaml @@ -7,8 +7,8 @@ metadata: name: enexa-shared-dir-claim spec: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: - storage: 100Mi + storage: 25000Mi status: {} diff --git a/enexa-shared-local-volume.yaml b/enexa-shared-local-volume.yaml index c09f8bb..b1cc04f 100644 --- a/enexa-shared-local-volume.yaml +++ b/enexa-shared-local-volume.yaml @@ -4,14 +4,14 @@ metadata: name: enexa-shared-pv spec: capacity: - storage: 10Gi + storage: 25000Mi volumeMode: Filesystem accessModes: - - ReadWriteOnce + - ReadWriteMany persistentVolumeReclaimPolicy: Delete storageClassName: local-storage local: - path: /home/mroeder/data/enexa/shared + path: /home/farshad/test/enexa/shared nodeAffinity: required: nodeSelectorTerms: diff --git a/exena-module-local-volume.yaml b/exena-module-local-volume.yaml new file mode 100644 index 0000000..5e1269a --- /dev/null +++ b/exena-module-local-volume.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: enexa-module-pv +spec: + capacity: + storage: 500Mi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Delete + storageClassName: local-storage + local: + path: /home/farshad/test/enexa/shared + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - minikube diff --git a/fuseki-service-service.yaml b/fuseki-service-service.yaml new file mode 100644 index 0000000..31a0543 --- /dev/null +++ b/fuseki-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: fuseki-devwd-service +spec: + selector: + app: fuseki-devwd # Match pods in the Deployment + ports: + - protocol: TCP + port: 3030 # Match container port + targetPort: 3030 # Target the container port diff --git a/fuseki-service.yaml b/fuseki-service.yaml index 81aad74..a2545c4 100644 --- a/fuseki-service.yaml +++ b/fuseki-service.yaml @@ -1,19 +1,31 @@ -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment #TODO should it be service ! metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: fuseki - name: fuseki + name: fuseki-devwd # Match your container name spec: - ports: - - name: "3030" - port: 3030 - targetPort: 3030 - selector: - io.kompose.service: fuseki -status: - loadBalancer: {} + replicas: 1 # Number of pods to run + selector: + matchLabels: + app: fuseki-devwd # Label for pods + template: + metadata: + labels: + app: fuseki-devwd + spec: + containers: + - name: fuseki-devwd + image: stain/jena-fuseki # Your container image + ports: + - containerPort: 3030 + name: http # Optional name for the port + env: + - name: ADMIN_PASSWORD + value: "pw123" # Environment variable + volumeMounts: + - name: fuseki-data # Volume name + mountPath: /fuseki # Path to mount volume in container + volumes: + - name: fuseki-data # Persistent volume claim (PVC) needed later + emptyDir: {} # EmptyDir volume for data (consider Persistent Volume for persistence) + + diff --git a/pom.xml b/pom.xml index ab1105e..d16e110 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ org.dice-research enexa.java-utils - 0.0.1-SNAPSHOT + 0.0.2-SNAPSHOT diff --git a/src/main/java/eu/enexa/docker/ContainerManagerImpl.java b/src/main/java/eu/enexa/docker/ContainerManagerImpl.java index 8c41c46..6c6a59c 100644 --- a/src/main/java/eu/enexa/docker/ContainerManagerImpl.java +++ b/src/main/java/eu/enexa/docker/ContainerManagerImpl.java @@ -11,33 +11,40 @@ import eu.enexa.service.ContainerManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Primary; + +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +import org.springframework.beans.factory.annotation.Value; + import java.util.*; -import java.io.File; -/** - * Manages Docker containers, providing functionality to start, stop, and get the status of containers. - */ -//TODO: remove primary after using config for beans -@Primary + +// Manages Docker containers, providing functionality to start, stop, and get the status of containers. + +@Profile("docker") @Component("dockerContainerManager") public class ContainerManagerImpl implements ContainerManager { - /** - * Logger for ContainerManagerImpl class. - */ + + // Logger for ContainerManagerImpl class. + + private static final Logger LOGGER = LoggerFactory.getLogger(ContainerManagerImpl.class); - /** - * Name of the Docker network to which containers are attached. - */ + + // Name of the Docker network to which containers are attached. + + private static final String NETWORK_NAME = System.getenv("DOCKER_NET_NAME"); - /** - * Docker client used for interacting with the Docker daemon. - */ - private DockerClient dockerClient; + + // Docker client used for interacting with the Docker daemon. + + + private final DockerClient dockerClient; + + +// + // Constructs a new ContainerManagerImpl and initializes the Docker client. @Value("${docker.traefik.hostname}") private String hostName ; @@ -45,9 +52,6 @@ public class ContainerManagerImpl implements ContainerManager { @Value("${docker.traefik.loadbalancer.port}") private String traefikLoadBalancerPort; - /** - * Constructs a new ContainerManagerImpl and initializes the Docker client. - */ public ContainerManagerImpl(){ LOGGER.info("start initiating the ContainerManagerImpl"); DockerClientConfig standard = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); @@ -72,40 +76,8 @@ public ContainerManagerImpl(){ LOGGER.info("docker client is pinged"); } - /** - * Combines two path components to create a valid path. - * the directory will create if not exist - * - * @param partOneOfPath The first part of the path. - * @param partTwoOfPath The second part of the path. - * @return The combined path. - */ - private String makeTheDirectoryInThisPath(String partOneOfPath, String partTwoOfPath) { - String path = combinePaths(partOneOfPath, partTwoOfPath); - File appPathDirectory = new File(path); - if(!appPathDirectory.exists()){ - appPathDirectory.mkdirs(); - } - return path; - } - - /** - * Combines two path components to create a valid path, taking into account trailing separators. - * - * @param partOne The first part of the path. - * @param partTwo The second part of the path. - * @return The combined path. - */ - public static String combinePaths(String partOne, String partTwo) { - String path = partOne + File.separator + partTwo; - if (partOne.endsWith(File.separator)) { - path = partOne + partTwo; - } - return path; - } - @Override - public String startContainer(String image, String containerName, List> variables,String hostSharedDirectory, String appName) { + public String startContainer(String image, String containerName, List> variables,String hostSharedDirectory, String appName,MapcontainerSettings) { if (variables == null ){ variables = new ArrayList<>(); } @@ -119,7 +91,7 @@ public String startContainer(String image, String containerName, List labels = new HashMap<>(); @@ -193,7 +165,7 @@ public String startContainer(String image, String containerName, List allBinds, HostConfig dockerHostConfig) { HostConfig changedDockerHostConfig = dockerHostConfig; @@ -228,24 +201,26 @@ private HostConfig addExceptionalConditions(String image, List allBinds, H return changedDockerHostConfig; } - /** +/** * Trims the Docker image name to remove the "docker:image:" prefix. * * @param image The Docker image name. * @return The trimmed image name. - */ + */ + private String trimImageName(String image) { String[] parts = image.split("docker:image:"); if(parts.length<2){return image;} return parts[1]; } - /** +/** * Converts a list of environment variable entries into an array of strings. * * @param environmentVariables List of environment variable entries. * @return Array of environment variable strings. - */ + */ + private String[] mapToEnvironmentArray(List> environmentVariables) { if (environmentVariables == null || environmentVariables.isEmpty()) { return new String[0]; @@ -285,12 +260,20 @@ public String getContainerStatus(String containerNameOrId) { } } + @Override + public Map resolveContainerEndpoint(String containerId, Integer port) { + //todo implement this for docker + // get container name and use as url + return new HashMap<>(); + } + /** * Searches for a Docker container by name or ID and returns the corresponding Container object. * * @param containerNameOrId The name or ID of the container. * @return The Container object representing the container. - */ + */ + private Container searchContainerByNameOrId(String containerNameOrId){ List containers = this.dockerClient.listContainersCmd() .withShowAll(true) diff --git a/src/main/java/eu/enexa/example/ExampleApplication.java b/src/main/java/eu/enexa/example/ExampleApplication.java index 9fec2f1..cc5302b 100644 --- a/src/main/java/eu/enexa/example/ExampleApplication.java +++ b/src/main/java/eu/enexa/example/ExampleApplication.java @@ -35,23 +35,17 @@ public class ExampleApplication implements AutoCloseable { private static final String STATUS_PENDING = "Pending"; private static final String STATUS_RUNNING = "Running"; - private CloseableHttpClient client; - private String enexaURL; - private String sharedDirPath; - private String appPath; + private final CloseableHttpClient client; + private final String enexaURL; + private final String appPath; private String experimentIRI; private String instanceIRI; private String metaDataEndpoint; private String metaDataGraph; - private String kgFileIri; private String jsonFileLocation; private String urlsIri; private String jsonIri; - private String resultFileIri; - private String resultFileLocation; - - private final String preFix = "http://w3id.org/dice-research/enexa/module/dice-embeddings/"; private final String appName = "app1"; @@ -59,9 +53,7 @@ public ExampleApplication(String enexaURL, String sharedDirPath, String appPath) super(); this.enexaURL = enexaURL; if (sharedDirPath.endsWith(File.separator)) { - this.sharedDirPath = sharedDirPath.substring(0, sharedDirPath.length() - 1); } else { - this.sharedDirPath = sharedDirPath; } if (appPath.endsWith(File.separator)) { this.appPath = appPath.substring(0, appPath.length() - 1); @@ -79,19 +71,19 @@ public void startExperiment() throws Exception { } Resource expResource = RdfHelper.getSubjectResource(model, RDF.type, ENEXA.Experiment); if (expResource == null) { - throw new Exception("Couldn't find experiment resource."); + throw new IllegalStateException("Couldn't find experiment resource."); } experimentIRI = expResource.getURI(); LOGGER.info("Started an experiment: {}", experimentIRI); // Get meta data endpoint and graph Resource resource = RdfHelper.getObjectResource(model, expResource, ENEXA.metaDataEndpoint); if (resource == null) { - throw new Exception("Couldn't find the experiment's meta data endpoint."); + throw new IllegalStateException("Couldn't find the experiment's meta data endpoint."); } metaDataEndpoint = resource.getURI(); resource = RdfHelper.getObjectResource(model, expResource, ENEXA.metaDataGraph); if (resource == null) { - throw new Exception("Couldn't find the experiment's meta data graph."); + throw new IllegalStateException("Couldn't find the experiment's meta data graph."); } metaDataGraph = resource.getURI(); LOGGER.info("Meta data can be found at {} in graph {}", metaDataEndpoint, metaDataGraph); @@ -128,17 +120,17 @@ public void addKGFile(String kgFile) throws Exception { Model response = requestRDF(enexaURL + "/add-resource", fileDescription); if (response == null) { - throw new Exception("Couldn't add a resource to the meta data."); + throw new IllegalStateException("Couldn't add a resource to the meta data."); } // Get the new IRI of the resource Resource fileResource = RdfHelper.getSubjectResource(response, RDF.type, response.createResource("http://www.w3.org/ns/prov#Entity")); if (fileResource == null) { - throw new Exception("Couldn't find the file resource."); + throw new IllegalStateException("Couldn't find the file resource."); } LOGGER.info("File resource {} has been created.", fileResource.getURI()); - kgFileIri = fileResource.getURI(); + //String kgFileIri = fileResource.getURI(); // } @@ -172,14 +164,14 @@ public void addUrls(String urlsFile, String moduleName) throws Exception { Model response = requestRDF(enexaURL + "/add-resource", fileDescription); if (response == null) { - throw new Exception("Couldn't add a resource to the meta data."); + throw new IllegalStateException("Couldn't add a resource to the meta data."); } // Get the new IRI of the resource Resource fileResource = RdfHelper.getSubjectResource(response, RDF.type, response.createResource("http://www.w3.org/ns/prov#Entity")); if (fileResource == null) { - throw new Exception("Couldn't find the file resource."); + throw new IllegalStateException("Couldn't find the file resource."); } LOGGER.info("File resource {} has been created.", fileResource.getURI()); urlsIri = fileResource.getURI(); @@ -215,14 +207,14 @@ public void addJson(String jsonFile, String moduleName) throws Exception { Model response = requestRDF(enexaURL + "/add-resource", fileDescription); if (response == null) { - throw new Exception("Couldn't add a resource to the meta data."); + throw new IllegalStateException("Couldn't add a resource to the meta data."); } // Get the new IRI of the resource Resource fileResource = RdfHelper.getSubjectResource(response, RDF.type, response.createResource("http://www.w3.org/ns/prov#Entity")); if (fileResource == null) { - throw new Exception("Couldn't find the file resource."); + throw new IllegalStateException("Couldn't find the file resource."); } LOGGER.info("File resource {} has been created.", fileResource.getURI()); jsonIri = fileResource.getURI(); @@ -247,12 +239,12 @@ private void startExtraction() throws Exception { Model response = requestRDF(enexaURL + "/start-container", instanceModel); if (response == null) { - throw new Exception("Couldn't start a container."); + throw new IllegalStateException("Couldn't start a container."); } // Get the new IRI of the newly created module instance Resource instanceResource = RdfHelper.getSubjectResource(response, RDF.type, ENEXA.ModuleInstance); if (instanceResource == null) { - throw new Exception("Couldn't find module instance resource."); + throw new IllegalStateException("Couldn't find module instance resource."); } instanceIRI = instanceResource.getURI(); LOGGER.info("module instance {} has been created.", instanceIRI); @@ -263,34 +255,34 @@ private void startEmbeddingGeneration() throws Exception { // The module instance itself will be a blank node Resource instance = instanceModel.createResource(); instanceModel.add(instance, RDF.type, ENEXA.ModuleInstance); - //TODO is it correct ? - instanceModel.add(instance, Algorithm.instanceOf, instanceModel.createResource(preFix+"v1.0")); + String preFix = "http://w3id.org/dice-research/enexa/module/dice-embeddings/"; + instanceModel.add(instance, Algorithm.instanceOf, instanceModel.createResource(preFix +"v1.0")); instanceModel.add(instance, ENEXA.experiment, instanceModel.createResource(experimentIRI)); // Add parameters /*instanceModel.add(instance, instanceModel.createProperty(preFix+"parameters/model"), instanceModel.createResource(kgFileIri));*/ - instanceModel.add(instance, instanceModel.createProperty(preFix+"parameter/model"), + instanceModel.add(instance, instanceModel.createProperty(preFix +"parameter/model"), instanceModel.createTypedLiteral("ConEx")); //http://module-instance-1> . - instanceModel.add(instance,instanceModel.createProperty(preFix+"parameter/path_dataset_folder"),instanceModel.createResource(jsonFileLocation.replace("enexa-dir:","http:"))); + instanceModel.add(instance,instanceModel.createProperty(preFix +"parameter/path_dataset_folder"),instanceModel.createResource(jsonFileLocation.replace("enexa-dir:","http:"))); // enexa:location "enexa-dir://something" . instanceModel.add(instanceModel.createResource(jsonFileLocation.replace("enexa-dir:","http:")),ENEXA.location,instanceModel.createLiteral(jsonFileLocation)); - instanceModel.add(instance, instanceModel.createProperty(preFix+"parameter/num_epochs"), + instanceModel.add(instance, instanceModel.createProperty(preFix +"parameter/num_epochs"), instanceModel.createTypedLiteral(5)); - instanceModel.add(instance, instanceModel.createProperty(preFix+"parameter/embedding_dim"), + instanceModel.add(instance, instanceModel.createProperty(preFix +"parameter/embedding_dim"), instanceModel.createTypedLiteral(2)); // Send the model Model response = requestRDF(enexaURL + "/start-container", instanceModel); if (response == null) { - throw new Exception("Couldn't start a container."); + throw new IllegalStateException("Couldn't start a container."); } // Get the new IRI of the newly created module instance Resource instanceResource = RdfHelper.getSubjectResource(response, RDF.type, ENEXA.ModuleInstance); if (instanceResource == null) { - throw new Exception("Couldn't find module instance resource."); + throw new IllegalStateException("Couldn't find module instance resource."); } instanceIRI = instanceResource.getURI(); LOGGER.info("module instance {} has been created.", instanceIRI); @@ -337,12 +329,12 @@ private void waitForEmbeddings() throws Exception { Model response = requestRDF(enexaURL + "/container-status", requestModel); if (response == null) { - throw new Exception("Couldn't get the status of a container."); + throw new IllegalStateException("Couldn't get the status of a container."); } // Get the new IRI of the newly created module instance status = RdfHelper.getStringValue(requestModel, instance, ENEXA.containerStatus); if (status == null) { - throw new Exception("Couldn't find the status of the module instance resource."); + throw new IllegalStateException("Couldn't find the status of the module instance resource."); } if (status.equals(STATUS_PENDING) || status.equals(STATUS_RUNNING)) { Thread.sleep(1000); @@ -360,8 +352,8 @@ private void queryFilePath() throws Exception { ResultSet rs = qe.execSelect(); if (rs.hasNext()) { QuerySolution qs = rs.next(); - resultFileIri = qs.getResource("fileIri").getURI(); - resultFileLocation = qs.getLiteral("fileLocation").getString(); + String resultFileIri = qs.getResource("fileIri").getURI(); + String resultFileLocation = qs.getLiteral("fileLocation").getString(); LOGGER.info("Result file {} located at {}.", resultFileIri, resultFileLocation); } else { LOGGER.error("Couldn't get the expected result file from the meta data endpoint."); diff --git a/src/main/java/eu/enexa/example/SimpleClient.java b/src/main/java/eu/enexa/example/SimpleClient.java new file mode 100644 index 0000000..69e9798 --- /dev/null +++ b/src/main/java/eu/enexa/example/SimpleClient.java @@ -0,0 +1,375 @@ +package eu.enexa.example; + +import org.aksw.jena_sparql_api.http.QueryExecutionFactoryHttp; +import org.aksw.jena_sparql_api.pagination.core.QueryExecutionFactoryPaginated; +import org.aksw.jenax.arq.connection.core.QueryExecutionFactory; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.jena.query.*; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.core.DatasetDescription; +import org.apache.jena.vocabulary.RDF; +import org.dice_research.enexa.vocab.ENEXA; +import org.dice_research.rdf.RdfHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.net.http.HttpClient; +public class SimpleClient implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleClient.class); + private final CloseableHttpClient client; + private static final String SHARED_DIR_PREFIX = "enexa-dir:/"; + private String experimentIRI; + private final String enexaURL = "http://192.168.49.2:30479"; // "http://localhost:8081"; + + /** + * A simple application demonstrating the development of a client that utilizes the ENEXA service. + */ + public SimpleClient() { + client = HttpClients.createDefault(); + + } + + /** + * Send a start experiment request to the service and set the experimentIRI, metaDataEndpoint, and metaDataGraph parameters. + */ + + public void startExperiment() throws Exception { + Model model = requestRDF(enexaURL + "/start-experiment", null); + if (model == null) { + throw new IOException("Couldn't create experiment."); + } + Resource expResource = RdfHelper.getSubjectResource(model, RDF.type, ENEXA.Experiment); + if (expResource == null) { + throw new IllegalStateException("Couldn't find experiment resource."); + } + experimentIRI = expResource.getURI(); + LOGGER.info("Started an experiment: {}", experimentIRI); + // Get meta data endpoint and graph + Resource resource = RdfHelper.getObjectResource(model, expResource, ENEXA.metaDataEndpoint); + if (resource == null) { + throw new IllegalStateException("Couldn't find the experiment's meta data endpoint."); + } + //private String instanceIRI; + String metaDataEndpoint = resource.getURI(); + resource = RdfHelper.getObjectResource(model, expResource, ENEXA.metaDataGraph); + if (resource == null) { + throw new IllegalStateException("Couldn't find the experiment's meta data graph."); + } + String metaDataGraph = resource.getURI(); + LOGGER.info("Meta data can be found at {} in graph {}", metaDataEndpoint, metaDataGraph); + } + + + /*protected String requestGet(String url) { + HttpGet request = new HttpGet(url); + try (CloseableHttpResponse httpResponse = client.execute(request)) { + if (httpResponse.getCode() >= 300) { + throw new IllegalStateException("Received HTTP response with code " + httpResponse.getCode()); + } + + try (InputStream is = httpResponse.getEntity().getContent()) { + return IOUtils.toString(is, "UTF-8"); + } + } catch (Exception e) { + LOGGER.error("Caught an exception while running request. Returning null."); + return null; + } + }*/ + + + /** + * Send an HTTP GET request and return the result as a Model. + * @param url the URL to which the request is sent + */ + protected Model requestGetRecieveModel(String url) { + HttpGet request = new HttpGet(url); + Model model = null; + try (CloseableHttpResponse httpResponse = client.execute(request)) { + if (httpResponse.getCode() >= 300) { + throw new IllegalStateException("Received HTTP response with code " + httpResponse.getCode()); + } + + try (InputStream is = httpResponse.getEntity().getContent()) { + model = ModelFactory.createDefaultModel(); + model.read(is, "", "JSON-LD"); + } + } catch (Exception e) { + LOGGER.error("Caught an exception while running request. Returning null."); + return null; + } + return model; + } + + /** + * Sends an HTTP POST request to the specified URL with the given request body. + * + * @param url the URL to which the request is sent + * @param body the body of the HTTP POST request + * @return the response body of the HTTP POST request + * @throws IllegalStateException if the HTTP response code is greater than or equal to 300 + */ + protected String requestPost(String url, String body) { + HttpPost request = new HttpPost(url); + request.setHeader("Accept", "text/turtle"); + request.setHeader("Content-type", "application/json"); + // Set the request body + StringEntity stringEntity = new StringEntity(body); + request.setEntity(stringEntity); + + try (CloseableHttpResponse httpResponse = client.execute(request)) { + if (httpResponse.getCode() >= 300) { + throw new IllegalStateException("Received HTTP response with code " + httpResponse.getCode()); + } + + try (InputStream is = httpResponse.getEntity().getContent()) { + return IOUtils.toString(is, "UTF-8"); + } + } catch (Exception e) { + LOGGER.error("Caught an exception while running request. Returning null."); + return null; + } + } + + /** + * Sends an HTTP POST request to the specified URL with optional RDF data, expecting RDF data in response. + * + * @param url the URL to which the request is sent + * @param data the RDF data to include in the request (optional) + * @return a Model object containing the RDF data received in response, or null if an error occurs + * @throws IllegalStateException if the HTTP response code is greater than or equal to 300 + */ + protected Model requestRDF(String url, Model data) { + HttpPost request = new HttpPost(url); + request.setHeader("Accept", "application/ld+json"); + request.setHeader("Content-type", "application/ld+json"); + if (data != null) { + try (StringWriter writer = new StringWriter()) { + data.write(writer, "JSON-LD"); + request.setEntity(new StringEntity(writer.toString())); + } catch (IOException e) { + LOGGER.error("Catched unexpected exception while adding data to the request. Returning null.", e); + return null; + } + } + Model model = null; + try (CloseableHttpResponse httpResponse = client.execute(request)) { + if (httpResponse.getCode() >= 300) { + throw new IllegalStateException("Received HTTP response with code " + httpResponse.getCode()); + } + + try (InputStream is = httpResponse.getEntity().getContent()) { + model = ModelFactory.createDefaultModel(); + model.read(is, "", "JSON-LD"); + } + } catch (Exception e) { + LOGGER.error("Caught an exception while running request. Returning null."); + return null; + } + return model; + } + + /** + * Adds a file to the Enexa and returns its IRI. + * + * @param fileToAdd the path of the file to be added + * @param moduleName the name of the module to which the file belongs + * @return the IRI of the added file + * @throws Exception if an error occurs during the file addition process + */ + public String addFile(String fileToAdd, String moduleName) throws Exception { + // Move file if it is not located in the shared directory + File json = new File(fileToAdd); + String appName = "app3"; + String appPath = "/home/farshad/test/enexa/shared"; + File dest = new File(appPath + File.separator+ appName +File.separator+experimentIRI.split("/")[experimentIRI.split("/").length -1] +File.separator +moduleName +File.separator+json.getName()); + try { + FileUtils.copyFile(json, dest); + } catch (IOException e) { + throw new IOException("Couldn't copy the kg file into the shared directory.", e); + } + + String jsonFileDestination = dest.getAbsolutePath(); + + // Get relative path in the shared directory + String addedFileLocation = SHARED_DIR_PREFIX + jsonFileDestination.substring(appPath.length()); + + // Create a model with the meta data of our file + Model fileDescription = ModelFactory.createDefaultModel(); + // The file itself will be a blank node + String addFile = "@prefix enexa: .\n" + + " @prefix prov: .\n" + + "\n" + + " [] a prov:Entity ; \n" + + " enexa:experiment <"+experimentIRI+"> ; \n" + + " enexa:location \""+ addedFileLocation +"\" ."; + fileDescription.read(new java.io.StringReader(addFile),null,"TURTLE"); + + // Send the model + Model response = requestRDF(enexaURL + "/add-resource", fileDescription); + + if (response == null) { + throw new IllegalStateException("Couldn't add a resource to the meta data."); + } + + // Get the new IRI of the resource + Resource fileResource = RdfHelper.getSubjectResource(response, RDF.type, + response.createResource("http://www.w3.org/ns/prov#Entity")); + if (fileResource == null) { + throw new IllegalStateException("Couldn't find the file resource."); + } + LOGGER.info("File resource {} has been created.", fileResource.getURI()); + return fileResource.getURI(); + } + + /** + * Starts the extraction process in the Enexa with the specified parameters and returns the IRI of the instance. + * + * @param iriAddedFile the IRI of the file added for extraction + * @param iriGenerationParameter the IRI of the generation parameter + * @return the IRI of the newly created module instance + * @throws Exception if an error occurs during the extraction process + */ + private String startExtraction(String iriAddedFile , String iriGenerationParameter) throws Exception { + Model instanceModel = ModelFactory.createDefaultModel(); + + String start_module_message = "@prefix alg: .\n" + + "@prefix enexa: .\n" + + "@prefix prov: .\n" + + "@prefix hobbit: . \n" + + "@prefix rdf: .\n" + + "[] rdf:type enexa:ModuleInstance ; " + + "enexa:experiment <"+experimentIRI+"> ; " + + "alg:instanceOf ; " + + " <"+iriAddedFile+">;" + + " <"+iriGenerationParameter+">."; + + instanceModel.read(new java.io.StringReader(start_module_message), null, "TURTLE"); + + Model response = requestRDF(enexaURL + "/start-container", instanceModel); + + if (response == null) { + throw new IllegalStateException("Couldn't start a container."); + } + // Get the new IRI of the newly created module instance + Resource instanceResource = RdfHelper.getSubjectResource(response, RDF.type, ENEXA.ModuleInstance); + if (instanceResource == null) { + throw new IllegalStateException("Couldn't find module instance resource."); + } + return instanceResource.getURI(); + } + + /** + * The main method of the SimpleClient class, responsible for initiating the Enexa experiment, adding files, + * starting extraction, waiting for container running, retrieving metadata, and finding the result path. + * + * @param args the command-line arguments (not used) + */ + public static void main(String[] args) { + + try (SimpleClient app = new SimpleClient()) { + app.startExperiment(); + String moduleName = "extraction"; + String companyJsonIRI = app.addFile("/home/farshad/test/enexa/shared/wikipedia_company_urls_short.json", moduleName); + String generativeParameterIRI = app.addFile("/home/farshad/test/enexa/shared/generation_parameters.json", moduleName); + String extractionInstanceIRI = app.startExtraction(companyJsonIRI, generativeParameterIRI); + app.waitContainerRunning(extractionInstanceIRI); + String metaDataEndPoint = app.getMeta(); + String resultPath=app.findResultPath(metaDataEndPoint, extractionInstanceIRI); + System.out.println("result is saved at : "+ resultPath); + } catch (Exception e) { + throw new IllegalStateException("An unexpected error occurred", e); + } + } + + /** + * Finds the result path of the extraction instance from the metadata endpoint. + * + * @param metaDataEndPoint the metadata endpoint URL + * @param instanceIRI the IRI of the extraction instance + * @return the path of the result file + */ + private String findResultPath(String metaDataEndPoint, String instanceIRI) { + + + LOGGER.info("METADATA endpoint is :"+metaDataEndPoint); + QueryExecutionFactory queryExecFactory = new QueryExecutionFactoryHttp(metaDataEndPoint, new DatasetDescription(), HttpClient.newHttpClient()); + queryExecFactory = new QueryExecutionFactoryPaginated(queryExecFactory, 10); + + + String queryStr = "SELECT ?path FROM WHERE {\n" + + " <"+instanceIRI+"> ?tmp.\n" + + "\t ?tmp ?path.\n" + + "}"; + // Create a SPARQL query object + LOGGER.info("query is :"+queryStr); + QueryExecution qe = queryExecFactory.createQueryExecution(queryStr); + + ResultSet rs = qe.execSelect(); + if (rs.hasNext()) { + QuerySolution qs = rs.next(); + return qs.getLiteral("path").getString(); + } else { + LOGGER.error("Couldn't get the expected result file from the meta data endpoint."); + } + return null; + } + + /** + * Retrieves the metadata endpoint URI based on the experiment IRI. + * + * @return the URI of the metadata endpoint + */ + private String getMeta() { + Model model = requestGetRecieveModel(enexaURL + "/meta?experimentIRI="+experimentIRI); + Resource expResource = RdfHelper.getObjectResource(model,null, ENEXA.metaDataEndpoint); + return expResource.getURI(); + } + + /** + * Waits until the container associated with the specified module instance IRI is running. + * + * @param instanceIRI the IRI of the module instance + * @throws Exception if an error occurs while waiting for the container to run + */ + private void waitContainerRunning(String instanceIRI) throws Exception { + String body = " {\n" + + " \"moduleInstanceIRI\":\""+instanceIRI+"\",\n" + + " \"experimentIRI\":\""+experimentIRI+"\"\n" + + " }"; + boolean isRunning = true; + while (isRunning) { + String response = requestPost(enexaURL + "/container-status", body); + + if (response == null) { + throw new IllegalStateException("Couldn't get the status of a container."); + } + // Get the new IRI of the newly created module instance + if (response.contains("run")) { + Thread.sleep(1000); + } else { + return; + } + } + } + + @Override + public void close() throws Exception { + if (client != null) { + client.close(); + } + } +} diff --git a/src/main/java/eu/enexa/kubernetes/ContainerManagerImpl.java b/src/main/java/eu/enexa/kubernetes/ContainerManagerImpl.java index 0f3565a..398a80f 100644 --- a/src/main/java/eu/enexa/kubernetes/ContainerManagerImpl.java +++ b/src/main/java/eu/enexa/kubernetes/ContainerManagerImpl.java @@ -1,26 +1,28 @@ package eu.enexa.kubernetes; import java.io.IOException; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; +import java.util.stream.Collectors; +import io.kubernetes.client.custom.IntOrString; +import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.*; +import io.kubernetes.client.util.generic.KubernetesApiResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; import eu.enexa.service.ContainerManager; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.Configuration; -import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.util.Config; import io.kubernetes.client.util.generic.GenericKubernetesApi; +@Profile("kubernetes") @Component("kubernetesContainerManager") public class ContainerManagerImpl implements ContainerManager { @@ -28,8 +30,47 @@ public class ContainerManagerImpl implements ContainerManager { private ApiClient client; + @Value("${container.manager.namespace:default}") + private String nameSpace; + @Value("${container.manager.timeoutMilliSeconds:60000}") + private int TIMEOUT_MILLISECONDS; + @Value("${container.manager.defaultMemorySize:8}") + private String memoryInGiB; + protected ContainerManagerImpl() { + try { + client = initiateClient(); + }catch (Exception ex){ + LOGGER.error(ex.getMessage()); + } + } + /** + * Initializes and configures a Kubernetes API client with custom timeout settings. + * + * This method: + * - Creates a default Kubernetes API client using the default configuration. + * - Sets custom connection, read, and write timeouts. + * - Sets the configured client as the default client in the global Configuration class. + * + * @return ApiClient configured with specified timeouts. + * @throws IOException if an error occurs during client initialization. + */ + protected ApiClient initiateClient() throws IOException { + try{ + LOGGER.info("initiating Kubernetes client"); + ApiClient client = Config.defaultClient(); + client.setConnectTimeout(TIMEOUT_MILLISECONDS); + client.setReadTimeout(TIMEOUT_MILLISECONDS); + client.setWriteTimeout(TIMEOUT_MILLISECONDS); + Configuration.setDefaultApiClient(client); + LOGGER.info("Kubernetes API client initiated with default configuration."); + return client; + } + catch (Exception ex){ + LOGGER.error("Failed to initiate Kubernetes API client", ex); + throw ex; + } } protected ContainerManagerImpl(ApiClient client) { @@ -38,53 +79,147 @@ protected ContainerManagerImpl(ApiClient client) { @Override public String startContainer(String image, String podName, - List> variables, String hostSharedDirectory, String appName) { - return startContainerKub(image, podName, variables, null, null); + List> variables, String hostSharedDirectory, String appName, Map containerSettings) { + return startContainerKub(image, podName, variables, hostSharedDirectory, null, appName,containerSettings); } - // podName is the containerName for kubernetes - public String startContainerKub(String image, String podName, List> variables,String hostSharedDirectory ,String[] command) { - - List env = new ArrayList<>(); - if (variables != null) { - for (Map.Entry entry : variables) { - V1EnvVar v1env = new V1EnvVar(); - v1env.setName(entry.getKey()); - v1env.setValue(entry.getValue()); - env.add(v1env); + /** + * Creates a Kubernetes service of type NodePort for the given pod. + * + * @param podUid the unique identifier of the pod + * @param podPort the port of the pod to expose + * @param labels the labels for the service selector + * @return the NodePort assigned to the created service + * @throws ApiException if there's an error interacting with the Kubernetes API + */ + public Map getServiceAsNodePort(String podUid, Integer podPort ,Map labels) throws ApiException { + LOGGER.info("Creating service as NodePort for pod UID: {}", podUid); + try { + if (client == null) { + LOGGER.error("client is null"); + client = initiateClient(); } + }catch (Exception ex){ + LOGGER.error("Error initializing client: ", ex); } - /* - * // Create a shared volume V1Volume sharedVolume = new V1Volume(); - * sharedVolume.setName("shared-volume"); - * - * // Define the shared volume source V1EmptyDirVolumeSource - * emptyDirVolumeSource = new V1EmptyDirVolumeSource(); - * sharedVolume.setEmptyDir(emptyDirVolumeSource); - */ + // Create a unique service name using the pod UID and a random UUID + String name = String.format("ser-%s-%s", podUid, UUID.randomUUID()); + LOGGER.info("Service name is: {}", name); - // Create a VolumeClaim - V1PersistentVolumeClaimVolumeSource persistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource(); - persistentVolumeClaim.setClaimName("enexa-shared-dir-claim"); + // Trim the name if it exceeds 60 characters to make a valid name + if (name.length() > 60) { + // add extra S at the end because it could be - and make the name invalid + name = name.substring(0, 59)+"s"; + LOGGER.info("Service name after trimming: {}", name); + } - V1Volume volume = new V1Volume(); - volume.setName("enexa-volume"); - volume.setPersistentVolumeClaim(persistentVolumeClaim); + V1ServicePort serviceport = new V1ServicePortBuilder().withPort(podPort).withTargetPort(new IntOrString(0)).build(); + LOGGER.info("service is :{}", name); + V1Service service = new V1ServiceBuilder() + .withNewMetadata().withName(name).endMetadata() + .withNewSpec() + .withSelector(labels) + .withType("NodePort") // Set service type to NodePort + .withPorts(serviceport) // Exposed port + .endSpec() + .build(); - // Create a container and set the volume mount - V1Container container = new V1Container(); - container.setName("b"); - container.setImage(image); - if (command != null) { - for (int i = 0; i < command.length; ++i) { - container.addCommandItem(command[i]); + LOGGER.info(service.toString()); + + // Create Service in Kubernetes + CoreV1Api api = new CoreV1Api(client); + LOGGER.info("api client initiated :{}", name); + //TODO if need check if the service exist ! + try { + api.createNamespacedService(nameSpace, service, null, null, null, null); + } catch (NullPointerException e) { + System.err.println("Caught NullPointerException while creating service: " + e.getMessage()); + } catch (ApiException e) { + System.err.println("API Exception: " + e.getResponseBody()); + } + // Retrieve the created service to fetch the NodePort assigned by Kubernetes + V1Service createdService = api.readNamespacedService(name, nameSpace, null); + LOGGER.info("Service created with NodePort: {}", createdService.getSpec().getPorts().get(0).getNodePort()); + + Map serviceSpecs = new HashMap<>(); + serviceSpecs.put("port", Objects.requireNonNull(createdService.getSpec().getPorts().get(0).getNodePort()).toString()); + serviceSpecs.put("externalName",createdService.getSpec().getExternalName()); + serviceSpecs.put("clusterIP",createdService.getSpec().getClusterIP()); + + return serviceSpecs; + } + + /** + * Starts a container by creating a pod in Kubernetes with the specified configuration. + * + * @param image The image to use for the container. + * @param podName The name of the pod. + * @param variables Environment variables for the container. + * @param hostSharedDirectory Path for the host's shared directory. + * @param command Command to run inside the container. + * @param appName The application name for directory structure. + * @return The UID of the created pod, or null if creation failed. + */ + public String startContainerKub(String image, String podName, List> variables,String hostSharedDirectory ,String[] command, String appName, Map containerSettings) { + LOGGER.info("Starting container: Image = {}, Pod Name = {}, Variables Size = {}, Host Shared Directory = {}, App Name = {}", + image, podName, variables.size(), hostSharedDirectory, appName); + + try { + if (client == null) { + client = initiateClient(); } + }catch (Exception ex){ + LOGGER.error("Error initializing Kubernetes client: ", ex); + return null; + } + // TODO : make this part not hardcoded if there is chance of changing the "urn:container:docker:image:" + String replacedImage = image.replace("urn:container:docker:image:",""); + LOGGER.info("Using image: {}", replacedImage); + + // Create environment variables + List env = createEnvVariables(variables); + + // Process experiment IRI variable + String expIRI=extractExperimentIRI(variables); + LOGGER.info("ENEXA_EXPERIMENT_IRI: {}", expIRI); + + // Validate IRI + if(expIRI.length() < 10){ + LOGGER.warn("ENEXA_EXPERIMENT_IRI is null or less than 10 character"); + } + + // Set up paths for shared directories + String containerBasePath = "/enexa"; + String hostBasePath = buildHostPath(hostSharedDirectory, appName); + String containerWritablePath = buildWritableContainerPath(expIRI, hostBasePath); + String containerModuleInstancePath = combinePaths(containerWritablePath, UUID.randomUUID().toString()); + + + // Add environment variables related to shared directories + addSharedDirectoryEnvVars(env, containerBasePath, containerModuleInstancePath, containerWritablePath); + + // Set up the persistent volume claim for shared directory + V1Volume volume = createVolume(); + + // Set up resource requirements + // here if the memory exist replace and container start with that for example 2Gi + Optional memoryTheContainerNeeds = Optional.empty(); + if(containerSettings.containsKey("memSize")){ + memoryTheContainerNeeds = Optional.of(containerSettings.get("memSize")); } + V1ResourceRequirements resourceRequirements = createResourceRequirements(memoryTheContainerNeeds); + + // Create container + V1Container container = createContainer(replacedImage, command, env, resourceRequirements); + V1VolumeMount volumeMount = new V1VolumeMount(); - volumeMount.setName("enexa-volume"); + volumeMount.setName("enexa-shared-dir"); volumeMount.setMountPath("/enexa"); + + LOGGER.info("mouth path is /enexa"); + container.setVolumeMounts(Arrays.asList(volumeMount)); // Create a PodSpec and add the shared volume and container @@ -94,24 +229,128 @@ public String startContainerKub(String image, String podName, List podClient = new GenericKubernetesApi<>(V1Pod.class, V1PodList.class, "", + // Create a Pod + pod.setMetadata(new V1ObjectMeta().name(podName).namespace(nameSpace).labels(new HashMap(){{ + put("app",container.getName()); + }})); + pod.setSpec(podSpec); + try { + GenericKubernetesApi podClient = new GenericKubernetesApi<>(V1Pod.class, V1PodList.class, "", "v1", "pods", client); - try { + pod.getSpec().getContainers().get(0).setEnv(env); // add to the first container because we assume runnig one container in each pod + V1Pod latestPod = podClient.create(pod).throwsApiException().getObject(); - return latestPod.getMetadata().getName(); + String latestPodUid = latestPod.getMetadata().getUid(); + LOGGER.info("latestPodUID : {}", latestPodUid); + return latestPodUid; } catch (ApiException e) { - LOGGER.error("Got an exception while trying to create an instance of \"" + image + "\". Returning null.", - e); + LOGGER.error("Got an exception while trying to create an instance of \"{}\". Returning null.", replacedImage, e); return null; } } + /** + * Creates environment variables from the provided list of key-value pairs. + */ + private List createEnvVariables(List> variables) { + List env = new ArrayList<>(); + if (variables != null) { + for (Map.Entry entry : variables) { + V1EnvVar v1env = new V1EnvVar(); + v1env.setName(entry.getKey()); + v1env.setValue(entry.getValue()); + env.add(v1env); + } + } + return env; + } + + /** + * Extracts the experiment IRI from the environment variables list. + */ + private String extractExperimentIRI(List> variables) { + for (AbstractMap.SimpleEntry v : variables) { + if ("ENEXA_EXPERIMENT_IRI".equals(v.getKey())) { + return v.getValue(); + } + } + return ""; + } + + + /** + * Builds the host path using the shared directory path and app name. + */ + private String buildHostPath(String hostSharedDirectory, String appName) { + LOGGER.info("build host path"); + return makeTheDirectoryInThisPath(hostSharedDirectory, appName); + } + + /** + * Creates a writable path for the container based on the experiment IRI. + */ + private String buildWritableContainerPath(String expIRI, String hostBasePath) { + String writeableDirectory = expIRI.split("/")[expIRI.split("/").length - 1]; + LOGGER.info("build writeable path at {}", writeableDirectory); + return makeTheDirectoryInThisPath(hostBasePath, writeableDirectory); + } + + /** + * Adds environment variables related to shared directories to the list of env variables. + */ + private void addSharedDirectoryEnvVars(List env, String containerBasePath, String containerModuleInstancePath, + String containerWritablePath) { + env.add(new V1EnvVar().name("ENEXA_SHARED_DIRECTORY").value(containerBasePath)); + env.add(new V1EnvVar().name("ENEXA_MODULE_INSTANCE_DIRECTORY").value(containerModuleInstancePath)); + env.add(new V1EnvVar().name("ENEXA_WRITEABLE_DIRECTORY").value(containerWritablePath)); + } + + /** + * Creates a volume for the shared directory. + */ + private V1Volume createVolume() { + V1PersistentVolumeClaimVolumeSource persistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource(); + persistentVolumeClaim.setClaimName("enexa-shared-dir-claim"); + + V1Volume volume = new V1Volume(); + volume.setName("enexa-shared-dir"); + volume.setPersistentVolumeClaim(persistentVolumeClaim); + + return volume; + } + + /** + * Creates the resource requirements for the container. + */ + private V1ResourceRequirements createResourceRequirements(Optional memoryValue) { + V1ResourceRequirements resourceRequirements = new V1ResourceRequirements(); + Map requests = new HashMap<>(); + requests.put("memory", new Quantity(memoryInGiB + "Gi")); + + resourceRequirements.setRequests(requests); + return resourceRequirements; + } + + /** + * Creates a container with the given image, command, environment variables, and resource requirements. + */ + private V1Container createContainer(String image, String[] command, List env, V1ResourceRequirements resourceRequirements) { + V1Container container = new V1Container(); + String containerName = image.replace("/", "-").replace(".", "").replace(":", "").toLowerCase(); + container.setName(containerName); + container.setImage(image); + container.setResources(resourceRequirements); + + if (command != null) { + container.setCommand(Arrays.asList(command)); + } + + container.setEnv(env); + return container; + } @Override public String stopContainer(String containerId) { @@ -121,22 +360,132 @@ public String stopContainer(String containerId) { @Override public String getContainerStatus(String podName) { - CoreV1Api api = new CoreV1Api(client); + LOGGER.info("client is base path:{} timeout : {}", client.getBasePath(), client.getConnectTimeout()); + GenericKubernetesApi podClient = new GenericKubernetesApi<>( + V1Pod.class, V1PodList.class, "", "v1", "pods", client); + LOGGER.info("GenericKubernetesApi for Pods initiated "); try { - V1PodList list = api.listNamespacedPod("default", "false", null, null, null, null, null, null, null, null, - null); - for (V1Pod pod : list.getItems()) { - if (pod.getMetadata().getName().equals(podName)) { - return pod.getStatus().getPhase(); + // List all pods in the specified namespace + KubernetesApiResponse response = podClient.list(nameSpace); + if (response.isSuccess()) { + V1PodList podList = response.getObject(); + LOGGER.info("the list of pods size is {}", podList.getItems().size()); + LOGGER.info("looking for {}", podName); + for (V1Pod pod : podList.getItems()) { + LOGGER.info("podName : {}", pod.getMetadata().getName()); + if (pod.getMetadata().getName().equals(podName)) { + return pod.getStatus().getPhase(); + } } + } else { + LOGGER.error("Failed to list pods. Status code: " + response.getHttpStatusCode()); + return null; } - } catch (ApiException e) { + } catch (Exception e) { LOGGER.error("Got an exception while trying to get the status of \"" + podName + "\". Returning null.", e); return null; } return null; } + +@Override +public Map resolveContainerEndpoint(String containerId, Integer port){ + LOGGER.info("start resolving the IP from ID: {}",containerId); + Map endpointMap = new HashMap<>(); + + GenericKubernetesApi podClient = new GenericKubernetesApi<>( + V1Pod.class, V1PodList.class, "", "v1", "pods", client); + + +try { + // List all pods (you can filter by namespace if needed, e.g., "default" namespace) + V1PodList podList = podClient.list(nameSpace).getObject(); + if (podList == null || podList.getItems().isEmpty()) { + LOGGER.error("No pods found in the '{}' namespace.",nameSpace); + return null; + } + LOGGER.info("Number of pods retrieved: {}", podList.getItems().size()); + + V1Pod targetPod = podList.getItems().stream() + .filter(pod -> containerId.equals(pod.getMetadata().getUid())) + .findFirst() + .orElse(null); + + if (targetPod == null) { + LOGGER.warn("No pod found with UID: {}", containerId); + return null; + } + LOGGER.info("Target pod found: {}", targetPod.getMetadata().getName()); + +// Wait for the pod to start running (with a timeout of 60 seconds) + long startTime = System.currentTimeMillis(); + while (true) { + LOGGER.info("checking the status of the pod"); + // Check if the pod is in the 'Running' state + String podPhase = targetPod.getStatus().getPhase(); + LOGGER.info("podPhase : {}", podPhase); + if ("Running".equals(podPhase)) { + // Create the service as a Nodeport + // get new port + // make complete endpoint andk return it + LOGGER.info("pod is running"); + Map pod_labels = targetPod.getMetadata().getLabels(); + if(pod_labels==null){ + LOGGER.error("there is no label in a pod to make selector ! podname is "+containerId); + }else{ + LOGGER.info("pod_labels size: " + pod_labels.size()); + } + String podIp = targetPod.getStatus().getPodIP(); + String HostIP = targetPod.getStatus().getHostIP(); + LOGGER.info("status is {}",targetPod.getStatus().toString()); + Map serviceSpecs = getServiceAsNodePort(containerId,port,pod_labels); + + if(serviceSpecs==null){ + LOGGER.error("could not get a port for this"); + return null; + }else { + String serviceSpecsMapAsString = serviceSpecs.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(", ")); + + LOGGER.info(serviceSpecsMapAsString); + + endpointMap.put("internalEndpointURL",podIp + ":" + port); + endpointMap.put("hostIP",HostIP); + endpointMap.put("externalEndpointURL",serviceSpecs.get("clusterIP") + ":" + serviceSpecs.get("port")); + + String endpointMapAsString = endpointMap.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(", ")); + + LOGGER.info(endpointMapAsString); + + return endpointMap; + } + } + + + if (System.currentTimeMillis() - startTime > 180 * 1000) { + LOGGER.error("Pod with UID {} did not start running within 180 seconds", containerId); + return null; + } + + // Sleep for a short interval before checking again + Thread.sleep(5000); // Sleep for 2 seconds + + // Re-fetch the pod to update its status + targetPod = podClient.get("default", targetPod.getMetadata().getName()) + .throwsApiException() + .getObject(); + } + +}catch (Exception ex){ + LOGGER.error("Failed to resolve endpoint{}", ex.getMessage()); + return null; +} +} + public static ContainerManagerImpl create() throws IOException { ApiClient client = Config.defaultClient(); Configuration.setDefaultApiClient(client); @@ -145,7 +494,7 @@ public static ContainerManagerImpl create() throws IOException { public static void main(String[] args) throws Exception { ContainerManagerImpl manager = ContainerManagerImpl.create(); - String containerId = manager.startContainerKub("busybox", "test" + UUID.randomUUID().toString(), null, null,new String[] { "sleep", "10000" }); + String containerId = manager.startContainerKub("busybox", "test" + UUID.randomUUID().toString(), null, null,new String[] { "sleep", "10000" },null,new HashMap<>()); System.out.println(containerId); String status = null; diff --git a/src/main/java/eu/enexa/model/ModuleModel.java b/src/main/java/eu/enexa/model/ModuleModel.java index 374577b..ddbda26 100644 --- a/src/main/java/eu/enexa/model/ModuleModel.java +++ b/src/main/java/eu/enexa/model/ModuleModel.java @@ -7,6 +7,7 @@ public class ModuleModel { private String moduleIri; private String moduleUrl; private String image; + private Integer port; private Model model; /** @@ -45,6 +46,20 @@ public String getImage() { public void setImage(String image) { this.image = image; } + + /** + * @return the port + */ + public Integer getPort() { + return port; + } + /** + * @param port the port to set if it is null the module expose no port + */ + public void setPort(Integer port) { + this.port = port; + } + /** * @return the model */ @@ -58,13 +73,10 @@ public void setModel(Model model) { this.model = model; } - @Override + + public String toString() { - return "ModuleModel{" + - "moduleIri='" + moduleIri + '\'' + - ", moduleUrl='" + moduleUrl + '\'' + - ", image='" + image + '\'' + - ", model=" + model + - '}'; + return moduleIri + " - " + moduleUrl + " - " + image + " - " + model.toString(); + } } diff --git a/src/main/java/eu/enexa/model/ModuleNotFoundException.java b/src/main/java/eu/enexa/model/ModuleNotFoundException.java index 8bfbefa..b994b5d 100644 --- a/src/main/java/eu/enexa/model/ModuleNotFoundException.java +++ b/src/main/java/eu/enexa/model/ModuleNotFoundException.java @@ -2,7 +2,7 @@ public class ModuleNotFoundException extends Exception { - private static final long serialVersionUID = 1L; + //private static final long serialVersionUID = 1L; public ModuleNotFoundException(StartContainerModel scModel) { super("Module with IRI " + scModel.getModuleIri() + " could not be found."); diff --git a/src/main/java/eu/enexa/module/FileBasedModuleManager.java b/src/main/java/eu/enexa/module/FileBasedModuleManager.java index 95841a8..5f4cba6 100644 --- a/src/main/java/eu/enexa/module/FileBasedModuleManager.java +++ b/src/main/java/eu/enexa/module/FileBasedModuleManager.java @@ -6,11 +6,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import org.apache.commons.compress.utils.FileNameUtils; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.*; import org.apache.jena.util.FileUtils; import org.dice_research.enexa.vocab.ENEXA; import org.dice_research.enexa.vocab.HOBBIT; @@ -40,12 +39,15 @@ public FileBasedModuleManager() throws IOException { }else { File enexaModuleDirectory = new File(path); if (!enexaModuleDirectory.exists()) { - LOGGER.error(enexaModuleDirectory.getAbsolutePath() + " is not exist"); + LOGGER.error("{} is not exist", enexaModuleDirectory.getAbsolutePath()); //throw new IOException(path + " is not exist"); } else { - LOGGER.info(enexaModuleDirectory.getAbsolutePath() + " exist"); + LOGGER.info("{} exist", enexaModuleDirectory.getAbsolutePath()); addFileOrDirectory(enexaModuleDirectory); - LOGGER.info("modules size is : "+ modules.size()); + LOGGER.info("modules size is : {}", modules.size()); +// for (ModuleModel module : modules.values()) { +// LOGGER.info(" -{}", module.toString()); +// } } } } @@ -72,7 +74,7 @@ public void addFileOrDirectory(File moduleFile) throws IOException { } // If it is a directory, iterate over its content. if (moduleFile.isDirectory()) { - for (File file : moduleFile.listFiles()) { + for (File file : Objects.requireNonNull(moduleFile.listFiles())) { addFileOrDirectory(file); } } else { @@ -115,12 +117,21 @@ public void addFileOrDirectory(File moduleFile) throws IOException { } } + // find if it exposes a Port + //TODO use version 0.0.3 of enexa java utils after merged with develop branch and remove hardcoded part "exposes_port" + Literal port = RdfHelper.getLiteral(model, enexaModule, ResourceFactory.createProperty("http://w3id.org/dice-research/enexa/ontology#", "exposes_port")); + + // create the module representation and add the values ModuleModel moduleModel = new ModuleModel(); moduleModel.setModel(model); moduleModel.setModuleIri(enexaModule.getURI()); moduleModel.setImage(image.getURI()); + if(port!=null){ + moduleModel.setPort(port.getInt()); + } + // Add it to the internal index modules.put(moduleModel.getModuleIri(), moduleModel); } else { diff --git a/src/main/java/eu/enexa/service/ContainerManager.java b/src/main/java/eu/enexa/service/ContainerManager.java index aff9462..32656b8 100644 --- a/src/main/java/eu/enexa/service/ContainerManager.java +++ b/src/main/java/eu/enexa/service/ContainerManager.java @@ -1,7 +1,9 @@ package eu.enexa.service; +import java.io.File; import java.util.AbstractMap; import java.util.List; +import java.util.Map; /** * The interface of a manager for containers that can be started and stopped by @@ -13,7 +15,7 @@ public interface ContainerManager { // containerName is the podName for kubernetes - String startContainer(String image, String containerName, List> variables, String hostSharedDirectory, String appName); + String startContainer(String image, String containerName, List> variables, String hostSharedDirectory, String appName, Map containerSettings); /** * Stop the container with the given ID. @@ -32,4 +34,46 @@ public interface ContainerManager { * container does not exist */ String getContainerStatus(String containerId); + + /** + * + * @param containerId + * @param port + * @return url as callable endpoint from outside of the container + */ + Map resolveContainerEndpoint(String containerId, Integer port); + + /** + * Combines two path components to create a valid path. + * the directory will create if not exist + * + * @param partOneOfPath The first part of the path. + * @param partTwoOfPath The second part of the path. + * @return The combined path. + */ + default String makeTheDirectoryInThisPath(String partOneOfPath, String partTwoOfPath) { + if(partOneOfPath ==null && partTwoOfPath == null) return ""; + assert partOneOfPath != null; + String path = combinePaths(partOneOfPath, partTwoOfPath); + File appPathDirectory = new File(path); + if(!appPathDirectory.exists()){ + appPathDirectory.mkdirs(); + } + return path; + } + + /** + * Combines two path components to create a valid path, taking into account trailing separators. + * + * @param partOne The first part of the path. + * @param partTwo The second part of the path. + * @return The combined path. + */ + default String combinePaths(String partOne, String partTwo) { + String path = partOne + File.separator + partTwo; + if (partOne.endsWith(File.separator)) { + path = partOne + partTwo; + } + return path; + } } diff --git a/src/main/java/eu/enexa/service/EnexaServiceImpl.java b/src/main/java/eu/enexa/service/EnexaServiceImpl.java index 15ea4f1..379c726 100644 --- a/src/main/java/eu/enexa/service/EnexaServiceImpl.java +++ b/src/main/java/eu/enexa/service/EnexaServiceImpl.java @@ -1,18 +1,15 @@ package eu.enexa.service; import java.io.File; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.List; - -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Property; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; + +import org.apache.jena.rdf.model.*; import org.apache.jena.vocabulary.RDF; import org.dice_research.enexa.utils.EnexaPathUtils; import org.dice_research.enexa.vocab.ENEXA; +import org.dice_research.enexa.vocab.HOBBIT; import org.dice_research.rdf.ModelHelper; import org.dice_research.rdf.RdfHelper; import org.slf4j.Logger; @@ -41,7 +38,6 @@ public class EnexaServiceImpl implements EnexaService { private static final String metaDataEndpoint = System.getenv("ENEXA_META_DATA_ENDPOINT"); - // If service need used by different applications then this should set from outside ( with a setter) @Value("${enexa.appName}") private String appName ; private static final String sharedDirectory = System.getenv("ENEXA_SHARED_DIRECTORY"); @@ -70,7 +66,6 @@ public Model startExperiment() { } } // 3. Start default containers - // TODO : implement this // 4. Update experiment meta data with data from steps 2 and 3 model.add(experiment, RDF.type, ENEXA.Experiment); @@ -82,17 +77,17 @@ public Model startExperiment() { * graph IRI in which the metadata of the experiment can be found. */ - String[] metaDataInfos = metadataManager.getMetadataEndpointInfo(experimentIRI); - if (metaDataInfos.length == 0) { + Map metaDataInfos = metadataManager.getMetadataEndpointInfo(); + if (metaDataInfos.isEmpty() || (metaDataInfos.get("sparqlEndpointUrl") == null && metaDataInfos.get("defaultMetaDataGraphIRI") == null)) { LOGGER.error("there is no data in metadata for this experiments: " + experimentIRI); } else { - Property sparqlEndpoint = ResourceFactory.createProperty(metaDataInfos[0]); + Property sparqlEndpoint = ResourceFactory.createProperty(metaDataInfos.get("sparqlEndpointUrl")); model.add(experiment, ENEXA.metaDataEndpoint, sparqlEndpoint); - if (metaDataInfos.length > 1) { + if (metaDataInfos.get("defaultMetaDataGraphIRI") != null) { // graphIRI // TODO : check if ENEXA.metaDataGraph is correct - Property graphIRI = ResourceFactory.createProperty(metaDataInfos[1]); + Property graphIRI = ResourceFactory.createProperty(metaDataInfos.get("defaultMetaDataGraphIRI")); model.add(experiment, ENEXA.metaDataGraph, graphIRI); } } @@ -105,12 +100,12 @@ public Model startExperiment() { @Override public Model getMetadataEndpoint(String experimentIri) { // todo : check if experimentIri is not valid then return error , or the check should be done on calling this method - String[] endpointInfo = metadataManager.getMetadataEndpointInfo(experimentIri); + Map endpointInfo = metadataManager.getMetadataEndpointInfo(); Model model = ModelFactory.createDefaultModel(); Resource experimentResource = model.createResource(experimentIri); model.add(experimentResource, RDF.type, ENEXA.Experiment); - model.add(experimentResource, ENEXA.metaDataEndpoint, model.createResource(endpointInfo[0])); - model.add(experimentResource, ENEXA.metaDataGraph, model.createResource(endpointInfo[1])); + model.add(experimentResource, ENEXA.metaDataEndpoint, model.createResource(endpointInfo.get("sparqlEndpointUrl"))); + model.add(experimentResource, ENEXA.metaDataGraph, model.createResource(endpointInfo.get("defaultMetaDataGraphIRI"))); return model; } @@ -161,14 +156,14 @@ public Model startContainer(StartContainerModel scModel) throws ModuleNotFoundEx variables.add(new AbstractMap.SimpleEntry<>("ENEXA_META_DATA_ENDPOINT", metaDataEndpoint)); variables.add(new AbstractMap.SimpleEntry<>("ENEXA_META_DATA_GRAPH", - metadataManager.getMetadataEndpointInfo(scModel.getExperiment())[1])); + metadataManager.getMetadataEndpointInfo().get("defaultMetaDataGraphIRI"))); variables.add(new AbstractMap.SimpleEntry<>("ENEXA_MODULE_IRI", instanceIri)); variables.add(new AbstractMap.SimpleEntry<>("ENEXA_MODULE_INSTANCE_IRI", scModel.getInstanceIri())); // TODO: update this - if (System.getenv("ENEXA_SERVICE_URL").equals("")) { + if (System.getenv("ENEXA_SERVICE_URL").isEmpty()) { LOGGER.error("ENEXA_SERVICE_URL environment is null"); } else { LOGGER.info("ENEXA_SERVICE_URL is : " + System.getenv("ENEXA_SERVICE_URL")); @@ -176,18 +171,32 @@ public Model startContainer(StartContainerModel scModel) throws ModuleNotFoundEx variables.add(new AbstractMap.SimpleEntry<>("ENEXA_SERVICE_URL", System.getenv("ENEXA_SERVICE_URL"))); String containerName = generatePodName(module.getModuleIri()); - String containerId = containerManager.startContainer(module.getImage(), containerName, variables, sharedDirectory, appName); - // TODO take point in time + Map containerSettings = new HashMap<>(); + String containerId = containerManager.startContainer(module.getImage(), containerName, variables, sharedDirectory, appName, containerSettings); + + - /* - * 4. Add start time (or error code in case it couldn’t be started) to the TODO - * create RDF model with new metadata metadataManager.addMetaData(null); - */ Model createdContainerModel = scModel.getModel(); Resource instanceRes = createdContainerModel.getResource(instanceIri); createdContainerModel.add(instanceRes, ENEXA.containerId, containerId); - createdContainerModel.add(instanceRes, ENEXA.containerName, containerName); - // TODO add start time + + + // when port is not null it means container provide an endpoint + if(module.getPort()!=null){ + LOGGER.info("the port exist for this module the map port: {}", module.getPort().toString()); + Map containerProvidedEndpoint = containerManager.resolveContainerEndpoint(containerId, module.getPort()); + createdContainerModel.add(instanceRes, ENEXA.externalEndpointURL, containerProvidedEndpoint.get("externalEndpointURL")); + createdContainerModel.add(instanceRes, ENEXA.internalEndpointURL, containerProvidedEndpoint.get("internalEndpointURL")); + } + + /* + * 4. Add start time (or error code in case it couldn’t be started) to the + * create RDF model with new metadata metadataManager.addMetaData(null); + */ + LocalDateTime currentDateTime = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); + String dateTimeString = currentDateTime.format(formatter); + createdContainerModel.add(instanceRes, HOBBIT.startTime, dateTimeString); metadataManager.addMetaData(createdContainerModel); @@ -204,7 +213,9 @@ public String generatePodName(String moduleIri) { * messageDigest.update(moduleIri.getBytes()); String stringHash = new * String(messageDigest.digest()); */ - return "enexa_" + Integer.toString(moduleIri.hashCode() * 31 + (int) (System.currentTimeMillis())); + + return "enexa-" + (moduleIri.hashCode() * 31 + (int) (System.currentTimeMillis())); + } @Override @@ -226,14 +237,18 @@ public AddedResource addResource(String experimentIri, String resource, String t @Override public Model containerStatus(String experimentIri, String instanceIRI) { - LOGGER.info("experimentIri is : "+ experimentIri); - LOGGER.info("instanceIRI is : "+ instanceIRI); + LOGGER.info(">> get container status"); + LOGGER.info("experimentIri is : {}", experimentIri); + LOGGER.info("instanceIRI is : {}", instanceIRI); // Query container / pod name String podName = metadataManager.getContainerName(experimentIri, instanceIRI); - LOGGER.info("pod/container name is : "+ podName); + if(podName == null){ + LOGGER.error(" could not find the pod name !!!"); + } + LOGGER.info("pod/container name is : {}", podName); // Get status String status = containerManager.getContainerStatus(podName); - LOGGER.info("container status is : "+ status); + LOGGER.info("container status is : {}", status); Model result = ModelFactory.createDefaultModel(); Resource instance = result.createResource(instanceIRI); result.add(instance, ENEXA.experiment, result.createResource(experimentIri)); @@ -243,9 +258,10 @@ public Model containerStatus(String experimentIri, String instanceIRI) { @Override public Model stopContainer(String experimentIri, String containerID) { + //Model model = ModelFactory.createDefaultModel(); - String resultOfStoppingTheContainer = containerManager.stopContainer(containerID); - LOGGER.info(resultOfStoppingTheContainer); + //String ResultOfStoppingTheContainer = containerManager.stopContainer(containerID); + Model result = ModelFactory.createDefaultModel(); // TODO : check this part do we need an IRI or ID ? Resource instance = result.createResource(containerID); @@ -258,12 +274,14 @@ public Model finishExperiment(String experimentIri) { // finishes the experiment with the given IRI by stopping all its remaining // containers. // list of all containers + // TODO : read from meta data or use labels ( we use meta data for now) get List containerNames = metadataManager.getAllContainerNames(experimentIri); + Model result = ModelFactory.createDefaultModel(); for(String containerName : containerNames){ String resultOfStop = containerManager.stopContainer(containerName); - LOGGER.info(containerName+ " stopped result is : "+ resultOfStop); + LOGGER.info("{} stopped result is : {}", containerName, resultOfStop); Resource instance = result.createResource(containerName); result.add(instance, ENEXA.containerName, result.createResource(experimentIri)); } diff --git a/src/main/java/eu/enexa/service/MetadataManager.java b/src/main/java/eu/enexa/service/MetadataManager.java index 2298284..cbc69e7 100644 --- a/src/main/java/eu/enexa/service/MetadataManager.java +++ b/src/main/java/eu/enexa/service/MetadataManager.java @@ -3,18 +3,21 @@ import org.apache.jena.rdf.model.Model; import java.util.List; +import java.util.Map; public interface MetadataManager { /** - * Returns IRIs that are necessary to access an experiment's metadata. The first - * String is the URL of the SPARQL endpoint while the second is the graph IRI in + * Returns IRIs that are necessary to access an experiment's metadata.It returns a Map + * with these keys , "sparqlEndpointUrl" and "defaultMetaDataGraphIRI" , The "sparqlEndpointUrl" + * is the URL of the SPARQL endpoint while the "defaultMetaDataGraphIRI" is the graph IRI in * which the metadata of the experiment can be found. * - * @param experimentIri * @return */ - String[] getMetadataEndpointInfo(String experimentIri); + + public Map getMetadataEndpointInfo(); + /** * Returns a random IRI that can be used within the metadata graph to name diff --git a/src/main/java/eu/enexa/service/web/EnexaController.java b/src/main/java/eu/enexa/service/web/EnexaController.java index 81648e2..097be4a 100644 --- a/src/main/java/eu/enexa/service/web/EnexaController.java +++ b/src/main/java/eu/enexa/service/web/EnexaController.java @@ -15,7 +15,6 @@ import org.apache.jena.riot.RDFLanguages; import org.apache.jena.riot.WebContent; import org.apache.jena.vocabulary.RDF; -import org.dice_research.enexa.vocab.Algorithm; import org.dice_research.enexa.vocab.ENEXA; import org.dice_research.rdf.RdfHelper; import org.dice_research.rdf.spring_jena.Jena2SpringUtils; @@ -30,7 +29,6 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -208,7 +206,7 @@ public ResponseEntity stopContainer(@RequestBody Model request) { } - @GetMapping(value = "/meta") + @GetMapping(value = "/meta",consumes = "*/*") public ResponseEntity meta(@RequestParam String experimentIRI) { /* * Errors · HTTP 400: o Experiment IRI is not known / not available. o The diff --git a/src/main/java/eu/enexa/sparql/SparqlBasedMetadataManager.java b/src/main/java/eu/enexa/sparql/SparqlBasedMetadataManager.java index 604c30b..dfe6c81 100644 --- a/src/main/java/eu/enexa/sparql/SparqlBasedMetadataManager.java +++ b/src/main/java/eu/enexa/sparql/SparqlBasedMetadataManager.java @@ -1,18 +1,14 @@ package eu.enexa.sparql; import java.net.http.HttpClient; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; import org.aksw.jena_sparql_api.core.UpdateExecutionFactoryHttp; import org.aksw.jena_sparql_api.http.QueryExecutionFactoryHttp; import org.aksw.jena_sparql_api.pagination.core.QueryExecutionFactoryPaginated; import org.aksw.jenax.arq.connection.core.QueryExecutionFactory; import org.aksw.jenax.arq.connection.core.UpdateExecutionFactory; -import org.apache.jena.query.QueryExecution; -import org.apache.jena.query.QuerySolution; -import org.apache.jena.query.ResultSet; +import org.apache.jena.query.*; import org.apache.jena.rdf.model.Model; import org.apache.jena.sparql.core.DatasetDescription; import org.apache.jena.update.UpdateProcessor; @@ -58,9 +54,6 @@ public class SparqlBasedMetadataManager implements MetadataManager, AutoCloseabl */ private UpdateExecutionFactory updateExecFactory = null; -// public SparqlBasedMetadataManager(@Value("${ENEXA_META_DATA_ENDPOINT}") String sparqlEndpointUrl) { -// this(sparqlEndpointUrl, DEFAULT_META_DATA_GRAPH_IRI, DEFAULT_RESOURCE_NAMESPACE); -// } public SparqlBasedMetadataManager(@Value("${ENEXA_META_DATA_ENDPOINT}") String sparqlEndpointUrl, @Value("${ENEXA_META_DATA_GRAPH}") String defaultMetaDataGraphIRI, @@ -68,7 +61,10 @@ public SparqlBasedMetadataManager(@Value("${ENEXA_META_DATA_ENDPOINT}") String s this.sparqlEndpointUrl = sparqlEndpointUrl; this.defaultMetaDataGraphIRI = defaultMetaDataGraphIRI; this.resourceNamespace = resourceNamespace; - + LOGGER.info("initiate query execute"); + LOGGER.info(" sparqlEndpointUrl is: {}",sparqlEndpointUrl); + LOGGER.info(" defaultMetaDataGraphIRI is{}: ",defaultMetaDataGraphIRI); + LOGGER.info(" resourceNamespace is: {}",resourceNamespace); HttpClient client = HttpClient.newHttpClient(); DatasetDescription desc = new DatasetDescription(); desc.addNamedGraphURI(defaultMetaDataGraphIRI); @@ -78,14 +74,23 @@ public SparqlBasedMetadataManager(@Value("${ENEXA_META_DATA_ENDPOINT}") String s } @Override - public String[] getMetadataEndpointInfo(String experimentIri) { - return new String[] { sparqlEndpointUrl, defaultMetaDataGraphIRI }; + public Map getMetadataEndpointInfo() { + Map info = new HashMap<>(); + if(sparqlEndpointUrl == null) { + LOGGER.warn("sparqlEndpointUrl is null"); + } + if(defaultMetaDataGraphIRI == null) { + LOGGER.warn("defaultMetaDataGraphIRI is null"); + } + + info.put("sparqlEndpointUrl",sparqlEndpointUrl); + info.put("defaultMetaDataGraphIRI",defaultMetaDataGraphIRI); + return info; } @Override public String generateResourceIRI() { - String resourceIri = resourceNamespace + UUID.randomUUID().toString(); - return resourceIri; + return resourceNamespace + UUID.randomUUID(); } @Override @@ -94,17 +99,15 @@ public void addMetaData(Model model) { String[] queries = SparqlQueryUtils.getUpdateQueriesFromDiff(null, model, defaultMetaDataGraphIRI); UpdateProcessor update; for (String query : queries) { - //TODO if work should not be hardcoded - //query = query.replace("WITH ",""); - LOGGER.info("query is :"+query); - //query = query.replace("INSERT","INSERT DATA").replace("WHERE","").replace("{}",""); - //LOGGER.info("after replacing the query where clause :"+query); - update = updateExecFactory.createUpdateProcessor(query); - update.execute(); - //todo what happened if update can not find a triple to update ! + try { + update = updateExecFactory.createUpdateProcessor(query); + update.execute(); + }catch (Exception ex){ + LOGGER.error("Error executing query: {}", query, ex); + } } }catch (Exception ex){ - LOGGER.error(ex.getMessage()); + LOGGER.error("error happened",ex); } } @@ -124,8 +127,12 @@ public void close() throws Exception { @Override public String getContainerName(String experimentIri, String instanceIRI) { - QueryExecution qe = queryExecFactory.createQueryExecution("SELECT ?name FROM <"+defaultMetaDataGraphIRI+"> WHERE {" + "<" + instanceIRI - + "> ?name }"); + //LOGGER.info("getting container name "); + String query = "SELECT ?name FROM <"+defaultMetaDataGraphIRI+"> WHERE {" + "<" + instanceIRI + + "> ?name }"; + LOGGER.info(" - query is: {}", query); + QueryExecution qe = queryExecFactory.createQueryExecution(query); + ResultSet rs = qe.execSelect(); if (rs.hasNext()) { QuerySolution qs = rs.next(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0d844fc..ec145c1 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,4 +1,9 @@ -enexa.appName=app2 -##server.port=8081 +enexa.appName=app3 +server.port=8081 +container.manager.namespace=default +container.manager.timeoutMilliSeconds=60000 +container.manager.defaultMemorySize=8 +#spring.profiles.active=docker +spring.profiles.active=kubernetes docker.traefik.hostname=enexa-demo.cs.uni-paderborn.de docker.traefik.loadbalancer.port="8501" diff --git a/src/test/java/eu/enexa/docker/ContainerManagerImplTest.java b/src/test/java/eu/enexa/docker/ContainerManagerImplTest.java index 7bcd9df..987c7e6 100644 --- a/src/test/java/eu/enexa/docker/ContainerManagerImplTest.java +++ b/src/test/java/eu/enexa/docker/ContainerManagerImplTest.java @@ -16,11 +16,7 @@ import org.springframework.test.context.junit4.SpringRunner; import java.io.IOException; -import java.util.AbstractMap; -import java.util.List; -import java.util.UUID; -import java.util.ArrayList; -import java.util.LinkedList; +import java.util.*; import java.util.concurrent.TimeUnit; @@ -39,7 +35,7 @@ public void testStartContainerWithVariables() throws IOException, InterruptedExc List> variables = new ArrayList<>(); variables.add(new AbstractMap.SimpleEntry<>("FIRST_VARIABLE","FIRST_VARIABLE_VALUE")); variables.add(new AbstractMap.SimpleEntry<>("SECOND_VARIABLE","SECOND_VARIABLE_VALUE")); - String containerId = cm.startContainer(imageName, name, variables,System.getProperty("user.dir"),null); + String containerId = cm.startContainer(imageName, name, variables,System.getProperty("user.dir"),null, new HashMap<>()); Assert.assertTrue(containerId.length() > 5); //TODO : need removed after test } @@ -66,7 +62,7 @@ public void containerStatusShouldReturnStatusForExistContainers() throws Interru String name = "test"+ UUID.randomUUID().toString(); List> variables = new LinkedList<>(); variables.add(new AbstractMap.SimpleEntry<>("ENEXA_EXPERIMENT_IRI", "testEXPRIMENTID12341234")); - String containerId = cm.startContainer(imageName, name, variables,System.getProperty("user.dir"),null); + String containerId = cm.startContainer(imageName, name, variables,System.getProperty("user.dir"),null,new HashMap<>()); // wait to container exit TimeUnit.SECONDS.sleep(5); String status = cm.getContainerStatus(containerId); diff --git a/src/test/java/eu/enexa/kubernetes/ContainerManagerImplTest.java b/src/test/java/eu/enexa/kubernetes/ContainerManagerImplTest.java index eb7a82c..fe9445c 100644 --- a/src/test/java/eu/enexa/kubernetes/ContainerManagerImplTest.java +++ b/src/test/java/eu/enexa/kubernetes/ContainerManagerImplTest.java @@ -12,10 +12,7 @@ import org.springframework.test.context.junit4.SpringRunner; import java.io.IOException; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; /*@RunWith(SpringRunner.class) @SpringBootTest(classes = BootApplication.class)*/ @@ -27,7 +24,7 @@ public void testStartContainer() throws IOException { Configuration.setDefaultApiClient(client); ContainerManagerImpl cm = new ContainerManagerImpl(client); String name = "test"+ UUID.randomUUID().toString(); - String posName = cm.startContainer(imageName,name,null,null,null); + String posName = cm.startContainer(imageName,name,null,null,null,new HashMap<>()); Assert.assertEquals(posName, name); //TODO : need this pod removed after test } @@ -42,7 +39,7 @@ public void testStartContainerWithVariables() throws IOException { List> variables = new ArrayList<>(); variables.add(new AbstractMap.SimpleEntry<>("FIRST_VARIABLE","FIRST_VARIABLE_VALUE")); variables.add(new AbstractMap.SimpleEntry<>("SECOND_VARIABLE","SECOND_VARIABLE_VALUE")); - String posName = cm.startContainer(imageName, name, variables, null,null); + String posName = cm.startContainer(imageName, name, variables, null,null,new HashMap<>()); Assert.assertEquals(posName, name); //TODO : need this pod removed after test }