Merge pull request #4585 from IQSS/4396-support-multiple-file-storage…

…-locations

4396 support multiple file storage locations

doc/sphinx-guides/source/developers/big-data-support.rst

@@ -155,7 +155,29 @@ TODO: Document these in the Installation Guide once they're final.
 
 To specify replication sites that appear in rsync URLs:
 
-``curl http://localhost:8080/api/admin/settings/:ReplicationSites -X PUT -d "dv.sbgrid.org:Harvard Medical School:USA,sbgrid.icm.uu.se:Uppsala University:Sweden,sbgrid.ncpss.org:Institut Pasteur de Montevideo:Uruguay,sbgrid.ncpss.org:Shanghai Institutes for Biological Sciences:China"``
+Download :download:`add-storage-site.json <../../../../scripts/api/data/storageSites/add-storage-site.json>` and adjust it to meet your needs. The file should look something like this:
+
+.. literalinclude:: ../../../../scripts/api/data/storageSites/add-storage-site.json
+
+Then add the storage site using curl:
+
+``curl -H "Content-type:application/json" -X POST http://localhost:8080/api/admin/storageSites --upload-file add-storage-site.json``
+
+You make a storage site the primary site by passing "true". Pass "false" to make it not the primary site. (id "1" in the example):
+
+``curl -X PUT -d true http://localhost:8080/api/admin/storageSites/1/primaryStorage``
+
+You can delete a storage site like this (id "1" in the example):
+
+``curl -X DELETE http://localhost:8080/api/admin/storageSites/1``
+
+You can view a single storage site like this: (id "1" in the example):
+
+``curl http://localhost:8080/api/admin/storageSites/1``
+
+You can view all storage site like this:
+
+``curl http://localhost:8080/api/admin/storageSites``
 
 In the GUI, this is called "Local Access". It's where you can compute on files on your cluster.
 

scripts/api/data/storageSites/add-storage-site.json

@@ -0,0 +1,6 @@
+{
+  "hostname": "dataverse.librascholar.edu",
+  "name": "LibraScholar, USA",
+  "primaryStorage": true,
+  "transferProtocols": "rsync,posix,globus"
+}

src/main/java/edu/harvard/iq/dataverse/DvObject.java

@@ -110,6 +110,17 @@ public String visit(DataFile df) {
     @Column
     private String storageIdentifier;
 
+//    @OneToMany(mappedBy="dvObject", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
+//    private List<DvObjectStorageLocation> dvObjectStorageLocations;
+//
+//    public List<DvObjectStorageLocation> getDvObjectStorageLocations() {
+//        return dvObjectStorageLocations;
+//    }
+//
+//    public void setDvObjectStorageLocations(List<DvObjectStorageLocation> dvObjectStorageLocations) {
+//        this.dvObjectStorageLocations = dvObjectStorageLocations;
+//    }
+
     /**
      * previewImageAvailable could also be thought of as "thumbnail has been
      * generated. However, were all three thumbnails generated? We might need a

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -39,6 +39,7 @@
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
+import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
@@ -230,6 +231,9 @@ String getWrappedMessageWhenJson() {
     @EJB
     DataFileServiceBean fileSvc;
 
+    @EJB
+    StorageSiteServiceBean storageSiteSvc;
+
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     protected EntityManager em;
 

src/main/java/edu/harvard/iq/dataverse/api/StorageSites.java

@@ -0,0 +1,96 @@
+package edu.harvard.iq.dataverse.api;
+
+import edu.harvard.iq.dataverse.locality.StorageSite;
+import edu.harvard.iq.dataverse.locality.StorageSiteUtil;
+import java.util.List;
+import javax.json.Json;
+import javax.json.JsonArrayBuilder;
+import javax.json.JsonObject;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Response;
+
+@Path("admin/storageSites")
+public class StorageSites extends AbstractApiBean {
+
+    @GET
+    public Response listAll() {
+        List<StorageSite> storageSites = storageSiteSvc.findAll();
+        if (storageSites != null && !storageSites.isEmpty()) {
+            JsonArrayBuilder sites = Json.createArrayBuilder();
+            storageSites.forEach((storageSite) -> {
+                sites.add(storageSite.toJsonObjectBuilder());
+            });
+            return ok(sites);
+        } else {
+            return error(Response.Status.NOT_FOUND, "No storage sites were found.");
+        }
+    }
+
+    @GET
+    @Path("{id}")
+    public Response get(@PathParam("id") long id) {
+        StorageSite storageSite = storageSiteSvc.find(id);
+        if (storageSite == null) {
+            return error(Response.Status.NOT_FOUND, "Could not find a storage site based on id " + id + ".");
+        }
+        return ok(storageSite.toJsonObjectBuilder());
+    }
+
+    @POST
+    public Response addSite(JsonObject jsonObject) {
+        StorageSite toPersist = null;
+        try {
+            toPersist = StorageSiteUtil.parse(jsonObject);
+        } catch (Exception ex) {
+            return error(Response.Status.BAD_REQUEST, "JSON could not be parsed: " + ex.getLocalizedMessage());
+        }
+        List<StorageSite> exitingSites = storageSiteSvc.findAll();
+        try {
+            StorageSiteUtil.ensureOnlyOnePrimary(toPersist, exitingSites);
+        } catch (Exception ex) {
+            return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage());
+        }
+        StorageSite saved = storageSiteSvc.add(toPersist);
+        if (saved != null) {
+            return ok(saved.toJsonObjectBuilder());
+        } else {
+            return error(Response.Status.BAD_REQUEST, "Storage site could not be added.");
+        }
+    }
+
+    @PUT
+    @Path("{id}/primaryStorage")
+    public Response setPrimary(@PathParam("id") long id, String input) {
+        StorageSite toModify = storageSiteSvc.find(id);
+        if (toModify == null) {
+            return error(Response.Status.NOT_FOUND, "Could not find a storage site based on id " + id + ".");
+        }
+        // "junk" gets parsed into "false".
+        toModify.setPrimaryStorage(Boolean.valueOf(input));
+        List<StorageSite> exitingSites = storageSiteSvc.findAll();
+        try {
+            StorageSiteUtil.ensureOnlyOnePrimary(toModify, exitingSites);
+        } catch (Exception ex) {
+            return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage());
+        }
+        StorageSite updated = storageSiteSvc.save(toModify);
+        return ok(updated.toJsonObjectBuilder());
+    }
+
+    @DELETE
+    @Path("{id}")
+    public Response delete(@PathParam("id") long id) {
+        boolean deleted = storageSiteSvc.delete(id);
+        if (deleted) {
+            return ok("Storage site id  " + id + " has been deleted.");
+        } else {
+            return error(Response.Status.NOT_FOUND, "Could not find a storage site based on id " + id + ".");
+        }
+    }
+
+}

src/main/java/edu/harvard/iq/dataverse/locality/DvObjectStorageLocation.java

@@ -0,0 +1,43 @@
+package edu.harvard.iq.dataverse.locality;
+
+import edu.harvard.iq.dataverse.DvObject;
+import java.io.Serializable;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.OneToOne;
+
+/**
+ * Future use, maybe. Once we're happy with the design we'll enable it as an
+ * entity.
+ *
+ * TODO: Think more about what problems we're solving, what need to persist and
+ * why.
+ */
+//@Entity
+public class DvObjectStorageLocation implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @OneToOne
+    @JoinColumn(nullable = false)
+    private DvObject dvObject;
+
+    @OneToOne
+    @JoinColumn(nullable = false)
+    private StorageSite storageSite;
+
+    private String storageLocationAddress;
+
+    /**
+     * See "primary" on the StorageSite object, which we are using instead.
+     *
+     * TODO: Consider deleting this field if we don't need it.
+     */
+    private Boolean primaryLocation;
+
+}

src/main/java/edu/harvard/iq/dataverse/locality/StorageSite.java

@@ -0,0 +1,135 @@
+package edu.harvard.iq.dataverse.locality;
+
+import java.io.Serializable;
+import java.util.Objects;
+import javax.json.Json;
+import javax.json.JsonObjectBuilder;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+
+@Entity
+public class StorageSite implements Serializable {
+
+    public static final String ID = "id";
+    public static final String NAME = "name";
+    public static final String HOSTNAME = "hostname";
+    public static final String PRIMARY_STORAGE = "primaryStorage";
+    public static final String TRANSFER_PROTOCOLS = "transferProtocols";
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    // FIXME: Why is nullable=false having no effect?
+    @Column(name = "name", columnDefinition = "TEXT", nullable = false)
+    private String name;
+
+    /**
+     * Sites around the world to which data has been replicated using RSAL
+     * (Repository Storage Abstraction Layer). Formerly, the :ReplicationSites
+     * database setting.
+     *
+     * TODO: Think about how this is a duplication of the following JVM options:
+     *
+     * - dataverse.fqdn
+     *
+     * - dataverse.siteUrl
+     */
+    // FIXME: Why is nullable=false having no effect?
+    @Column(name = "hostname", columnDefinition = "TEXT", nullable = false)
+    private String hostname;
+
+    /**
+     * TODO: Consider adding a constraint to only allow one row to be true. The
+     * following was suggested...
+     *
+     * create unique index on my_table (actual)
+     *
+     * where actual = true;
+     *
+     * ... at
+     * https://stackoverflow.com/questions/28166915/postgresql-constraint-only-one-row-can-have-flag-set/28167225#28167225
+     */
+    @Column(nullable = false)
+    private boolean primaryStorage;
+
+    /**
+     * For example, "rsync,posix,globus". A comma-separated list. These
+     * protocols are what we might advertise to end users who want to download
+     * the data from us. In the future, we can imagine adding S3.
+     */
+    // FIXME: Why is nullable=false having no effect?
+    @Column(name = "transferProtocols", columnDefinition = "TEXT", nullable = false)
+    private String transferProtocols;
+
+//    @OneToMany(mappedBy = "storageSite", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
+//    private List<DvObjectStorageLocation> dvObjectStorageLocations;
+//    public List<DvObjectStorageLocation> getDvObjectStorageLocations() {
+//        return dvObjectStorageLocations;
+//    }
+//
+//    public void setDvObjectStorageLocations(List<DvObjectStorageLocation> dvObjectStorageLocations) {
+//        this.dvObjectStorageLocations = dvObjectStorageLocations;
+//    }
+//    
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getHostname() {
+        return hostname;
+    }
+
+    public void setHostname(String hostname) {
+        this.hostname = hostname;
+    }
+
+    public boolean isPrimaryStorage() {
+        return primaryStorage;
+    }
+
+    public void setPrimaryStorage(boolean primaryStorage) {
+        this.primaryStorage = primaryStorage;
+    }
+
+    public String getTransferProtocols() {
+        return transferProtocols;
+    }
+
+    public void setTransferProtocols(String transferProtocols) {
+        this.transferProtocols = transferProtocols;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (!(object instanceof StorageSite)) {
+            return false;
+        }
+        StorageSite other = (StorageSite) object;
+        return Objects.equals(getId(), other.getId());
+    }
+
+    public JsonObjectBuilder toJsonObjectBuilder() {
+        return Json.createObjectBuilder()
+                .add(ID, id)
+                .add(HOSTNAME, hostname)
+                .add(NAME, name)
+                .add(PRIMARY_STORAGE, primaryStorage)
+                .add(TRANSFER_PROTOCOLS, transferProtocols);
+    }
+}