diff --git a/pom.xml b/pom.xml index d1bb280f86e..37af517f883 100644 --- a/pom.xml +++ b/pom.xml @@ -319,17 +319,31 @@ jacoco-maven-plugin 0.7.5.201505241946 + + + org.slf4j + slf4j-api + 1.7.7 + + + org.slf4j + slf4j-log4j12 + 1.7.7 + org.mockito mockito-core 1.10.19 - + axis axis @@ -355,6 +369,12 @@ commons-codec 1.9 + + + org.javaswift + joss + 0.9.10 + com.maxmind.geoip2 geoip2 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index b8d6f793eef..01d34850026 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -567,7 +567,7 @@ public String getOriginalChecksumType() { } public DataFileIO getAccessObject() throws IOException { - DataFileIO dataAccess = DataAccess.createDataAccessObject(this); + DataFileIO dataAccess = DataAccess.getDataFileIO(this); if (dataAccess == null) { throw new IOException("Failed to create access object for datafile."); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 08479dfcd6f..fe5e7bfd213 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.DatasetVersionServiceBean.RetrieveDatasetVersionResponse; +import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper; @@ -616,7 +617,20 @@ public boolean isPubliclyDownloadable() { } public String getPublicDownloadUrl() { - return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + if (System.getProperty("dataverse.files.storage-driver-id").equals("swift")) { + String fileDownloadUrl = null; + try { + SwiftAccessIO swiftIO = (SwiftAccessIO) getFile().getAccessObject(); + swiftIO.open(); + fileDownloadUrl = swiftIO.getRemoteUrl(); + logger.info("Swift url: " + fileDownloadUrl); + } catch (Exception e) { + e.printStackTrace(); + } + return fileDownloadUrl; + } else { + return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java index f4070e9b4e5..7e32d4ae231 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java @@ -56,7 +56,7 @@ public void writeTo(BundleDownloadInstance di, Class clazz, Type type, Annota if (di.getDownloadInfo() != null && di.getDownloadInfo().getDataFile() != null) { DataAccessRequest daReq = new DataAccessRequest(); DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index d41b83c476d..a81817f04f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -64,7 +64,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index c4a64bff350..82907486895 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -21,8 +21,8 @@ package edu.harvard.iq.dataverse.dataaccess; import edu.harvard.iq.dataverse.DataFile; - import java.io.IOException; +import org.javaswift.joss.model.StoredObject; /** * @@ -34,21 +34,35 @@ public DataAccess() { } - public static DataFileIO createDataAccessObject (DataFile df) throws IOException { - return createDataAccessObject (df, null); + // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift + public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); + public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start"); + public static String swiftFileUri; + public static String swiftContainerUri; + + // The getDataFileIO() methods initialize DataFileIO objects for + // datafiles that are already saved using one of the supported Dataverse + // DataAccess IO drivers. + public static DataFileIO getDataFileIO(DataFile df) throws IOException { + return getDataFileIO(df, null); } - public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest req) throws IOException { + public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throws IOException { - if (df == null || - df.getStorageIdentifier() == null || - df.getStorageIdentifier().equals("")) { - throw new IOException ("createDataAccessObject: null or invalid datafile."); + if (df == null + || df.getStorageIdentifier() == null + || df.getStorageIdentifier().equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); } if (df.getStorageIdentifier().startsWith("file://") || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) { return new FileAccessIO (df, req); + } else if (df.getStorageIdentifier().startsWith("swift://") + || df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + return new SwiftAccessIO(df, req); + } else if (df.getStorageIdentifier().startsWith("tmp://")) { + throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet."); } // No other storage methods are supported as of now! -- 4.0.1 @@ -58,6 +72,55 @@ public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest // "storage identifier". // -- L.A. 4.0.2 - throw new IOException ("createDataAccessObject: Unsupported storage method."); + throw new IOException("getDataAccessObject: Unsupported storage method."); + } + + // createDataAccessObject() methods create a *new*, empty DataAccess objects, + // for saving new, not yet saved datafiles. + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag) throws IOException { + + return createNewDataFileIO(df, storageTag, DEFAULT_STORAGE_DRIVER_IDENTIFIER); } + + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, String driverIdentifier) throws IOException { + if (df == null + || storageTag == null + || storageTag.equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); + } + + DataFileIO dataFileIO = null; + + df.setStorageIdentifier(storageTag); + + if (driverIdentifier.equals("file")) { + dataFileIO = new FileAccessIO(df, null); + } else if (driverIdentifier.equals("swift")) { + dataFileIO = new SwiftAccessIO(df, null); + } else { + throw new IOException("createDataAccessObject: Unsupported storage method " + driverIdentifier); + } + + dataFileIO.open(DataAccessOption.WRITE_ACCESS); + return dataFileIO; + } + + public static String getSwiftFileURI(StoredObject fileObject) throws IOException { + String fileUri; + try { + fileUri = fileObject.getPublicURL(); + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to get file storage location"); + } + return fileUri; + } + + public static String getSwiftContainerURI(StoredObject fileObject) throws IOException { + String containerUri; + containerUri = getSwiftFileURI(fileObject); + containerUri = containerUri.substring(0, containerUri.lastIndexOf('/')); + return containerUri; + } + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java index e75f326c3fa..d9022d87654 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java @@ -221,7 +221,7 @@ public long addFileToZipStream(DataFile dataFile) throws IOException { boolean createManifest = fileManifest != null; DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java new file mode 100644 index 00000000000..adc6518c819 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -0,0 +1,404 @@ +package edu.harvard.iq.dataverse.dataaccess; + +import edu.harvard.iq.dataverse.DataFile; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channel; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.Properties; +import java.util.logging.Logger; +import org.javaswift.joss.client.factory.AccountFactory; +import static org.javaswift.joss.client.factory.AuthenticationMethod.BASIC; +import org.javaswift.joss.model.Account; +import org.javaswift.joss.model.Container; +import org.javaswift.joss.model.StoredObject; + +/** + * + * @author leonid andreev + */ +/* + Experimental Swift driver, implemented as part of the Dataverse - Mass Open Cloud + collaboration. + Read-only access, for now. + */ +public class SwiftAccessIO extends DataFileIO { + + private String swiftFolderPath; + private String swiftFileName = null; + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO"); + + public SwiftAccessIO() throws IOException { + this(null); + } + + public SwiftAccessIO(DataFile dataFile) throws IOException { + this(dataFile, null); + + } + + public SwiftAccessIO(DataFile dataFile, DataAccessRequest req) throws IOException { + + super(dataFile, req); + + this.setIsLocalFile(false); + } + + private boolean isReadAccess = false; + private boolean isWriteAccess = false; + private Properties swiftProperties = null; + private Account account = null; + private StoredObject swiftFileObject = null; + + @Override + public boolean canRead() { + return isReadAccess; + } + + @Override + public boolean canWrite() { + return isWriteAccess; + } + + @Override + public void open(DataAccessOption... options) throws IOException { + + DataFile dataFile = this.getDataFile(); + DataAccessRequest req = this.getRequest(); + + if (req != null && req.getParameter("noVarHeader") != null) { + this.setNoVarHeader(true); + } + + if (isWriteAccessRequested(options)) { + isWriteAccess = true; + isReadAccess = false; + } else { + isWriteAccess = false; + isReadAccess = true; + } + + if (this.getDataFile().getStorageIdentifier() == null || "".equals(this.getDataFile().getStorageIdentifier())) { + throw new IOException("Data Access: No local storage identifier defined for this datafile."); + } + + if (isReadAccess) { + InputStream fin = openSwiftFileAsInputStream(); + + if (fin == null) { + throw new IOException("Failed to open Swift file " + getStorageLocation()); + } + + this.setInputStream(fin); + setChannel(Channels.newChannel(fin)); + + } else if (isWriteAccess) { + swiftFileObject = initializeSwiftFileObject(true); + } + + this.setMimeType(dataFile.getContentType()); + try { + this.setFileName(dataFile.getFileMetadata().getLabel()); + } catch (Exception ex) { + this.setFileName("unknown"); + } + + // This "status" is a leftover from 3.6; we don't have a use for it + // in 4.0 yet; and we may not need it at all. + // -- L.A. 4.0.2 + this.setStatus(200); + } + + // this is a Swift-specific override of the convenience method provided in the + // DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location): + @Override + public void copyPath(Path fileSystemPath) throws IOException { + long newFileSize = -1; + Properties p = getSwiftProperties(); + String swiftEndPoint = p.getProperty("swift.default.endpoint"); + String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + + if (swiftFileObject == null || !this.canWrite()) { + open(DataAccessOption.WRITE_ACCESS); + } + + File inputFile = null; + + try { + inputFile = fileSystemPath.toFile(); + + //@author Anuj Thakur + swiftFileObject.uploadObject(inputFile); + //After the files object is uploaded the identifier is changed. + logger.info(this.swiftFileName + " " + this.swiftFolderPath); + this.getDataFile().setStorageIdentifier(swiftDirectory + "/" + this.swiftFolderPath + "/" + this.swiftFileName); + + newFileSize = inputFile.length(); + + } catch (Exception ioex) { + String failureMsg = ioex.getMessage(); + if (failureMsg == null) { + failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; + } + + throw new IOException(failureMsg); + } + + // if it has uploaded successfully, we can reset the size + // of the object: + setSize(newFileSize); + + } + + @Override + public void delete() throws IOException { + //throw new IOException("SwiftAccessIO: delete() not yet implemented in this storage driver."); + if (swiftFileObject == null) { + try { + swiftFileObject = initializeSwiftFileObject(false); + } catch (IOException ioex) { + swiftFileObject = null; + } + } + + if (swiftFileObject != null) { + swiftFileObject.delete(); + } + } + + @Override + public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException { + + throw new IOException("SwiftAccessIO: openAuxChannel() not yet implemented in this storage driver."); + } + + @Override + public boolean isAuxObjectCached(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: isAuxObjectCached() not yet implemented in this storage driver."); + } + + @Override + public long getAuxObjectSize(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: getAuxObjectSize() not yet implemented in this storage driver."); + } + + @Override + public void backupAsAux(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: backupAsAux() not yet implemented in this storage driver."); + } + + @Override + public String getStorageLocation() { + // What should this be, for a Swift file? + // A Swift URL? + // Or a Swift URL with an authenticated Auth token? + + return null; + } + + @Override + public Path getFileSystemPath() throws IOException { + throw new IOException("SwiftAccessIO: this is a remote AccessIO object, it has no local filesystem path associated with it."); + } + + // Auxilary helper methods, Swift-specific: + private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOException { + String storageIdentifier = this.getDataFile().getStorageIdentifier(); + + String swiftEndPoint = null; + String swiftContainer = null; + + // + // if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + // This is a call on an already existing swift object. + + //String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + //The Storage identifier now is the Object store service endpoint + //followed by the endpoint named container + String[] swiftStorageTokens = storageIdentifier.substring(46).split("/", 2); + + if (swiftStorageTokens.length != 2) { + // bad storage identifier + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + + Properties p = getSwiftProperties(); + swiftEndPoint = p.getProperty("swift.default.endpoint"); + + swiftContainer = swiftStorageTokens[0]; + swiftFileName = swiftStorageTokens[1]; + + if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null + || "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) { + // all of these things need to be specified, for this to be a valid Swift location + // identifier. + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + } else if (this.isReadAccess) { + // An attempt to call Swift driver, in a Read mode on a non-swift stored datafile + // object! + throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); + } else if (this.isWriteAccess) { + Properties p = getSwiftProperties(); + swiftEndPoint = p.getProperty("swift.default.endpoint"); + swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); + //swiftFolderPath = this.getDataFile().getOwner().getIdentifier(); /* TODO: ? */ + swiftFileName = storageIdentifier; + //swiftFileName = this.getDataFile().getDisplayName(); + //Storage Identifier is now updated after the object is uploaded on Swift. + // this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftContainer+":"+swiftFileName); + } else { + throw new IOException("SwiftAccessIO: unknown access mode."); + } + // Authenticate with Swift: + + account = authenticateWithSwift(swiftEndPoint); + + /* + The containers created is swiftEndPoint concatenated with the swiftContainer + property. Creating container with certain names throws 'Unable to create + container' error on Openstack. + Any datafile with http://rdgw storage identifier i.e present on Object + store service endpoint already only needs to look-up for container using + just swiftContainer which is the concatenated name. + In future, a container for the endpoint can be created and for every + other swiftContainer Object Store pseudo-folder can be created, which is + not provide by the joss Java swift library as of yet. + */ + Container dataContainer; + + if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + dataContainer = account.getContainer(swiftContainer); + } else { + dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint + } + + if (!dataContainer.exists()) { + if (writeAccess) { + dataContainer.create(); + //dataContainer.makePublic(); + } else { + // This is a fatal condition - it has to exist, if we were to + // read an existing object! + throw new IOException("SwiftAccessIO: container " + swiftContainer + " does not exist."); + } + } + + StoredObject fileObject = dataContainer.getObject(swiftFileName); + //file download url for public files + DataAccess.swiftFileUri = DataAccess.getSwiftFileURI(fileObject); + setRemoteUrl(DataAccess.getSwiftFileURI(fileObject)); + + logger.info(DataAccess.swiftFileUri + " success"); + //shows contents of container for public containers + DataAccess.swiftContainerUri = DataAccess.getSwiftContainerURI(fileObject); + logger.info(DataAccess.swiftContainerUri + " success"); + + if (!writeAccess && !fileObject.exists()) { + throw new IOException("SwiftAccessIO: File object " + swiftFileName + " does not exist (Dataverse datafile id: " + this.getDataFile().getId()); + } + + return fileObject; + } + + private InputStream openSwiftFileAsInputStream() throws IOException { + InputStream in = null; + + swiftFileObject = initializeSwiftFileObject(false); + + in = swiftFileObject.downloadObjectAsInputStream(); + this.setSize(swiftFileObject.getContentLength()); + + return in; + } + + private Properties getSwiftProperties() throws IOException { + if (swiftProperties == null) { + String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); + String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; + swiftProperties = new Properties(); + swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); + } + + return swiftProperties; + } + + Account authenticateWithSwift(String swiftEndPoint) throws IOException { + + Properties p = getSwiftProperties(); + + // (this will throw an IOException, if the swift properties file + // is missing or corrupted) + String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); + String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); + String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); + String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); + + if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null + || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { + // again, all of these things need to be defined, for this Swift endpoint to be + // accessible. + throw new IOException("SwiftAccessIO: no configuration available for endpoint " + swiftEndPoint); + } + + // Authenticate: + Account account = null; + + /* + This try { } now authenticates using either the KEYSTONE mechanism which uses + the tenant name in addition to the Username Password and AuthUrl OR the BASIC method + Also, the AuthUrl is now the identity service endpoint of MOC Openstack + environment instead of the Object store service endpoint. + */ + // Keystone vs. Basic + try { + if (swiftEndPointAuthMethod.equals("keystone")) { + account = new AccountFactory() + .setTenantName(swiftEndPointTenantName) + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .createAccount(); + } else { // assume BASIC + account = new AccountFactory() + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .setAuthenticationMethod(BASIC) + .createAccount(); + } + + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to authenticate " + swiftEndPointAuthMethod + " for the end point " + swiftEndPoint); + } + + return account; + } + + private boolean isWriteAccessRequested(DataAccessOption... options) throws IOException { + + for (DataAccessOption option : options) { + // In the future we may need to be able to open read-write + // Channels; no support, or use case for that as of now. + + if (option == DataAccessOption.READ_ACCESS) { + return false; + } + + if (option == DataAccessOption.WRITE_ACCESS) { + return true; + } + } + + // By default, we open the file in read mode: + return false; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index f9ab137a603..be538442d22 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -37,6 +37,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.DataFileIO; import edu.harvard.iq.dataverse.dataaccess.FileAccessIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -267,12 +268,14 @@ public void addFiles (DatasetVersion version, List newFiles) { try { - DataFileIO dataAccess = dataFile.getAccessObject(); + logger.info("Attempting to create a new DataFileIO object for " + storageId); + DataFileIO dataAccess = DataAccess.createNewDataFileIO(dataFile, storageId); if (dataAccess.isLocalFile()) { localFile = true; } - + + logger.info("Successfully created a new DataFileIO object."); /* This commented-out code demonstrates how to copy bytes from a local InputStream (or a readChannel) into the @@ -311,7 +314,7 @@ from a local InputStream (or a readChannel) into the savedSuccess = true; } catch (IOException ioex) { - logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier()); + logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); } finally { if (readChannel != null) {try{readChannel.close();}catch(IOException e){}} if (writeChannel != null) {try{writeChannel.close();}catch(IOException e){}} diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java index 791b376fb1e..df81d0328aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java @@ -564,7 +564,7 @@ public File runDataPreprocessing(DataFile dataFile) { // send the tabular data file to the Rserve side: DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0211fedf0e7..fbe67d4c6e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -222,8 +222,11 @@ public enum Key { Default is false; */ TwoRavensTabularView, - - + /** + * Whether to save data files locally in files or store them in swift + * storage, default is file. + */ + Driver, /** The message added to a popup upon dataset publish *