Skip to content

Commit

Permalink
Merge pull request #3412 from IQSS/3405-filemetadata-directory-label
Browse files Browse the repository at this point in the history 
add directorylabel column to filemetadata table
  • Loading branch information
@kcondon
kcondon authored Oct 28, 2016
2 parents 02a4c0d + b866b5d commit 0f76c0b
Show file tree
Hide file tree
Showing 16 changed files with 1,392 additions and 105 deletions.
1 change: 1 addition & 0 deletions scripts/database/upgrades/3405-filemetadata-directory-label.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ALTER TABLE filemetadata ADD COLUMN directorylabel character varying(255);
7 changes: 6 additions & 1 deletion src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,7 +656,7 @@ private List<FileMetadata> retrieveFileMetadataForVersion(Dataset dataset, Datas
logger.fine("Retrieved and mapped "+i+" file categories attached to files in the version "+version.getId());
categoryResults = null;

List<Object[]> metadataResults = em.createNativeQuery("select id, datafile_id, DESCRIPTION, LABEL, RESTRICTED from FileMetadata where datasetversion_id = "+version.getId()).getResultList();
List<Object[]> metadataResults = em.createNativeQuery("select id, datafile_id, DESCRIPTION, LABEL, RESTRICTED, DIRECTORYLABEL from FileMetadata where datasetversion_id = "+version.getId()).getResultList();

for (Object[] result : metadataResults) {
Integer filemeta_id = (Integer) result[0];
Expand Down Expand Up @@ -707,6 +707,11 @@ private List<FileMetadata> retrieveFileMetadataForVersion(Dataset dataset, Datas
fileMetadata.setRestricted(restricted);
}

String dirLabel = (String) result[5];
if (dirLabel != null){
fileMetadata.setDirectoryLabel(dirLabel);
}

retList.add(fileMetadata);
}

Expand Down
1 change: 1 addition & 0 deletions src/main/java/edu/harvard/iq/dataverse/Dataset.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ private DatasetVersion createNewDatasetVersion(Template template) {
newFm.setCategories(fm.getCategories());
newFm.setDescription(fm.getDescription());
newFm.setLabel(fm.getLabel());
newFm.setDirectoryLabel(fm.getDirectoryLabel());
newFm.setRestricted(fm.isRestricted());
newFm.setDataFile(fm.getDataFile());
newFm.setDatasetVersion(dsv);
Expand Down
22 changes: 22 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
uniqueConstraints = @UniqueConstraint(columnNames = {"dataset_id,versionnumber,minorversionnumber"}))
public class DatasetVersion implements Serializable {

private static final Logger logger = Logger.getLogger(DatasetVersion.class.getCanonicalName());

/**
* Convenience comparator to compare dataset versions by their version number.
* The draft version is considered the latest.
Expand Down Expand Up @@ -1034,6 +1036,26 @@ public Set<ConstraintViolation> validate() {
}
}
}
List<FileMetadata> dsvfileMetadatas = this.getFileMetadatas();
if (dsvfileMetadatas != null) {
for (FileMetadata fileMetadata : dsvfileMetadatas) {
Set<ConstraintViolation<FileMetadata>> constraintViolations = validator.validate(fileMetadata);
if (constraintViolations.size() > 0) {
// currently only support one message
ConstraintViolation<FileMetadata> violation = constraintViolations.iterator().next();
/**
* @todo How can we expose this more detailed message
* containing the invalid value to the user?
*/
String message = "Constraint violation found in FileMetadata. "
+ violation.getMessage() + " "
+ "The invalid value is \"" + violation.getInvalidValue().toString() + "\".";
logger.info(message);
returnSet.add(violation);
break; // currently only support one message, so we can break out of the loop after the first constraint violation
}
}
}
return returnSet;
}
}
22 changes: 21 additions & 1 deletion src/main/java/edu/harvard/iq/dataverse/FileMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,15 @@ public class FileMetadata implements Serializable {

private static final Logger logger = Logger.getLogger(FileMetadata.class.getCanonicalName());

@Pattern(regexp="^[^:<>;#/\"\\*\\|\\?\\\\]*$", message = "File Name cannot contain any of the following characters: \\ / : * ? \" < > | ; # .")
@Pattern(regexp="^[^:<>;#/\"\\*\\|\\?\\\\]*$",
message = "File Name cannot contain any of the following characters: \\ / : * ? \" < > | ; # .")
@NotBlank(message = "Please specify a file name.")
@Column( nullable=false )
private String label = "";
@Pattern(regexp="|[^/\\\\]|^[^/\\\\]+.*[^/\\\\]+$",
message = "Directory Name cannot contain leading or trailing file separators.")
@Column ( nullable=true )
private String directoryLabel = "";
@Column(columnDefinition = "TEXT")
private String description = "";

Expand Down Expand Up @@ -80,6 +85,13 @@ public void setLabel(String label) {
this.label = label;
}

public String getDirectoryLabel() {
return directoryLabel;
}

public void setDirectoryLabel(String directoryLabel) {
this.directoryLabel = directoryLabel;
}

public String getDescription() {
return description;
Expand Down Expand Up @@ -344,6 +356,14 @@ public boolean contentEquals(FileMetadata other) {
} else if (other.getLabel() != null) {
return false;
}

if (this.getDirectoryLabel() != null) {
if (!this.getDirectoryLabel().equals(other.getDirectoryLabel())) {
return false;
}
} else if (other.getDirectoryLabel() != null) {
return false;
}

if (this.getDescription() != null) {
if (!this.getDescription().equals(other.getDescription())) {
Expand Down
9 changes: 8 additions & 1 deletion src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.EJBException;
Expand Down Expand Up @@ -282,7 +283,13 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + ex.getMessage());
}
if (!dataFiles.isEmpty()) {
ingestService.addFiles(editVersion, dataFiles);
Set<ConstraintViolation> constraintViolations = editVersion.validate();
if (constraintViolations.size() > 0) {
ConstraintViolation violation = constraintViolations.iterator().next();
throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\".");
} else {
ingestService.addFiles(editVersion, dataFiles);
}
} else {
throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "No files to add to dataset. Perhaps the zip file was empty.");
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
String validationFailedString = "Validation failed:";
for (ConstraintViolation constraintViolation : constraintViolations) {
validationFailedString += " " + constraintViolation.getMessage();
validationFailedString += " Invalid value: '" + constraintViolation.getInvalidValue() + "'.";
}
throw new IllegalCommandException(validationFailedString, this);
}
Expand Down
107 changes: 5 additions & 102 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ private String checkForDuplicateFileNames(DatasetVersion version, String fileNam
if (fm.getDataFile().isTabularData()) {
String originalMimeType = fm.getDataFile().getDataTable().getOriginalFileFormat();
if ( originalMimeType != null) {
String origFileExtension = generateOriginalExtension(originalMimeType);
String origFileExtension = FileUtil.generateOriginalExtension(originalMimeType);
fileNamesExisting.add(existingName.replaceAll(".tab$", origFileExtension));
} else {
fileNamesExisting.add(existingName.replaceAll(".tab$", ""));
Expand All @@ -648,112 +648,14 @@ private String checkForDuplicateFileNames(DatasetVersion version, String fileNam
fileNamesExisting.add(existingName);
}
}

while (fileNamesExisting.contains(fileName)) {
fileName = generateNewFileName(fileName);
fileName = IngestServiceBeanHelper.generateNewFileName(fileName);
}

return fileName;
}

private void checkForDuplicateFileNamesFinal(DatasetVersion version, List<DataFile> newFiles) {
Set<String> fileNamesExisting = new HashSet<String>();

Iterator<FileMetadata> fmIt = version.getFileMetadatas().iterator();
while (fmIt.hasNext()) {
FileMetadata fm = fmIt.next();
if (fm.getId() != null) {
String existingName = fm.getLabel();

if (existingName != null) {
// if it's a tabular file, we need to restore the original file name;
// otherwise, we may miss a match. e.g. stata file foobar.dta becomes
// foobar.tab once ingested!
if (fm.getDataFile().isTabularData()) {
String originalMimeType = fm.getDataFile().getDataTable().getOriginalFileFormat();
if ( originalMimeType != null) {
String origFileExtension = generateOriginalExtension(originalMimeType);
existingName = existingName.replaceAll(".tab$", origFileExtension);
} else {
existingName = existingName.replaceAll(".tab$", "");
}
}
fileNamesExisting.add(existingName);
}
}
}

Iterator<DataFile> dfIt = newFiles.iterator();
while (dfIt.hasNext()) {
FileMetadata fm = dfIt.next().getFileMetadata();
String fileName = fm.getLabel();
while (fileNamesExisting.contains(fileName)) {
fileName = generateNewFileName(fileName);
}
if (!fm.getLabel().equals(fileName)) {
fm.setLabel(fileName);
fileNamesExisting.add(fileName);
}
}
}

// TODO:
// Move this method (duplicated in StoredOriginalFile.java) to
// FileUtil.java.
// -- L.A. 4.0 beta

private static String generateOriginalExtension(String fileType) {

if (fileType.equalsIgnoreCase("application/x-spss-sav")) {
return ".sav";
} else if (fileType.equalsIgnoreCase("application/x-spss-por")) {
return ".por";
} else if (fileType.equalsIgnoreCase("application/x-stata")) {
return ".dta";
} else if (fileType.equalsIgnoreCase( "application/x-rlang-transport")) {
return ".RData";
} else if (fileType.equalsIgnoreCase("text/csv")) {
return ".csv";
} else if (fileType.equalsIgnoreCase( "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) {
return ".xlsx";
}

return "";
}


private String generateNewFileName(String fileName) {
String newName = null;
String baseName = null;
String extension = null;

int extensionIndex = fileName.lastIndexOf(".");
if (extensionIndex != -1 ) {
extension = fileName.substring(extensionIndex+1);
baseName = fileName.substring(0, extensionIndex);
} else {
baseName = fileName;
}

if (baseName.matches(".*-[0-9][0-9]*$")) {
int dashIndex = baseName.lastIndexOf("-");
String numSuffix = baseName.substring(dashIndex+1);
String basePrefix = baseName.substring(0,dashIndex);
int numSuffixValue = new Integer(numSuffix).intValue();
numSuffixValue++;
baseName = basePrefix + "-" + numSuffixValue;
} else {
baseName = baseName + "-1";
}

newName = baseName;
if (extension != null) {
newName = newName + "." + extension;
}

return newName;
}

/**
* Returns a content type string for a FileObject
*
Expand Down Expand Up @@ -803,6 +705,7 @@ private DataFile createSingleDataFile(DatasetVersion version, InputStream inputS
datafile.setPermissionModificationTime(new Timestamp(new Date().getTime()));
FileMetadata fmd = new FileMetadata();

// TODO: add directoryLabel?
fmd.setLabel(checkForDuplicateFileNames(version,fileName));

if (addToDataset) {
Expand Down Expand Up @@ -880,7 +783,7 @@ public void addFiles (DatasetVersion version, List<DataFile> newFiles) {
// the user may have edited them on the "add files" page, and
// renamed FOOBAR-1.txt back to FOOBAR.txt...

checkForDuplicateFileNamesFinal(version, newFiles);
IngestServiceBeanHelper.checkForDuplicateFileNamesFinal(version, newFiles);

Dataset dataset = version.getDataset();

Expand Down
Loading

0 comments on commit 0f76c0b

Please sign in to comment.