Merge pull request #429 from Breeding-Insight/release/1.0-develop

Merge for Release/1.0

src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java

@@ -171,21 +171,6 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType
             throw new UnsupportedTypeException("Unsupported mime type");
         }
 
-        // replace "." with "" in column names to deal with json flattening issue in tablesaw
-        List<String> columnNames = df.columnNames();
-        List<String> namesToReplace = new ArrayList<>();
-        for (String name : columnNames) {
-            if (name.contains(".")) {
-                namesToReplace.add(name);
-            }
-        }
-
-        List<Column<?>> columns = df.columns(namesToReplace.stream().toArray(String[]::new));
-        for (int i=0; i<columns.size(); i++) {
-            Column<?> column = columns.get(i);
-            column.setName(namesToReplace.get(i).replace(".",""));
-        }
-
         return df;
     }
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java

@@ -16,6 +16,8 @@
  */
 package org.breedinginsight.brapps.importer.services.processors.experiment;
 
+import io.micronaut.http.HttpStatus;
+import io.micronaut.http.exceptions.HttpStatusException;
 import lombok.Getter;
 import tech.tablesaw.api.Table;
 import tech.tablesaw.columns.Column;
@@ -35,6 +37,15 @@ public class DynamicColumnParser {
      * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns.
      */
     public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) {
+
+        // don't allow periods (.) or square brackets in Dynamic Column Names
+        for (String dynamicColumnName: dynamicColumnNames) {
+            if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){
+                String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName);
+                throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg);
+            }
+        }
+
         List<Column<?>> dynamicCols = data.columns(dynamicColumnNames);
         List<Column<?>> phenotypeCols = new ArrayList<>();
         List<Column<?>> timestampCols = new ArrayList<>();

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java

@@ -22,6 +22,8 @@
 import com.google.gson.GsonBuilder;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.context.annotation.Prototype;
+import io.micronaut.http.HttpStatus;
+import io.micronaut.http.exceptions.HttpStatusException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.map.CaseInsensitiveMap;
 import org.apache.commons.lang3.StringUtils;
@@ -112,6 +114,14 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext
         ImportUpload upload = context.getImportContext().getUpload();
         Table data = context.getImportContext().getData();
         String[] dynamicColNames = upload.getDynamicColumnNames();
+
+        // don't allow periods (.) or square brackets in Dynamic Column Names
+        for (String dynamicColumnName: dynamicColNames) {
+            if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){
+                String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName);
+                throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg);
+            }
+        }
         List<Column<?>> dynamicCols = data.columns(dynamicColNames);
 
         // Collect the columns for observation variable data

src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java

@@ -66,8 +66,8 @@ public ValidationError getMissingScaleDataTypeMsg() {
     }
 
     @Override
-    public ValidationError getPeriodObsVarNameMsg() {
-        return new ValidationError("Name", "Period is invalid", HttpStatus.UNPROCESSABLE_ENTITY);
+    public ValidationError getInvalidCharObsVarNameMsg() {
+        return new ValidationError("Name", "Periods and brackets are invalid", HttpStatus.UNPROCESSABLE_ENTITY);
     }
 
     @Override

src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java

@@ -64,12 +64,13 @@ public ValidationError getMissingScaleUnitMsg() {
     public ValidationError getMissingScaleDataTypeMsg() {
         return new ValidationError("scale.dataType", "Missing scale class", HttpStatus.BAD_REQUEST);
     }
-
+    
     @Override
-    public ValidationError getPeriodObsVarNameMsg() {
-        return new ValidationError("observationVariableName", "Period in name is invalid", HttpStatus.BAD_REQUEST);
+    public ValidationError getInvalidCharObsVarNameMsg() {
+        return new ValidationError("observationVariableName", "Periods and brackets in name are invalid", HttpStatus.BAD_REQUEST);
     }
 
+    @Override
     public ValidationError getMissingObsVarNameMsg() {
         return new ValidationError("observationVariableName", "Missing Name", HttpStatus.BAD_REQUEST);
     }

src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java

@@ -31,7 +31,7 @@ public interface TraitValidatorErrorInterface {
     ValidationError getMissingScaleUnitMsg();
     ValidationError getMissingScaleDataTypeMsg();
     ValidationError getMissingObsVarNameMsg();
-    ValidationError getPeriodObsVarNameMsg();
+    ValidationError getInvalidCharObsVarNameMsg();
     ValidationError getMissingTraitEntityMsg();
     ValidationError getMissingTraitAttributeMsg();
     ValidationError getMissingTraitDescriptionMsg();

src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java

@@ -16,7 +16,6 @@
  */
 package org.breedinginsight.services.validators;
 
-import io.micronaut.http.HttpStatus;
 import org.breedinginsight.api.model.v1.response.ValidationError;
 import org.breedinginsight.api.model.v1.response.ValidationErrors;
 import org.breedinginsight.dao.db.enums.DataType;
@@ -209,7 +208,6 @@ public ValidationErrors checkTraitFieldsLength(List<Trait> traits, TraitValidato
         }
         return errors;
     }
-
     public ValidationErrors checkTraitFieldsFormat(List<Trait> traits, TraitValidatorErrorInterface traitValidatorErrors) {
 
         ValidationErrors errors = new ValidationErrors();
@@ -219,12 +217,12 @@ public ValidationErrors checkTraitFieldsFormat(List<Trait> traits, TraitValidato
             Trait trait = traits.get(i);
             String name = trait.getObservationVariableName();
 
-            Pattern pattern = Pattern.compile("\\.");
+            Pattern pattern = Pattern.compile("[\\.\\]\\[]");
             Matcher matcher = pattern.matcher(name);
             boolean containsInvalidCharacter = matcher.find();
 
             if (name != null && containsInvalidCharacter){
-                ValidationError error = traitValidatorErrors.getPeriodObsVarNameMsg();
+                ValidationError error = traitValidatorErrors.getInvalidCharObsVarNameMsg();
                 errors.addError(traitValidatorErrors.getRowNumber(i), error);
             }
         }

src/main/resources/brapi/sql/R__species.sql

@@ -26,7 +26,8 @@ INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'O
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING;
-INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING;
+-- for the Honey Bee case, want to overwrite name, not preserve existing
+INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING;

src/main/resources/db/migration/V1.31.0__update_species_honey.sql

@@ -0,0 +1,18 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+UPDATE species SET common_name = 'Honey Bee' WHERE common_name = 'Honey';

src/main/resources/version.properties

@@ -15,5 +15,5 @@
 #
 
 
-version=v1.1.0+887
-versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b7d65b01c976e1c729ef9bb450143e63dad4c014
+version=v1.1.0+889
+versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6afa35526f279ed0938cb12ce6bf53d3e67f3d92

src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java

@@ -370,6 +370,4 @@ public void periodInName() {
         assertEquals(0, validationErrors.getRowErrors().size(), "Wrong number of row errors returned");
     }
 
-
-
 }