From bde0bec24dcd716cf8aea7ed6cbd1b6799ae426f Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 18 Apr 2019 08:19:27 -0400 Subject: [PATCH 1/9] Add methods that pass locale when adding string-based values to entities --- .../moqui/impl/context/L10nFacadeImpl.java | 38 ++++++++++------- .../impl/entity/EntityDataLoaderImpl.groovy | 41 ++++++++++--------- .../moqui/impl/entity/EntityFacadeImpl.groovy | 16 ++++---- .../moqui/impl/entity/EntityFindBase.groovy | 2 +- .../org/moqui/impl/entity/EntityJavaUtil.java | 6 +-- .../moqui/impl/entity/EntityValueBase.java | 14 ++++--- .../org/moqui/impl/entity/FieldInfo.java | 13 +++--- .../runner/EntityAutoServiceRunner.groovy | 39 +++++++++++++----- .../java/org/moqui/entity/EntityValue.java | 24 +++++++++++ 9 files changed, 127 insertions(+), 66 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java index ead15089b..a8d49e58d 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java @@ -133,16 +133,19 @@ public String formatCurrency(Object amount, String uomId, Integer fractionDigits @Override public Time parseTime(String input, String format) { - Locale curLocale = getLocale(); + return parseTime(input, format, null); + } + public Time parseTime(String input, String format, Locale locale) { + if (locale == null) locale = getLocale(); TimeZone curTz = getTimeZone(); if (format == null || format.isEmpty()) format = "HH:mm:ss.SSS"; - Calendar cal = calendarValidator.validate(input, format, curLocale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "HH:mm:ss", curLocale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "HH:mm", curLocale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "h:mm a", curLocale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "h:mm:ss a", curLocale, curTz); + Calendar cal = calendarValidator.validate(input, format, locale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "HH:mm:ss", locale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "HH:mm", locale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "h:mm a", locale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "h:mm:ss a", locale, curTz); // also try the full ISO-8601, times may come in that way (even if funny with a date of 1970-01-01) - if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale, curTz); if (cal != null) { Time time = new Time(cal.getTimeInMillis()); // logger.warn("============== parseTime input=${input} cal=${cal} long=${cal.getTimeInMillis()} time=${time} time long=${time.getTime()} util date=${new java.util.Date(cal.getTimeInMillis())} timestamp=${new java.sql.Timestamp(cal.getTimeInMillis())}") @@ -170,8 +173,11 @@ public String formatTime(Time input, String format, Locale locale, TimeZone tz) @Override public java.sql.Date parseDate(String input, String format) { + return parseDate(input, format, null); + } + public java.sql.Date parseDate(String input, String format, Locale locale) { if (format == null || format.isEmpty()) format = "yyyy-MM-dd"; - Locale curLocale = getLocale(); + if (locale == null) locale = getLocale(); // NOTE DEJ 20150317 Date parsing in terms of time zone causes funny issues because the time part of the long // since epoch representation is lost going to/from the DB, especially since the time portion is set to 0 and @@ -183,15 +189,15 @@ public java.sql.Date parseDate(String input, String format) { /* TimeZone curTz = getTimeZone() Calendar cal = calendarValidator.validate(input, format, curLocale, curTz) - if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", curLocale, curTz) + if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", locale, curTz) // also try the full ISO-8601, dates may come in that way - if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale, curTz) + if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale, curTz) */ - Calendar cal = calendarValidator.validate(input, format, curLocale); - if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", curLocale); + Calendar cal = calendarValidator.validate(input, format, locale); + if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", locale); // also try the full ISO-8601, dates may come in that way - if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale); + if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale); if (cal != null) { java.sql.Date date = new java.sql.Date(cal.getTimeInMillis()); // logger.warn("============== parseDate input=${input} cal=${cal} long=${cal.getTimeInMillis()} date=${date} date long=${date.getTime()} util date=${new java.util.Date(cal.getTimeInMillis())} timestamp=${new java.sql.Timestamp(cal.getTimeInMillis())}") @@ -292,7 +298,11 @@ public String formatDateTime(Calendar input, String format, Locale locale, TimeZ } @Override public BigDecimal parseNumber(String input, String format) { - return bigDecimalValidator.validate(input, format, getLocale()); } + return parseNumber(input, format, null); + } + public BigDecimal parseNumber(String input, String format, Locale locale) { + if (locale == null) locale = getLocale(); + return bigDecimalValidator.validate(input, format, locale); } public String formatNumber(Number input, String format, Locale locale) { if (locale == null) locale = getLocale(); return bigDecimalValidator.format(input, format, locale); diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy index 555f8d8b8..8e1498138 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy @@ -44,6 +44,7 @@ import javax.xml.parsers.SAXParserFactory import java.nio.charset.StandardCharsets import java.util.zip.ZipEntry import java.util.zip.ZipInputStream +import java.util.Locale @CompileStatic class EntityDataLoaderImpl implements EntityDataLoader { @@ -182,6 +183,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.disableFkCreate) reenableFkCreate = !eci.artifactExecutionFacade.disableEntityFkCreate() boolean reenableDataFeed = false if (this.disableDataFeed) reenableDataFeed = !eci.artifactExecutionFacade.disableEntityDataFeed() + Locale locale = new Locale("en_US") // if no xmlText or locations, so find all of the component and entity-facade files if (!this.xmlText && !this.csvText && !this.jsonText && !this.locationList) { @@ -264,7 +266,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.csvText) { InputStream csvInputStream = new ByteArrayInputStream(csvText.getBytes("UTF-8")) try { - tf.runUseOrBegin(transactionTimeout, "Error loading CSV entity data", { ech.loadFile("csvText", csvInputStream) }) + tf.runUseOrBegin(transactionTimeout, "Error loading CSV entity data", { ech.loadFile("csvText", csvInputStream, locale) }) } finally { if (csvInputStream != null) csvInputStream.close() } @@ -274,14 +276,14 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.jsonText) { InputStream jsonInputStream = new ByteArrayInputStream(jsonText.getBytes("UTF-8")) try { - tf.runUseOrBegin(transactionTimeout, "Error loading JSON entity data", { ejh.loadFile("jsonText", jsonInputStream) }) + tf.runUseOrBegin(transactionTimeout, "Error loading JSON entity data", { ejh.loadFile("jsonText", jsonInputStream, locale) }) } finally { if (jsonInputStream != null) jsonInputStream.close() } } // load each file in its own transaction - for (String location in this.locationList) loadSingleFile(location, exh, ech, ejh) + for (String location in this.locationList) loadSingleFile(location, exh, ech, ejh, locale) }) if (reenableEeca) eci.artifactExecutionFacade.enableEntityEca() @@ -293,7 +295,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { // Thread.sleep(60*1000*100) } - void loadSingleFile(String location, EntityXmlHandler exh, EntityCsvHandler ech, EntityJsonHandler ejh) { + void loadSingleFile(String location, EntityXmlHandler exh, EntityCsvHandler ech, EntityJsonHandler ejh, Locale locale) { TransactionFacade tf = efi.ecfi.transactionFacade boolean beganTransaction = tf.begin(transactionTimeout) try { @@ -312,12 +314,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { logger.info("Loaded ${(exh.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } else if (location.endsWith(".csv")) { long beforeRecords = ech.valuesRead ?: 0 - if (ech.loadFile(location, inputStream)) { + if (ech.loadFile(location, inputStream, locale)) { logger.info("Loaded ${(ech.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } } else if (location.endsWith(".json")) { long beforeRecords = ejh.valuesRead ?: 0 - if (ejh.loadFile(location, inputStream)) { + if (ejh.loadFile(location, inputStream, locale)) { logger.info("Loaded ${(ejh.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } } else if (location.endsWith(".zip")) { @@ -383,7 +385,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { ValueHandler(EntityDataLoaderImpl edli) { this.edli = edli } abstract void handleValue(EntityValue value) - abstract void handlePlainMap(String entityName, Map value) + abstract void handlePlainMap(String entityName, Map value, Locale locale) abstract void handleService(ServiceCallSync scs) } static class CheckValueHandler extends ValueHandler { @@ -401,8 +403,8 @@ class EntityDataLoaderImpl implements EntityDataLoader { long getFieldsChecked() { return fieldsChecked } void handleValue(EntityValue value) { value.checkAgainstDatabase(messageList) } - void handlePlainMap(String entityName, Map value) { - EntityList el = edli.getEfi().getValueListFromPlainMap(value, entityName) + void handlePlainMap(String entityName, Map value, Locale locale) { + EntityList el = edli.getEfi().getValueListFromPlainMap(value, entityName, locale) // logger.warn("=========== Check value: ${value}\nel: ${el}") for (EntityValue ev in el) fieldsChecked += ev.checkAgainstDatabase(messageList) } @@ -461,11 +463,11 @@ class EntityDataLoaderImpl implements EntityDataLoader { value.createOrUpdate() } } - void handlePlainMap(String entityName, Map value) { + void handlePlainMap(String entityName, Map value, Locale locale) { EntityDefinition ed = ec.entityFacade.getEntityDefinition(entityName) if (ed == null) throw new BaseException("Could not find entity ${entityName}") if (edli.onlyCreate) { - EntityList el = ec.entityFacade.getValueListFromPlainMap(value, entityName) + EntityList el = ec.entityFacade.getValueListFromPlainMap(value, entityName, locale) int elSize = el.size() for (int i = 0; i < elSize; i++) { EntityValue curValue = (EntityValue) el.get(i) @@ -480,7 +482,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { } } else { Map results = new HashMap() - EntityAutoServiceRunner.storeEntity(ec, ed, value, results, null) + EntityAutoServiceRunner.storeEntity(ec, ed, value, results, null, locale) // no need to call the store auto service, use storeEntity directly: // Map results = sfi.sync().name('store', entityName).parameters(value).call() if (logger.isTraceEnabled()) logger.trace("Called store service for entity [${entityName}] in data load, results: ${results}") @@ -516,9 +518,9 @@ class EntityDataLoaderImpl implements EntityDataLoader { void handleValue(EntityValue value) { el.add(value) } - void handlePlainMap(String entityName, Map value) { + void handlePlainMap(String entityName, Map value, Locale locale) { EntityDefinition ed = edli.getEfi().getEntityDefinition(entityName) - edli.getEfi().addValuesFromPlainMapRecursive(ed, value, el, null) + edli.getEfi().addValuesFromPlainMapRecursive(ed, value, el, null, locale) } void handleService(ServiceCallSync scs) { logger.warn("For load to EntityList not calling service [${scs.getServiceName()}] with parameters ${scs.getCurrentParameters()}") } } @@ -545,6 +547,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { protected long valuesRead = 0 protected List messageList = new LinkedList<>() String location + Locale locale = new Locale("en_US") protected boolean loadElements = false @@ -758,11 +761,11 @@ class EntityDataLoaderImpl implements EntityDataLoader { // if (currentEntityDef.getFullEntityName().contains("DbForm")) logger.warn("========= DbForm rootValueMap: ${rootValueMap}") if (edli.dummyFks || edli.useTryInsert) { EntityValue curValue = currentEntityDef.makeEntityValue() - curValue.setAll(valueMap) + curValue.setAll(valueMap, locale) valueHandler.handleValue(curValue) valuesRead++ } else { - valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap) + valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap, locale) valuesRead++ } currentEntityDef = (EntityDefinition) null @@ -812,7 +815,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { long getValuesRead() { return valuesRead } List getMessageList() { return messageList } - boolean loadFile(String location, InputStream is) { + boolean loadFile(String location, InputStream is, Locale locale) { BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")) CSVParser parser = CSVFormat.newFormat(edli.csvDelimiter) @@ -917,7 +920,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { long getValuesRead() { return valuesRead } List getMessageList() { return messageList } - boolean loadFile(String location, InputStream is) { + boolean loadFile(String location, InputStream is, Locale locale) { JsonSlurper slurper = new JsonSlurper() Object jsonObj try { @@ -978,7 +981,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { valueHandler.handleService(currentScs) valuesRead++ } else { - valueHandler.handlePlainMap(entityName, value) + valueHandler.handlePlainMap(entityName, value, locale) // TODO: make this more complete, like counting nested Maps? valuesRead++ } diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityFacadeImpl.groovy index 137af9d0f..321afee4b 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityFacadeImpl.groovy @@ -1622,7 +1622,7 @@ class EntityFacadeImpl implements EntityFacade { if (lastEd.containsPrimaryKey(parameters)) { // if we have a full PK lookup by PK and return the single value Map pkValues = [:] - lastEd.entityInfo.setFields(parameters, pkValues, false, null, true) + lastEd.entityInfo.setFields(parameters, pkValues, false, null, true, new java.util.Locale("en_US")) if (masterName != null && masterName.length() > 0) { Map resultMap = find(lastEd.getFullEntityName()).condition(pkValues).oneMaster(masterName) @@ -1672,7 +1672,7 @@ class EntityFacadeImpl implements EntityFacade { } } - EntityList getValueListFromPlainMap(Map value, String entityName) { + EntityList getValueListFromPlainMap(Map value, String entityName, Locale locale) { if (entityName == null || entityName.length() == 0) entityName = value."_entity" if (entityName == null || entityName.length() == 0) throw new EntityException("No entityName passed and no _entity field in value Map") @@ -1680,10 +1680,10 @@ class EntityFacadeImpl implements EntityFacade { if (ed == null) throw new EntityNotFoundException("Not entity found with name ${entityName}") EntityList valueList = new EntityListImpl(this) - addValuesFromPlainMapRecursive(ed, value, valueList, null) + addValuesFromPlainMapRecursive(ed, value, valueList, null, locale) return valueList } - void addValuesFromPlainMapRecursive(EntityDefinition ed, Map value, EntityList valueList, Map parentPks) { + void addValuesFromPlainMapRecursive(EntityDefinition ed, Map value, EntityList valueList, Map parentPks, Locale locale) { // add in all of the main entity's primary key fields, this is necessary for auto-generated, and to // allow them to be left out of related records if (parentPks != null) { @@ -1692,7 +1692,7 @@ class EntityFacadeImpl implements EntityFacade { } EntityValue newEntityValue = makeValue(ed.getFullEntityName()) - newEntityValue.setFields(value, true, null, null) + newEntityValue.setFields(value, true, null, null, locale) valueList.add(newEntityValue) Map sharedPkMap = newEntityValue.getPrimaryKeys() @@ -1730,11 +1730,11 @@ class EntityFacadeImpl implements EntityFacade { boolean isEntityValue = relParmObj instanceof EntityValue if (relParmObj instanceof Map && !isEntityValue) { - addValuesFromPlainMapRecursive(subEd, (Map) relParmObj, valueList, pkMap) + addValuesFromPlainMapRecursive(subEd, (Map) relParmObj, valueList, pkMap, locale) } else if (relParmObj instanceof List) { for (Object relParmEntry in relParmObj) { if (relParmEntry instanceof Map) { - addValuesFromPlainMapRecursive(subEd, (Map) relParmEntry, valueList, pkMap) + addValuesFromPlainMapRecursive(subEd, (Map) relParmEntry, valueList, pkMap, locale) } else { logger.warn("In entity values from plain map for entity ${ed.getFullEntityName()} found list for sub-object ${entryName} with a non-Map entry: ${relParmEntry}") } @@ -1987,7 +1987,7 @@ class EntityFacadeImpl implements EntityFacade { // NOTE: the following uses the same pattern as EntityDataLoaderImpl.LoadValueHandler if (dummyFks || useTryInsert) { EntityValue curValue = ed.makeEntityValue() - curValue.setAll(entry.getEtlValues()) + curValue.setAll(entry.getEtlValues(), null) if (useTryInsert) { try { curValue.create() diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy index e1ce053f8..f6005e844 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy @@ -172,7 +172,7 @@ abstract class EntityFindBase implements EntityFind { singleCondField = (String) null singleCondValue = null } - getEntityDef().entityInfo.setFields(fields, simpleAndMap, true, null, null) + getEntityDef().entityInfo.setFields(fields, simpleAndMap, true, null, null, null) } return this } diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityJavaUtil.java b/framework/src/main/groovy/org/moqui/impl/entity/EntityJavaUtil.java index 9629373a3..1418271a0 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityJavaUtil.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityJavaUtil.java @@ -352,7 +352,7 @@ public static class EntityInfo { } } - void setFields(Map src, Map dest, boolean setIfEmpty, String namePrefix, Boolean pks) { + void setFields(Map src, Map dest, boolean setIfEmpty, String namePrefix, Boolean pks, Locale locale) { if (src == null || dest == null) return; ExecutionContextImpl eci = efi.ecfi.getEci(); @@ -416,7 +416,7 @@ void setFields(Map src, Map dest, boolean setIfE } } - void setFieldsEv(Map src, EntityValueBase dest, Boolean pks) { + void setFieldsEv(Map src, EntityValueBase dest, Boolean pks, Locale locale) { // like above with setIfEmpty=true, namePrefix=null, pks=null if (src == null || dest == null) return; @@ -447,7 +447,7 @@ void setFieldsEv(Map src, EntityValueBase dest, Boolean pks) { if (!isEmpty) { if (isCharSequence) { try { - Object converted = fi.convertFromString(value.toString(), eci.l10nFacade); + Object converted = fi.convertFromString(value.toString(), eci.l10nFacade, locale); dest.putNoCheck(fieldName, converted); } catch (BaseException be) { eci.messageFacade.addValidationError(null, fieldName, null, be.getMessage(), be); diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java b/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java index 626777de3..d385c486d 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java @@ -401,9 +401,10 @@ public boolean primaryKeyMatches(EntityValueBase evb) { } @Override public EntityValue set(String name, Object value) { put(name, value); return this; } - @Override public EntityValue setAll(Map fields) { + @Override public EntityValue setAll(Map fields) { return setAll(fields, null); } + @Override public EntityValue setAll(Map fields, Locale locale) { if (!mutable) throw new EntityException("Cannot set fields, this entity value is not mutable (it is read-only)"); - getEntityDefinition().entityInfo.setFieldsEv(fields, this, null); + getEntityDefinition().entityInfo.setFieldsEv(fields, this, null, locale); return this; } @Override public EntityValue setString(String name, String value) { @@ -467,10 +468,13 @@ public boolean primaryKeyMatches(EntityValueBase evb) { } @Override public EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks) { + return setFields(fields, setIfEmpty, namePrefix, pks, null); + } + @Override public EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks, Locale locale) { if (!setIfEmpty && (namePrefix == null || namePrefix.length() == 0)) { - getEntityDefinition().entityInfo.setFields(fields, this, false, namePrefix, pks); + getEntityDefinition().entityInfo.setFields(fields, this, false, namePrefix, pks, locale); } else { - getEntityDefinition().entityInfo.setFieldsEv(fields, this, pks); + getEntityDefinition().entityInfo.setFieldsEv(fields, this, pks, locale); } return this; @@ -506,7 +510,7 @@ public EntityValue setSequencedIdSecondary() { this.remove(seqFieldName); Map otherPkMap = new LinkedHashMap<>(); - getEntityDefinition().entityInfo.setFields(this, otherPkMap, false, null, true); + getEntityDefinition().entityInfo.setFields(this, otherPkMap, false, null, true, null); // temporarily disable authz for this, just doing lookup to get next value and to allow for a // authorize-skip="create" with authorize-skip of view too this is necessary diff --git a/framework/src/main/groovy/org/moqui/impl/entity/FieldInfo.java b/framework/src/main/groovy/org/moqui/impl/entity/FieldInfo.java index c0d3fa1ad..d336b6719 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/FieldInfo.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/FieldInfo.java @@ -162,6 +162,9 @@ static BigDecimal safeStripZeroes(BigDecimal input) { } public Object convertFromString(String value, L10nFacadeImpl l10n) { + return convertFromString(value, l10n, null); + } + public Object convertFromString(String value, L10nFacadeImpl l10n, Locale locale) { if (value == null) return null; if ("null".equals(value)) return null; @@ -173,17 +176,17 @@ public Object convertFromString(String value, L10nFacadeImpl l10n) { case 1: outValue = value; break; case 2: // outValue = java.sql.Timestamp.valueOf(value); if (isEmpty) { outValue = null; break; } - outValue = l10n.parseTimestamp(value, null); + outValue = l10n.parseTimestamp(value, null, locale, null); if (outValue == null) throw new BaseArtifactException("The value [" + value + "] is not a valid date/time for field " + entityName + "." + name); break; case 3: // outValue = java.sql.Time.valueOf(value); if (isEmpty) { outValue = null; break; } - outValue = l10n.parseTime(value, null); + outValue = l10n.parseTime(value, null, locale); if (outValue == null) throw new BaseArtifactException("The value [" + value + "] is not a valid time for field " + entityName + "." + name); break; case 4: // outValue = java.sql.Date.valueOf(value); if (isEmpty) { outValue = null; break; } - outValue = l10n.parseDate(value, null); + outValue = l10n.parseDate(value, null, locale); if (outValue == null) throw new BaseArtifactException("The value [" + value + "] is not a valid date for field " + entityName + "." + name); break; case 5: // outValue = Integer.valueOf(value); break @@ -192,7 +195,7 @@ public Object convertFromString(String value, L10nFacadeImpl l10n) { case 8: // outValue = Double.valueOf(value); break case 9: // outValue = new BigDecimal(value); break if (isEmpty) { outValue = null; break; } - BigDecimal bdVal = l10n.parseNumber(value, null); + BigDecimal bdVal = l10n.parseNumber(value, null, locale); if (bdVal == null) { throw new BaseArtifactException("The value [" + value + "] is not valid for type " + javaType + " for field " + entityName + "." + name); } else { @@ -220,7 +223,7 @@ public Object convertFromString(String value, L10nFacadeImpl l10n) { case 13: outValue = value; break; case 14: if (isEmpty) { outValue = null; break; } - Timestamp ts = l10n.parseTimestamp(value, null); + Timestamp ts = l10n.parseTimestamp(value, null, locale, null); outValue = new java.util.Date(ts.getTime()); break; // better way for Collection (15)? maybe parse comma separated, but probably doesn't make sense in the first place diff --git a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy index 9a677ff0a..00971144d 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy @@ -116,7 +116,11 @@ class EntityAutoServiceRunner implements ServiceRunner { } protected static boolean checkAllPkFields(EntityDefinition ed, Map parameters, Map tempResult, - EntityValue newEntityValue, ArrayList outParamNames) { + EntityValue newEntityValue, ArrayList outParamNames) { + return checkAllPkFields(ed, parameters, tempResult, newEntityValue, outParamNames, null); + } + protected static boolean checkAllPkFields(EntityDefinition ed, Map parameters, Map tempResult, + EntityValue newEntityValue, ArrayList outParamNames, java.util.Locale locale) { FieldInfo[] pkFieldInfos = ed.entityInfo.pkFieldInfoArray // see if all PK fields were passed in @@ -167,7 +171,7 @@ class EntityAutoServiceRunner implements ServiceRunner { } } else if (allPksIn) { /* **** plain specified primary key **** */ - newEntityValue.setFields(parameters, true, null, true) + newEntityValue.setFields(parameters, true, null, true, locale) } else { logger.error("Entity [${ed.fullEntityName}] auto create pk fields ${ed.getPkFieldNames()} incomplete: ${parameters}" + "\nCould not find a valid combination of primary key settings to do a create operation; options include: " + @@ -305,12 +309,20 @@ class EntityAutoServiceRunner implements ServiceRunner { /** Does a create if record does not exist, or update if it does. */ static void storeEntity(ExecutionContextImpl eci, EntityDefinition ed, Map parameters, - Map result, ArrayList outParamNames) { - storeRecursive(eci.ecfi, eci.getEntityFacade(), ed, parameters, result, outParamNames, null) + Map result, ArrayList outParamNames) { + storeEntity(eci, ed, parameters, result, outParamNames, null) + } + static void storeEntity(ExecutionContextImpl eci, EntityDefinition ed, Map parameters, + Map result, ArrayList outParamNames, Locale locale) { + storeRecursive(eci.ecfi, eci.getEntityFacade(), ed, parameters, result, outParamNames, null, locale) } static void storeRecursive(ExecutionContextFactoryImpl ecfi, EntityFacadeImpl efi, EntityDefinition ed, Map parameters, Map result, ArrayList outParamNames, Map parentPks) { + storeRecursive( ecfi, efi, ed, parameters, result, outParamNames, parentPks, null) + } + static void storeRecursive(ExecutionContextFactoryImpl ecfi, EntityFacadeImpl efi, EntityDefinition ed, Map parameters, + Map result, ArrayList outParamNames, Map parentPks, Locale locale) { EntityValue newEntityValue = efi.makeValue(ed.getFullEntityName()) // add in all of the main entity's primary key fields, this is necessary for auto-generated, and to @@ -323,12 +335,12 @@ class EntityAutoServiceRunner implements ServiceRunner { checkFromDate(ed, parameters, result, ecfi) Map tempResult = [:] - boolean allPksIn = checkAllPkFields(ed, parameters, tempResult, newEntityValue, outParamNames) + boolean allPksIn = checkAllPkFields(ed, parameters, tempResult, newEntityValue, outParamNames, locale) result.putAll(tempResult) if (!allPksIn) { // we had to fill some stuff in, so do a create - newEntityValue.setFields(parameters, true, null, false) + newEntityValue.setFields(parameters, true, null, false, locale) newEntityValue.create() storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks) return @@ -343,7 +355,7 @@ class EntityAutoServiceRunner implements ServiceRunner { checkStatus(ed, parameters, result, outParamNames, lookedUpValue, efi) } else { // no lookedUpValue at this point? doesn't exist so create - newEntityValue.setFields(parameters, true, null, false) + newEntityValue.setFields(parameters, true, null, false, locale) newEntityValue.create() storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks) return @@ -351,15 +363,20 @@ class EntityAutoServiceRunner implements ServiceRunner { } if (lookedUpValue == null) lookedUpValue = newEntityValue - lookedUpValue.setFields(parameters, true, null, false) + lookedUpValue.setFields(parameters, true, null, false, locale) // logger.info("In auto updateEntity lookedUpValue final [${lookedUpValue}] for parameters [${parameters}]") lookedUpValue.createOrUpdate() - storeRelated(ecfi, efi, (EntityValueBase) lookedUpValue, parameters, result, parentPks) + storeRelated(ecfi, efi, (EntityValueBase) lookedUpValue, parameters, result, parentPks, locale) } static void storeRelated(ExecutionContextFactoryImpl ecfi, EntityFacadeImpl efi, EntityValueBase parentValue, Map parameters, Map result, Map parentPks) { + storeRelated(ecfi, efi, parentValue, parameters, result, parentPks, null) + } + static void storeRelated(ExecutionContextFactoryImpl ecfi, EntityFacadeImpl efi, EntityValueBase parentValue, + Map parameters, Map result, Map parentPks, + Locale locale) { EntityDefinition ed = parentValue.getEntityDefinition() // NOTE: keep a separate Map of parent PK values to pass down, can't just be current record's PK fields because @@ -408,14 +425,14 @@ class EntityAutoServiceRunner implements ServiceRunner { boolean isEntityValue = relParmObj instanceof EntityValue if (relParmObj instanceof Map && !isEntityValue) { Map relResults = [:] - storeRecursive(ecfi, efi, subEd, (Map) relParmObj, relResults, null, pkMap) + storeRecursive(ecfi, efi, subEd, (Map) relParmObj, relResults, null, pkMap, locale) result.put(entryName, relResults) } else if (relParmObj instanceof List) { List relResultList = [] for (Object relParmEntry in relParmObj) { Map relResults = [:] if (relParmEntry instanceof Map) { - storeRecursive(ecfi, efi, subEd, (Map) relParmEntry, relResults, null, pkMap) + storeRecursive(ecfi, efi, subEd, (Map) relParmEntry, relResults, null, pkMap, locale) } else { logger.warn("In entity auto create for entity ${ed.getFullEntityName()} found list for sub-object ${entryName} with a non-Map entry: ${relParmEntry}") } diff --git a/framework/src/main/java/org/moqui/entity/EntityValue.java b/framework/src/main/java/org/moqui/entity/EntityValue.java index 9a066c435..a82d4f758 100644 --- a/framework/src/main/java/org/moqui/entity/EntityValue.java +++ b/framework/src/main/java/org/moqui/entity/EntityValue.java @@ -90,6 +90,16 @@ public interface EntityValue extends Map, Externalizable, Compar */ EntityValue setAll(Map fields); + /** Sets fields on this entity from the Map of fields passed in using the entity definition to only get valid + * fields from the Map. For any String values passed in this will call setString to convert based on the field + * definition, otherwise it sets the Object as-is. + * + * @param fields The fields Map to get the values from + * @param locale The locale to use when parsing the values + * @return reference to this for convenience + */ + EntityValue setAll(Map fields, java.util.Locale locale); + /** Sets the named field to the passed value, converting the value from a String to the corresponding type using * Type.valueOf() * @@ -131,6 +141,20 @@ public interface EntityValue extends Map, Externalizable, Compar */ EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks); + /** Sets fields on this entity from the Map of fields passed in using the entity definition to only get valid + * fields from the Map. For any String values passed in this will call setString to convert based on the field + * definition, otherwise it sets the Object as-is. + * + * @param fields The fields Map to get the values from + * @param setIfEmpty Used to specify whether empty/null values in the field Map should be set + * @param namePrefix If not null or empty will be pre-pended to each field name (upper-casing the first letter of + * the field name first), and that will be used as the fields Map lookup name instead of the field-name + * @param pks If null, get all values, if TRUE just get PKs, if FALSE just get non-PKs + * @param locale Locale to use for parsing values + * @return reference to this for convenience + */ + EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks, java.util.Locale locale); + /** Get the next guaranteed unique seq id for this entity, and set it in the primary key field. This will set it in * the first primary key field in the entity definition, but it really should be used for entities with only one * primary key field. From ebcfd5e50756a22a80882d4399b33961b04f5731 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 18 Apr 2019 16:13:30 -0400 Subject: [PATCH 2/9] Make parseNumber with Locale parameter part of the public interface. --- .../src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java | 2 +- framework/src/main/java/org/moqui/context/L10nFacade.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java index a8d49e58d..86356c276 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java @@ -300,7 +300,7 @@ public String formatDateTime(Calendar input, String format, Locale locale, TimeZ @Override public BigDecimal parseNumber(String input, String format) { return parseNumber(input, format, null); } - public BigDecimal parseNumber(String input, String format, Locale locale) { + @Override public BigDecimal parseNumber(String input, String format, Locale locale) { if (locale == null) locale = getLocale(); return bigDecimalValidator.validate(input, format, locale); } public String formatNumber(Number input, String format, Locale locale) { diff --git a/framework/src/main/java/org/moqui/context/L10nFacade.java b/framework/src/main/java/org/moqui/context/L10nFacade.java index 2fcdca82d..ca508b6e3 100644 --- a/framework/src/main/java/org/moqui/context/L10nFacade.java +++ b/framework/src/main/java/org/moqui/context/L10nFacade.java @@ -61,4 +61,5 @@ public interface L10nFacade { java.util.Calendar parseDateTime(String input, String format); java.math.BigDecimal parseNumber(String input, String format); + java.math.BigDecimal parseNumber(String input, String format, Locale locale); } From 21783f2ff58ae09e5797a1b300145ae12273b143 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Wed, 24 Apr 2019 18:02:59 -0400 Subject: [PATCH 3/9] Changes to comply with some Codacy issues --- .../moqui/impl/context/L10nFacadeImpl.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java index 86356c276..f2759071a 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java @@ -136,16 +136,16 @@ public Time parseTime(String input, String format) { return parseTime(input, format, null); } public Time parseTime(String input, String format, Locale locale) { - if (locale == null) locale = getLocale(); + Locale curLocale = locale != null ? locale : getLocale(); TimeZone curTz = getTimeZone(); if (format == null || format.isEmpty()) format = "HH:mm:ss.SSS"; - Calendar cal = calendarValidator.validate(input, format, locale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "HH:mm:ss", locale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "HH:mm", locale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "h:mm a", locale, curTz); - if (cal == null) cal = calendarValidator.validate(input, "h:mm:ss a", locale, curTz); + Calendar cal = calendarValidator.validate(input, format, curLocale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "HH:mm:ss", curLocale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "HH:mm", curLocale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "h:mm a", curLocale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "h:mm:ss a", curLocale, curTz); // also try the full ISO-8601, times may come in that way (even if funny with a date of 1970-01-01) - if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale, curTz); + if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale, curTz); if (cal != null) { Time time = new Time(cal.getTimeInMillis()); // logger.warn("============== parseTime input=${input} cal=${cal} long=${cal.getTimeInMillis()} time=${time} time long=${time.getTime()} util date=${new java.util.Date(cal.getTimeInMillis())} timestamp=${new java.sql.Timestamp(cal.getTimeInMillis())}") @@ -163,7 +163,7 @@ public Time parseTime(String input, String format, Locale locale) { return null; } public String formatTime(Time input, String format, Locale locale, TimeZone tz) { - if (locale == null) locale = getLocale(); + Locale curLocale = locale != null ? locale : getLocale(); if (tz == null) tz = getTimeZone(); if (format == null || format.isEmpty()) format = "HH:mm:ss"; String timeStr = calendarValidator.format(input, format, locale, tz); @@ -301,32 +301,30 @@ public String formatDateTime(Calendar input, String format, Locale locale, TimeZ return parseNumber(input, format, null); } @Override public BigDecimal parseNumber(String input, String format, Locale locale) { - if (locale == null) locale = getLocale(); - return bigDecimalValidator.validate(input, format, locale); } + Locale curLocale = locale != null? locale: getLocale(); + return bigDecimalValidator.validate(input, format, curLocale); } public String formatNumber(Number input, String format, Locale locale) { - if (locale == null) locale = getLocale(); - return bigDecimalValidator.format(input, format, locale); + Locale curLocale = locale != null ? locale : getLocale(); + return bigDecimalValidator.format(input, format, curLocale); } @Override - public String format(Object value, String format) { - return this.format(value, format, getLocale(), getTimeZone()); - } + public String format(Object value, String format) { return this.format(value, format, null, getTimeZone()); } @Override public String format(Object value, String format, Locale locale, TimeZone tz) { if (value == null) return ""; - if (locale == null) locale = getLocale(); + Locale curLocale = locale != null ? locale : getLocale(); if (tz == null) tz = getTimeZone(); Class valueClass = value.getClass(); if (valueClass == String.class) return (String) value; - if (valueClass == Timestamp.class) return formatTimestamp((Timestamp) value, format, locale, tz); - if (valueClass == java.util.Date.class) return formatTimestamp((java.util.Date) value, format, locale, tz); - if (valueClass == java.sql.Date.class) return formatDate((Date) value, format, locale, tz); - if (valueClass == Time.class) return formatTime((Time) value, format, locale, tz); + if (valueClass == Timestamp.class) return formatTimestamp((Timestamp) value, format, curLocale, tz); + if (valueClass == java.util.Date.class) return formatTimestamp((java.util.Date) value, format, curLocale, tz); + if (valueClass == java.sql.Date.class) return formatDate((Date) value, format, curLocale, tz); + if (valueClass == Time.class) return formatTime((Time) value, format, curLocale, tz); // this one needs to be instanceof to include the many sub-classes of Number - if (value instanceof Number) return formatNumber((Number) value, format, locale); + if (value instanceof Number) return formatNumber((Number) value, format, curLocale); // Calendar is an abstract class, so must use instanceof here as well - if (value instanceof Calendar) return formatDateTime((Calendar) value, format, locale, tz); + if (value instanceof Calendar) return formatDateTime((Calendar) value, format, curLocale, tz); return value.toString(); } } From f09537540208da31f702ff60e81b351f2abed3d4 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Wed, 24 Apr 2019 18:03:39 -0400 Subject: [PATCH 4/9] Add attribute to specify locale in xml based data --- .../moqui/impl/entity/EntityDataLoaderImpl.groovy | 13 +++++++++++-- .../moqui/impl/entity/EntityDataWriterImpl.groovy | 2 +- framework/xsd/entity-definition-2.1.xsd | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy index 8e1498138..af4e97491 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy @@ -563,8 +563,15 @@ class EntityDataLoaderImpl implements EntityDataLoader { void startElement(String ns, String localName, String qName, Attributes attributes) { // logger.info("startElement ns [${ns}], localName [${localName}] qName [${qName}]") String type = null - if (qName == "entity-facade-xml") { type = attributes.getValue("type") } - else if (qName == "seed-data") { type = "seed" } + String localeStr = null + if (qName == "entity-facade-xml") { + type = attributes.getValue("type") + localeStr = attributes.getValue("locale") + } + else if (qName == "seed-data") { + type = "seed" + localeStr = attributes.getValue("locale") + } if (type && edli.dataTypes && !edli.dataTypes.contains(type)) { if (logger.isInfoEnabled()) logger.info("Skipping file [${location}], is a type to skip (${type})") throw new TypeToSkipException() @@ -579,6 +586,8 @@ class EntityDataLoaderImpl implements EntityDataLoader { } if (!loadElements) return + if (localeStr != null) locale = new Locale(localeStr) + String elementName = qName // get everything after a colon, but replace - with # for verb#noun separation if (elementName.contains(':')) elementName = elementName.substring(elementName.indexOf(':') + 1) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy index b19f07a89..1f426574b 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy @@ -304,7 +304,7 @@ class EntityDataWriterImpl implements EntityDataWriter { writer.println("[") } else { writer.println("") - writer.println("") + writer.println("") } } private void endFile(Writer writer) { diff --git a/framework/xsd/entity-definition-2.1.xsd b/framework/xsd/entity-definition-2.1.xsd index 3208fff11..1881fe0bc 100644 --- a/framework/xsd/entity-definition-2.1.xsd +++ b/framework/xsd/entity-definition-2.1.xsd @@ -336,7 +336,10 @@ along with this software (see the LICENSE.md file). If not, see --> - + + Set the locale to be used when parsing the string data to the corresponding data type. + Relevant mostly for decimal numbers where differing decimal separators result in different interpretations. + From 98854ce11fd9119f7e292b3181bc7d7697704514 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Wed, 24 Apr 2019 21:49:55 -0400 Subject: [PATCH 5/9] Improve locale handling when loading data --- .../impl/entity/EntityDataLoaderImpl.groovy | 55 ++++++++++++++----- .../runner/EntityAutoServiceRunner.groovy | 4 +- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy index af4e97491..150ff4d1e 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy @@ -183,7 +183,6 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.disableFkCreate) reenableFkCreate = !eci.artifactExecutionFacade.disableEntityFkCreate() boolean reenableDataFeed = false if (this.disableDataFeed) reenableDataFeed = !eci.artifactExecutionFacade.disableEntityDataFeed() - Locale locale = new Locale("en_US") // if no xmlText or locations, so find all of the component and entity-facade files if (!this.xmlText && !this.csvText && !this.jsonText && !this.locationList) { @@ -266,7 +265,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.csvText) { InputStream csvInputStream = new ByteArrayInputStream(csvText.getBytes("UTF-8")) try { - tf.runUseOrBegin(transactionTimeout, "Error loading CSV entity data", { ech.loadFile("csvText", csvInputStream, locale) }) + tf.runUseOrBegin(transactionTimeout, "Error loading CSV entity data", { ech.loadFile("csvText", csvInputStream) }) } finally { if (csvInputStream != null) csvInputStream.close() } @@ -276,14 +275,14 @@ class EntityDataLoaderImpl implements EntityDataLoader { if (this.jsonText) { InputStream jsonInputStream = new ByteArrayInputStream(jsonText.getBytes("UTF-8")) try { - tf.runUseOrBegin(transactionTimeout, "Error loading JSON entity data", { ejh.loadFile("jsonText", jsonInputStream, locale) }) + tf.runUseOrBegin(transactionTimeout, "Error loading JSON entity data", { ejh.loadFile("jsonText", jsonInputStream) }) } finally { if (jsonInputStream != null) jsonInputStream.close() } } // load each file in its own transaction - for (String location in this.locationList) loadSingleFile(location, exh, ech, ejh, locale) + for (String location in this.locationList) loadSingleFile(location, exh, ech, ejh) }) if (reenableEeca) eci.artifactExecutionFacade.enableEntityEca() @@ -295,7 +294,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { // Thread.sleep(60*1000*100) } - void loadSingleFile(String location, EntityXmlHandler exh, EntityCsvHandler ech, EntityJsonHandler ejh, Locale locale) { + void loadSingleFile(String location, EntityXmlHandler exh, EntityCsvHandler ech, EntityJsonHandler ejh) { TransactionFacade tf = efi.ecfi.transactionFacade boolean beganTransaction = tf.begin(transactionTimeout) try { @@ -314,12 +313,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { logger.info("Loaded ${(exh.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } else if (location.endsWith(".csv")) { long beforeRecords = ech.valuesRead ?: 0 - if (ech.loadFile(location, inputStream, locale)) { + if (ech.loadFile(location, inputStream)) { logger.info("Loaded ${(ech.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } } else if (location.endsWith(".json")) { long beforeRecords = ejh.valuesRead ?: 0 - if (ejh.loadFile(location, inputStream, locale)) { + if (ejh.loadFile(location, inputStream)) { logger.info("Loaded ${(ejh.valuesRead?:0) - beforeRecords} records from ${location} in ${((System.currentTimeMillis() - beforeTime)/1000)}s") } } else if (location.endsWith(".zip")) { @@ -547,7 +546,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { protected long valuesRead = 0 protected List messageList = new LinkedList<>() String location - Locale locale = new Locale("en_US") + Locale locale = null protected boolean loadElements = false @@ -577,6 +576,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { throw new TypeToSkipException() } + if (localeStr != null) { + int usIdx = localeStr.indexOf("_") + locale = (usIdx < 0 ? new Locale(localeStr) : + new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase())) + } + if (qName == "entity-facade-xml") { loadElements = true return @@ -586,8 +591,6 @@ class EntityDataLoaderImpl implements EntityDataLoader { } if (!loadElements) return - if (localeStr != null) locale = new Locale(localeStr) - String elementName = qName // get everything after a colon, but replace - with # for verb#noun separation if (elementName.contains(':')) elementName = elementName.substring(elementName.indexOf(':') + 1) @@ -774,7 +777,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { valueHandler.handleValue(curValue) valuesRead++ } else { - valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap, locale) + valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap, getLocale()) valuesRead++ } currentEntityDef = (EntityDefinition) null @@ -806,6 +809,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { } void setDocumentLocator(Locator locator) { this.locator = locator } + + void setLocale(Locale locale) { this.locale = locale } + Locale getLocale() { + if (locale == null) locale = new Locale("en", "US") + return locale + } } static class EntityCsvHandler { @@ -815,6 +824,8 @@ class EntityDataLoaderImpl implements EntityDataLoader { protected long valuesRead = 0 protected List messageList = new LinkedList() + protected Locale locale = null + EntityCsvHandler(EntityDataLoaderImpl edli, ValueHandler valueHandler) { this.edli = edli this.valueHandler = valueHandler @@ -824,7 +835,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { long getValuesRead() { return valuesRead } List getMessageList() { return messageList } - boolean loadFile(String location, InputStream is, Locale locale) { + boolean loadFile(String location, InputStream is) { BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")) CSVParser parser = CSVFormat.newFormat(edli.csvDelimiter) @@ -911,6 +922,13 @@ class EntityDataLoaderImpl implements EntityDataLoader { } return true } + + void setLocale(Locale locale) { this.locale = locale } + Locale getLocale() { + if (locale == null) locale = new Locale("en", "US") + return locale + } + } static class EntityJsonHandler { @@ -920,6 +938,8 @@ class EntityDataLoaderImpl implements EntityDataLoader { protected long valuesRead = 0 protected List messageList = new LinkedList() + protected Locale locale = null + EntityJsonHandler(EntityDataLoaderImpl edli, ValueHandler valueHandler) { this.edli = edli this.valueHandler = valueHandler @@ -929,7 +949,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { long getValuesRead() { return valuesRead } List getMessageList() { return messageList } - boolean loadFile(String location, InputStream is, Locale locale) { + boolean loadFile(String location, InputStream is) { JsonSlurper slurper = new JsonSlurper() Object jsonObj try { @@ -990,7 +1010,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { valueHandler.handleService(currentScs) valuesRead++ } else { - valueHandler.handlePlainMap(entityName, value, locale) + valueHandler.handlePlainMap(entityName, value, getLocale()) // TODO: make this more complete, like counting nested Maps? valuesRead++ } @@ -998,5 +1018,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { return true } + + void setLocale(Locale locale) { this.locale = locale } + Locale getLocale() { + if (locale == null) locale = new Locale("en", "US") + return locale + } + } } diff --git a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy index 00971144d..78e40774d 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy @@ -120,7 +120,7 @@ class EntityAutoServiceRunner implements ServiceRunner { return checkAllPkFields(ed, parameters, tempResult, newEntityValue, outParamNames, null); } protected static boolean checkAllPkFields(EntityDefinition ed, Map parameters, Map tempResult, - EntityValue newEntityValue, ArrayList outParamNames, java.util.Locale locale) { + EntityValue newEntityValue, ArrayList outParamNames, Locale locale) { FieldInfo[] pkFieldInfos = ed.entityInfo.pkFieldInfoArray // see if all PK fields were passed in @@ -357,7 +357,7 @@ class EntityAutoServiceRunner implements ServiceRunner { // no lookedUpValue at this point? doesn't exist so create newEntityValue.setFields(parameters, true, null, false, locale) newEntityValue.create() - storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks) + storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks, locale) return } } From caae2f06c5e4f278789ee8dd3e6ccf54423c2be6 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 25 Apr 2019 09:19:19 -0400 Subject: [PATCH 6/9] Add third parameter for locale for CSV based data loading --- .../impl/entity/EntityDataLoaderImpl.groovy | 12 ++++++++++-- .../moqui/impl/entity/EntityDefinition.groovy | 5 +++-- .../org/moqui/impl/entity/EntityValueBase.java | 5 +++-- .../java/org/moqui/entity/EntityValue.java | 18 ++++++++++++++++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy index 150ff4d1e..ce83db6cc 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy @@ -876,6 +876,14 @@ class EntityDataLoaderImpl implements EntityDataLoader { return false } } + + if (firstLineRecord.size() > 2) { + // third field is locale + String localeStr = firstLineRecord.get(2) + int usIdx = localeStr.indexOf("_") + locale = usIdx < 0 ? new Locale(localeStr) : + new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase()) + } } Map headerMap = [:] @@ -903,9 +911,9 @@ class EntityDataLoaderImpl implements EntityDataLoader { valuesRead++ } else { EntityValueImpl currentEntityValue = (EntityValueImpl) edli.efi.makeValue(entityName) - if (edli.defaultValues) currentEntityValue.setFields(edli.defaultValues, true, null, null) + if (edli.defaultValues) currentEntityValue.setFields(edli.defaultValues, true, null, null, locale) for (Map.Entry header in headerMap) - currentEntityValue.setString(header.key, record.get(header.value)) + currentEntityValue.setString(header.key, record.get(header.value), locale) if (!currentEntityValue.containsPrimaryKey()) { if (currentEntityValue.getEntityDefinition().getPkFieldNames().size() == 1) { diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy index fc716aa79..f7a58b90f 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy @@ -952,11 +952,12 @@ class EntityDefinition { return mePkFieldToAliasNameMap } - Object convertFieldString(String name, String value, ExecutionContextImpl eci) { + Object convertFieldString(String name, String value, ExecutionContextImpl eci) { return convertFieldString(name, value, eci, null) } + Object convertFieldString(String name, String value, ExecutionContextImpl eci, Locale locale) { if (value == null) return null FieldInfo fieldInfo = getFieldInfo(name) if (fieldInfo == null) throw new EntityException("Invalid field name ${name} for entity ${fullEntityName}") - return fieldInfo.convertFromString(value, eci.l10nFacade) + return fieldInfo.convertFromString(value, eci.l10nFacade, locale) } static String getFieldStringForFile(FieldInfo fieldInfo, Object value) { diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java b/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java index d385c486d..0f443dfa8 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityValueBase.java @@ -407,10 +407,11 @@ public boolean primaryKeyMatches(EntityValueBase evb) { getEntityDefinition().entityInfo.setFieldsEv(fields, this, null, locale); return this; } - @Override public EntityValue setString(String name, String value) { + @Override public EntityValue setString(String name, String value) { return setString(name, value, null); } + @Override public EntityValue setString(String name, String value, Locale locale) { // this will do a field name check ExecutionContextImpl eci = getEntityFacadeImpl().ecfi.getEci(); - Object converted = getEntityDefinition().convertFieldString(name, value, eci); + Object converted = getEntityDefinition().convertFieldString(name, value, eci, locale); putNoCheck(name, converted); return this; } diff --git a/framework/src/main/java/org/moqui/entity/EntityValue.java b/framework/src/main/java/org/moqui/entity/EntityValue.java index a82d4f758..7eefa91e6 100644 --- a/framework/src/main/java/org/moqui/entity/EntityValue.java +++ b/framework/src/main/java/org/moqui/entity/EntityValue.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Locale; /** Entity Value Interface - Represents a single database record. */ @@ -98,7 +99,20 @@ public interface EntityValue extends Map, Externalizable, Compar * @param locale The locale to use when parsing the values * @return reference to this for convenience */ - EntityValue setAll(Map fields, java.util.Locale locale); + EntityValue setAll(Map fields, Locale locale); + + /** Sets the named field to the passed value, converting the value from a String to the corresponding type using + * Type.valueOf() + * + * If the String "null" is passed in it will be treated the same as a null value. If you really want to set a + * String of "null" then pass in "\null". + * + * @param name The field name to set + * @param value The String value to convert and set + * @param locale The locale to use when parsing the values + * @return reference to this for convenience + */ + EntityValue setString(String name, String value, Locale locale); /** Sets the named field to the passed value, converting the value from a String to the corresponding type using * Type.valueOf() @@ -153,7 +167,7 @@ public interface EntityValue extends Map, Externalizable, Compar * @param locale Locale to use for parsing values * @return reference to this for convenience */ - EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks, java.util.Locale locale); + EntityValue setFields(Map fields, boolean setIfEmpty, String namePrefix, Boolean pks, Locale locale); /** Get the next guaranteed unique seq id for this entity, and set it in the primary key field. This will set it in * the first primary key field in the entity definition, but it really should be used for entities with only one From 526ed9aa7111039e8f45a62e00c433e2afa82cc9 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 25 Apr 2019 10:08:44 -0400 Subject: [PATCH 7/9] Add option to specify global and per-value locale for Json based data imports --- .../impl/entity/EntityDataLoaderImpl.groovy | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy index ce83db6cc..a67bba023 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy @@ -969,18 +969,21 @@ class EntityDataLoaderImpl implements EntityDataLoader { } String type = null + String localeStr = null List valueList if (jsonObj instanceof Map) { Map jsonMap = (Map) jsonObj type = jsonMap.get("_dataType") + localeStr = jsonMap.get("_locale") valueList = [jsonObj] } else if (jsonObj instanceof List) { valueList = (List) jsonObj Object firstValue = valueList?.get(0) if (firstValue instanceof Map) { Map firstValMap = (Map) firstValue - if (firstValMap.get("_dataType")) { + if (firstValMap.get("_dataType") || firstValMap.get("_locale")) { type = firstValMap.get("_dataType") + localeStr = firstValMap.get("_locale") valueList.remove((int) 0I) } } @@ -993,6 +996,12 @@ class EntityDataLoaderImpl implements EntityDataLoader { return false } + if (localeStr != null) { + int usIdx = localeStr.indexOf("_") + locale = usIdx < 0 ? new Locale(localeStr) : + new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase()) + } + for (Object valueObj in valueList) { if (!(valueObj instanceof Map)) { logger.warn("Found non-Map object in JSON import, skipping: ${valueObj}") @@ -1004,6 +1013,13 @@ class EntityDataLoaderImpl implements EntityDataLoader { value.putAll((Map) valueObj) String entityName = value."_entity" + String valueLocaleStr = value."_locale" + Locale valueLocale = null + if (valueLocaleStr) { + int usIdx = valueLocaleStr.indexOf("_") + valueLocale = usIdx < 0 ? new Locale(valueLocaleStr) : + new Locale(valueLocaleStr.substring(0, usIdx), valueLocaleStr.substring(usIdx+1).toUpperCase()) + } boolean isService if (edli.efi.isEntityDefined(entityName)) { isService = false @@ -1018,7 +1034,7 @@ class EntityDataLoaderImpl implements EntityDataLoader { valueHandler.handleService(currentScs) valuesRead++ } else { - valueHandler.handlePlainMap(entityName, value, getLocale()) + valueHandler.handlePlainMap(entityName, value, valueLocale?:getLocale()) // TODO: make this more complete, like counting nested Maps? valuesRead++ } From 1ed0c36dd98f7c4a0119bcca8cfa090a5342c6cd Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 25 Apr 2019 10:25:39 -0400 Subject: [PATCH 8/9] fix locale handling --- .../groovy/org/moqui/impl/context/L10nFacadeImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java index f2759071a..568a7dd98 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java @@ -166,7 +166,7 @@ public String formatTime(Time input, String format, Locale locale, TimeZone tz) Locale curLocale = locale != null ? locale : getLocale(); if (tz == null) tz = getTimeZone(); if (format == null || format.isEmpty()) format = "HH:mm:ss"; - String timeStr = calendarValidator.format(input, format, locale, tz); + String timeStr = calendarValidator.format(input, format, curLocale, tz); // logger.warn("============= formatTime input=${input} timeStr=${timeStr} long=${input.getTime()}") return timeStr; } @@ -176,8 +176,8 @@ public java.sql.Date parseDate(String input, String format) { return parseDate(input, format, null); } public java.sql.Date parseDate(String input, String format, Locale locale) { + Locale curLocale = locale != null ? locale : getLocale(); if (format == null || format.isEmpty()) format = "yyyy-MM-dd"; - if (locale == null) locale = getLocale(); // NOTE DEJ 20150317 Date parsing in terms of time zone causes funny issues because the time part of the long // since epoch representation is lost going to/from the DB, especially since the time portion is set to 0 and @@ -194,10 +194,10 @@ public java.sql.Date parseDate(String input, String format, Locale locale) { if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale, curTz) */ - Calendar cal = calendarValidator.validate(input, format, locale); - if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", locale); + Calendar cal = calendarValidator.validate(input, format, curLocale); + if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", curLocale); // also try the full ISO-8601, dates may come in that way - if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale); + if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale); if (cal != null) { java.sql.Date date = new java.sql.Date(cal.getTimeInMillis()); // logger.warn("============== parseDate input=${input} cal=${cal} long=${cal.getTimeInMillis()} date=${date} date long=${date.getTime()} util date=${new java.util.Date(cal.getTimeInMillis())} timestamp=${new java.sql.Timestamp(cal.getTimeInMillis())}") From df11365fc801e5424fce7579f23fd5b6123bac6f Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Sat, 20 Jul 2019 21:33:43 -0400 Subject: [PATCH 9/9] Handle locale in store case that was left out --- .../moqui/impl/service/runner/EntityAutoServiceRunner.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy index 78e40774d..58345d103 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/runner/EntityAutoServiceRunner.groovy @@ -342,7 +342,7 @@ class EntityAutoServiceRunner implements ServiceRunner { // we had to fill some stuff in, so do a create newEntityValue.setFields(parameters, true, null, false, locale) newEntityValue.create() - storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks) + storeRelated(ecfi, efi, (EntityValueBase) newEntityValue, parameters, result, parentPks, locale) return }