Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Force XML and CSV sources to be interpreted using the en_US locale #356

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Force XML and CSV sources to be interpreted using the en_US locale #356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bde0bec
Add methods that pass locale when adding string-based values to entities
jenshp Apr 18, 2019
ebcfd5e
Make parseNumber with Locale parameter part of the public interface.
jenshp Apr 18, 2019
21783f2
Changes to comply with some Codacy issues
jenshp Apr 24, 2019
f095375
Add attribute to specify locale in xml based data
jenshp Apr 24, 2019
98854ce
Improve locale handling when loading data
jenshp Apr 25, 2019
caae2f0
Add third parameter for locale for CSV based data loading
jenshp Apr 25, 2019
526ed9a
Add option to specify global and per-value locale for Json based data…
jenshp Apr 25, 2019
1ed0c36
fix locale handling
jenshp Apr 25, 2019
e751569
Merge branch 'master' into entityLoadLocale
jenshp Jul 20, 2019
df11365
Handle locale in store case that was left out
jenshp Jul 21, 2019
822069e
Merge branch 'master' into entityLoadLocale
jenshp Jul 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 27 additions & 19 deletions framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,10 @@ public BigDecimal roundCurrency(BigDecimal amount, String uomId, boolean precise

@Override
public Time parseTime(String input, String format) {
Locale curLocale = getLocale();
return parseTime(input, format, null);
}
public Time parseTime(String input, String format, Locale locale) {
Locale curLocale = locale != null ? locale : getLocale();
TimeZone curTz = getTimeZone();
if (format == null || format.isEmpty()) format = "HH:mm:ss.SSS";
Calendar cal = calendarValidator.validate(input, format, curLocale, curTz);
Expand All @@ -172,18 +175,21 @@ public Time parseTime(String input, String format) {
return null;
}
public String formatTime(Time input, String format, Locale locale, TimeZone tz) {
if (locale == null) locale = getLocale();
Locale curLocale = locale != null ? locale : getLocale();
if (tz == null) tz = getTimeZone();
if (format == null || format.isEmpty()) format = "HH:mm:ss";
String timeStr = calendarValidator.format(input, format, locale, tz);
String timeStr = calendarValidator.format(input, format, curLocale, tz);
// logger.warn("============= formatTime input=${input} timeStr=${timeStr} long=${input.getTime()}")
return timeStr;
}

@Override
public java.sql.Date parseDate(String input, String format) {
return parseDate(input, format, null);
}
public java.sql.Date parseDate(String input, String format, Locale locale) {
Locale curLocale = locale != null ? locale : getLocale();
if (format == null || format.isEmpty()) format = "yyyy-MM-dd";
Locale curLocale = getLocale();

// NOTE DEJ 20150317 Date parsing in terms of time zone causes funny issues because the time part of the long
// since epoch representation is lost going to/from the DB, especially since the time portion is set to 0 and
Expand All @@ -195,9 +201,9 @@ public java.sql.Date parseDate(String input, String format) {
/*
TimeZone curTz = getTimeZone()
Calendar cal = calendarValidator.validate(input, format, curLocale, curTz)
if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", curLocale, curTz)
if (cal == null) cal = calendarValidator.validate(input, "MM/dd/yyyy", locale, curTz)
// also try the full ISO-8601, dates may come in that way
if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", curLocale, curTz)
if (cal == null) cal = calendarValidator.validate(input, "yyyy-MM-dd'T'HH:mm:ssZ", locale, curTz)
*/

Calendar cal = calendarValidator.validate(input, format, curLocale);
Expand Down Expand Up @@ -304,31 +310,33 @@ public String formatDateTime(Calendar input, String format, Locale locale, TimeZ
}

@Override public BigDecimal parseNumber(String input, String format) {
return bigDecimalValidator.validate(input, format, getLocale()); }
return parseNumber(input, format, null);
}
@Override public BigDecimal parseNumber(String input, String format, Locale locale) {
Locale curLocale = locale != null? locale: getLocale();
return bigDecimalValidator.validate(input, format, curLocale); }
public String formatNumber(Number input, String format, Locale locale) {
if (locale == null) locale = getLocale();
return bigDecimalValidator.format(input, format, locale);
Locale curLocale = locale != null ? locale : getLocale();
return bigDecimalValidator.format(input, format, curLocale);
}

@Override
public String format(Object value, String format) {
return this.format(value, format, getLocale(), getTimeZone());
}
public String format(Object value, String format) { return this.format(value, format, null, getTimeZone()); }
@Override
public String format(Object value, String format, Locale locale, TimeZone tz) {
if (value == null) return "";
if (locale == null) locale = getLocale();
Locale curLocale = locale != null ? locale : getLocale();
if (tz == null) tz = getTimeZone();
Class valueClass = value.getClass();
if (valueClass == String.class) return (String) value;
if (valueClass == Timestamp.class) return formatTimestamp((Timestamp) value, format, locale, tz);
if (valueClass == java.util.Date.class) return formatTimestamp((java.util.Date) value, format, locale, tz);
if (valueClass == java.sql.Date.class) return formatDate((Date) value, format, locale, tz);
if (valueClass == Time.class) return formatTime((Time) value, format, locale, tz);
if (valueClass == Timestamp.class) return formatTimestamp((Timestamp) value, format, curLocale, tz);
if (valueClass == java.util.Date.class) return formatTimestamp((java.util.Date) value, format, curLocale, tz);
if (valueClass == java.sql.Date.class) return formatDate((Date) value, format, curLocale, tz);
if (valueClass == Time.class) return formatTime((Time) value, format, curLocale, tz);
// this one needs to be instanceof to include the many sub-classes of Number
if (value instanceof Number) return formatNumber((Number) value, format, locale);
if (value instanceof Number) return formatNumber((Number) value, format, curLocale);
// Calendar is an abstract class, so must use instanceof here as well
if (value instanceof Calendar) return formatDateTime((Calendar) value, format, locale, tz);
if (value instanceof Calendar) return formatDateTime((Calendar) value, format, curLocale, tz);
return value.toString();
}
}
95 changes: 79 additions & 16 deletions framework/src/main/groovy/org/moqui/impl/entity/EntityDataLoaderImpl.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import javax.xml.parsers.SAXParserFactory
import java.nio.charset.StandardCharsets
import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream
import java.util.Locale

@CompileStatic
class EntityDataLoaderImpl implements EntityDataLoader {
Expand Down Expand Up @@ -383,7 +384,7 @@ class EntityDataLoaderImpl implements EntityDataLoader {
ValueHandler(EntityDataLoaderImpl edli) { this.edli = edli }

abstract void handleValue(EntityValue value)
abstract void handlePlainMap(String entityName, Map value)
abstract void handlePlainMap(String entityName, Map value, Locale locale)
abstract void handleService(ServiceCallSync scs)
}
static class CheckValueHandler extends ValueHandler {
Expand All @@ -401,8 +402,8 @@ class EntityDataLoaderImpl implements EntityDataLoader {

long getFieldsChecked() { return fieldsChecked }
void handleValue(EntityValue value) { value.checkAgainstDatabase(messageList) }
void handlePlainMap(String entityName, Map value) {
EntityList el = edli.getEfi().getValueListFromPlainMap(value, entityName)
void handlePlainMap(String entityName, Map value, Locale locale) {
EntityList el = edli.getEfi().getValueListFromPlainMap(value, entityName, locale)
// logger.warn("=========== Check value: ${value}\nel: ${el}")
for (EntityValue ev in el) fieldsChecked += ev.checkAgainstDatabase(messageList)
}
Expand Down Expand Up @@ -461,11 +462,11 @@ class EntityDataLoaderImpl implements EntityDataLoader {
value.createOrUpdate()
}
}
void handlePlainMap(String entityName, Map value) {
void handlePlainMap(String entityName, Map value, Locale locale) {
EntityDefinition ed = ec.entityFacade.getEntityDefinition(entityName)
if (ed == null) throw new BaseException("Could not find entity ${entityName}")
if (edli.onlyCreate) {
EntityList el = ec.entityFacade.getValueListFromPlainMap(value, entityName)
EntityList el = ec.entityFacade.getValueListFromPlainMap(value, entityName, locale)
int elSize = el.size()
for (int i = 0; i < elSize; i++) {
EntityValue curValue = (EntityValue) el.get(i)
Expand All @@ -480,7 +481,7 @@ class EntityDataLoaderImpl implements EntityDataLoader {
}
} else {
Map<String, Object> results = new HashMap()
EntityAutoServiceRunner.storeEntity(ec, ed, value, results, null)
EntityAutoServiceRunner.storeEntity(ec, ed, value, results, null, locale)
// no need to call the store auto service, use storeEntity directly:
// Map results = sfi.sync().name('store', entityName).parameters(value).call()
if (logger.isTraceEnabled()) logger.trace("Called store service for entity [${entityName}] in data load, results: ${results}")
Expand Down Expand Up @@ -516,9 +517,9 @@ class EntityDataLoaderImpl implements EntityDataLoader {
void handleValue(EntityValue value) {
el.add(value)
}
void handlePlainMap(String entityName, Map value) {
void handlePlainMap(String entityName, Map value, Locale locale) {
EntityDefinition ed = edli.getEfi().getEntityDefinition(entityName)
edli.getEfi().addValuesFromPlainMapRecursive(ed, value, el, null)
edli.getEfi().addValuesFromPlainMapRecursive(ed, value, el, null, locale)
}
void handleService(ServiceCallSync scs) { logger.warn("For load to EntityList not calling service [${scs.getServiceName()}] with parameters ${scs.getCurrentParameters()}") }
}
Expand All @@ -545,6 +546,7 @@ class EntityDataLoaderImpl implements EntityDataLoader {
protected long valuesRead = 0
protected List<String> messageList = new LinkedList<>()
String location
Locale locale = null

protected boolean loadElements = false

Expand All @@ -560,13 +562,26 @@ class EntityDataLoaderImpl implements EntityDataLoader {
void startElement(String ns, String localName, String qName, Attributes attributes) {
// logger.info("startElement ns [${ns}], localName [${localName}] qName [${qName}]")
String type = null
if (qName == "entity-facade-xml") { type = attributes.getValue("type") }
else if (qName == "seed-data") { type = "seed" }
String localeStr = null
if (qName == "entity-facade-xml") {
type = attributes.getValue("type")
localeStr = attributes.getValue("locale")
}
else if (qName == "seed-data") {
type = "seed"
localeStr = attributes.getValue("locale")
}
if (type && edli.dataTypes && !edli.dataTypes.contains(type)) {
if (logger.isInfoEnabled()) logger.info("Skipping file [${location}], is a type to skip (${type})")
throw new TypeToSkipException()
}

if (localeStr != null) {
int usIdx = localeStr.indexOf("_")
locale = (usIdx < 0 ? new Locale(localeStr) :
new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase()))
}

if (qName == "entity-facade-xml") {
loadElements = true
return
Expand Down Expand Up @@ -758,11 +773,11 @@ class EntityDataLoaderImpl implements EntityDataLoader {
// if (currentEntityDef.getFullEntityName().contains("DbForm")) logger.warn("========= DbForm rootValueMap: ${rootValueMap}")
if (edli.dummyFks || edli.useTryInsert) {
EntityValue curValue = currentEntityDef.makeEntityValue()
curValue.setAll(valueMap)
curValue.setAll(valueMap, locale)
valueHandler.handleValue(curValue)
valuesRead++
} else {
valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap)
valueHandler.handlePlainMap(currentEntityDef.getFullEntityName(), valueMap, getLocale())
valuesRead++
}
currentEntityDef = (EntityDefinition) null
Expand Down Expand Up @@ -794,6 +809,12 @@ class EntityDataLoaderImpl implements EntityDataLoader {
}

void setDocumentLocator(Locator locator) { this.locator = locator }

void setLocale(Locale locale) { this.locale = locale }
Locale getLocale() {
if (locale == null) locale = new Locale("en", "US")
return locale
}
}

static class EntityCsvHandler {
Expand All @@ -803,6 +824,8 @@ class EntityDataLoaderImpl implements EntityDataLoader {
protected long valuesRead = 0
protected List<String> messageList = new LinkedList()

protected Locale locale = null

EntityCsvHandler(EntityDataLoaderImpl edli, ValueHandler valueHandler) {
this.edli = edli
this.valueHandler = valueHandler
Expand Down Expand Up @@ -853,6 +876,14 @@ class EntityDataLoaderImpl implements EntityDataLoader {
return false
}
}

if (firstLineRecord.size() > 2) {
// third field is locale
String localeStr = firstLineRecord.get(2)
int usIdx = localeStr.indexOf("_")
locale = usIdx < 0 ? new Locale(localeStr) :
new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase())
}
}

Map<String, Integer> headerMap = [:]
Expand Down Expand Up @@ -880,9 +911,9 @@ class EntityDataLoaderImpl implements EntityDataLoader {
valuesRead++
} else {
EntityValueImpl currentEntityValue = (EntityValueImpl) edli.efi.makeValue(entityName)
if (edli.defaultValues) currentEntityValue.setFields(edli.defaultValues, true, null, null)
if (edli.defaultValues) currentEntityValue.setFields(edli.defaultValues, true, null, null, locale)
for (Map.Entry<String, Integer> header in headerMap)
currentEntityValue.setString(header.key, record.get(header.value))
currentEntityValue.setString(header.key, record.get(header.value), locale)

if (!currentEntityValue.containsPrimaryKey()) {
if (currentEntityValue.getEntityDefinition().getPkFieldNames().size() == 1) {
Expand All @@ -899,6 +930,13 @@ class EntityDataLoaderImpl implements EntityDataLoader {
}
return true
}

void setLocale(Locale locale) { this.locale = locale }
Locale getLocale() {
if (locale == null) locale = new Locale("en", "US")
return locale
}

}

static class EntityJsonHandler {
Expand All @@ -908,6 +946,8 @@ class EntityDataLoaderImpl implements EntityDataLoader {
protected long valuesRead = 0
protected List<String> messageList = new LinkedList()

protected Locale locale = null

EntityJsonHandler(EntityDataLoaderImpl edli, ValueHandler valueHandler) {
this.edli = edli
this.valueHandler = valueHandler
Expand All @@ -929,18 +969,21 @@ class EntityDataLoaderImpl implements EntityDataLoader {
}

String type = null
String localeStr = null
List valueList
if (jsonObj instanceof Map) {
Map jsonMap = (Map) jsonObj
type = jsonMap.get("_dataType")
localeStr = jsonMap.get("_locale")
valueList = [jsonObj]
} else if (jsonObj instanceof List) {
valueList = (List) jsonObj
Object firstValue = valueList?.get(0)
if (firstValue instanceof Map) {
Map firstValMap = (Map) firstValue
if (firstValMap.get("_dataType")) {
if (firstValMap.get("_dataType") || firstValMap.get("_locale")) {
type = firstValMap.get("_dataType")
localeStr = firstValMap.get("_locale")
valueList.remove((int) 0I)
}
}
Expand All @@ -953,6 +996,12 @@ class EntityDataLoaderImpl implements EntityDataLoader {
return false
}

if (localeStr != null) {
int usIdx = localeStr.indexOf("_")
locale = usIdx < 0 ? new Locale(localeStr) :
new Locale(localeStr.substring(0, usIdx), localeStr.substring(usIdx+1).toUpperCase())
}

for (Object valueObj in valueList) {
if (!(valueObj instanceof Map)) {
logger.warn("Found non-Map object in JSON import, skipping: ${valueObj}")
Expand All @@ -964,6 +1013,13 @@ class EntityDataLoaderImpl implements EntityDataLoader {
value.putAll((Map) valueObj)

String entityName = value."_entity"
String valueLocaleStr = value."_locale"
Locale valueLocale = null
if (valueLocaleStr) {
int usIdx = valueLocaleStr.indexOf("_")
valueLocale = usIdx < 0 ? new Locale(valueLocaleStr) :
new Locale(valueLocaleStr.substring(0, usIdx), valueLocaleStr.substring(usIdx+1).toUpperCase())
}
boolean isService
if (edli.efi.isEntityDefined(entityName)) {
isService = false
Expand All @@ -978,13 +1034,20 @@ class EntityDataLoaderImpl implements EntityDataLoader {
valueHandler.handleService(currentScs)
valuesRead++
} else {
valueHandler.handlePlainMap(entityName, value)
valueHandler.handlePlainMap(entityName, value, valueLocale?:getLocale())
// TODO: make this more complete, like counting nested Maps?
valuesRead++
}
}

return true
}

void setLocale(Locale locale) { this.locale = locale }
Locale getLocale() {
if (locale == null) locale = new Locale("en", "US")
return locale
}

}
}
2 changes: 1 addition & 1 deletion framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ class EntityDataWriterImpl implements EntityDataWriter {
writer.println("[")
} else {
writer.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
writer.println("<entity-facade-xml>")
writer.println("<entity-facade-xml locale=\"en_US\">")
}
}
private void endFile(Writer writer) {
Expand Down
5 changes: 3 additions & 2 deletions framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -952,11 +952,12 @@ class EntityDefinition {
return mePkFieldToAliasNameMap
}

Object convertFieldString(String name, String value, ExecutionContextImpl eci) {
Object convertFieldString(String name, String value, ExecutionContextImpl eci) { return convertFieldString(name, value, eci, null) }
Object convertFieldString(String name, String value, ExecutionContextImpl eci, Locale locale) {
if (value == null) return null
FieldInfo fieldInfo = getFieldInfo(name)
if (fieldInfo == null) throw new EntityException("Invalid field name ${name} for entity ${fullEntityName}")
return fieldInfo.convertFromString(value, eci.l10nFacade)
return fieldInfo.convertFromString(value, eci.l10nFacade, locale)
}

static String getFieldStringForFile(FieldInfo fieldInfo, Object value) {
Expand Down
Loading