diff --git a/package.json b/package.json index 5acc05b..875ac29 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "test": "run-s build test:*", "test:unit": "nyc --silent ava", "watch": "run-s clean build:main && run-p \"build:main -- -w\" \"test:unit -- --watch\"", + "watch-tsc": "tsc --watch", "cov": "run-s build test:unit cov:html && opn coverage/index.html", "cov:html": "nyc report --reporter=html", "cov:send": "nyc report --reporteyarnr=lcov > coverage.lcov && codecov", diff --git a/src/lib/dataInference.ts b/src/lib/dataInference.ts index 15322c2..e755dd0 100644 --- a/src/lib/dataInference.ts +++ b/src/lib/dataInference.ts @@ -1,5 +1,5 @@ import dayjs from 'dayjs'; -import { get, isFinite, isNaN, isNull, isUndefined, keys } from 'lodash'; +import { get, isFinite, isNaN, isNull, isUndefined, keys, isDate, isString } from 'lodash'; import { CategoricalDatum, ContinuousDatum, @@ -10,9 +10,40 @@ import { } from '../types/types'; import { fromPairs } from './parseObjects'; import { getAllKeys } from './stats'; - -import CustomParseFormat from 'dayjs/plugin/customParseFormat'; // load on demand -dayjs.extend(CustomParseFormat); // use plugin +import customParseFormat from 'dayjs/plugin/customParseFormat'; +dayjs.extend(customParseFormat); + +// references: +// https://day.js.org/docs/en/parse/string-format +// https://day.js.org/docs/en/display/format +export const DATE_FORMATS = [ + 'YYYY-MM-DD', + 'YYYY-MM-D', + 'YYYY-M-DD', + 'YYYY-M-D', + + 'YYYY-MM-DD HH:mm', + 'YYYY-MM-DD HH:mm[Z]', + 'YYYY-MM-DD[T]HH:mm', + 'YYYY-MM-DD[T]HH:mm[Z]', + + 'YYYY-MM-DD HH:mm:ss', + 'YYYY-MM-DD HH:mm:ss[Z]', + 'YYYY-MM-DD[T]HH:mm:ss', + 'YYYY-MM-DD[T]HH:mm:ss[Z]', + + 'YYYY-MM-DD HH:mm:ss.SSS', + 'YYYY-MM-DD HH:mm:ss.SSS[Z]', + 'YYYY-MM-DD[T]HH:mm:ss.SSS', + 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]', // ISO8601 + + 'YYYY-MM-DD HH:mm:ss A', + 'YYYY-MM-DD HH:mm:ss a', + + // these two formats don't work + // 'YYYY-MM-DD HH:mm:ssZ', // 2013-02-08 09:00:00+07:00 + // 'YYYY-MM-DD HH:mm:ssZZ', // 2013-02-08 09:00:00-0700 +]; export type GenericDatumValue = number | string | boolean | null; @@ -62,27 +93,28 @@ export function detectValue( if (!value || isNull(value) || typeof value === 'boolean') { return 'unknown'; } - if (inferIsNumber(value)) { return 'continuous'; - } else if (typeof value === 'string' && isFormatDateValid(value, parser)) { + } else if (isDate(value) || (isString(value) && isFormatDateValid(value, parser))) { return 'date'; } else { return 'categorical'; } } +function isValidDate(dateString: string, formats: string[]): boolean { + const strictMode = true + // @ts-ignore: necessary because of strictMode parameter: `Argument of type 'true' is not assignable to parameter of type 'string | undefined'.` + return formats.some(format => dayjs(dateString, format, strictMode).isValid()) +} + export function isFormatDateValid( value: string, parser?: ParserFunction ): boolean { if(!inferIfStringIsNumber(value[0])) return false - // this will match yyyy-mm-dd and also yyyy-m-d - const regDate = /^([1-9][0-9]{3})\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/; - // TODO: add other regex to accept also other date formats - - const isFormatDateValid = regDate.test(value.toString()); + const isFormatDateValid = isValidDate(value, DATE_FORMATS); // NOTE: we assume that if the user has written a parser, then the dates are in the correct format return isFormatDateValid || !isUndefined(parser); diff --git a/src/lib/test/dataInference.spec.ts b/src/lib/test/dataInference.spec.ts index 9960522..2905fdb 100644 --- a/src/lib/test/dataInference.spec.ts +++ b/src/lib/test/dataInference.spec.ts @@ -1,5 +1,7 @@ import test from 'ava'; import dayjs from 'dayjs'; +import customParseFormat from 'dayjs/plugin/customParseFormat'; +dayjs.extend(customParseFormat); import { autoInferenceType, detectValue, @@ -67,19 +69,47 @@ test('isDateValid', t => { const defaultParser = (d: string) => dayjs(d, 'YYYY-MM-DD').unix(); t.is(isFormatDateValid('1900-10-23'), true); - t.is(isFormatDateValid('2002-5-5'), true); - t.is(isFormatDateValid('2008-09-31'), true); - t.is(isFormatDateValid('1600-12-25'), true); t.is(isFormatDateValid('1942-11-1'), true); - t.is(isFormatDateValid('2000-10-10'), true); - t.is(isFormatDateValid('2018-02-29'), true); // this day doesn't exist but we check only the date format - t.is(isFormatDateValid('2017-02-30'), true); // this day doesn't exist but we check only the date format - t.is(isFormatDateValid('2020-04-31'), true); // this day doesn't exist but we check only the date format + t.is(isFormatDateValid('2002-5-15'), true); + t.is(isFormatDateValid('2000-01-10'), true); + + t.is(isFormatDateValid('2020-05-01 09:35'), true); + t.is(isFormatDateValid('2020-05-01 09:35Z'), true); + t.is(isFormatDateValid('2020-05-01T09:35'), true); + t.is(isFormatDateValid('2020-05-01T09:35Z'), true); + + t.is(isFormatDateValid('2020-05-01 09:35:20'), true); + t.is(isFormatDateValid('2019-01-15 13:12:29'), true); + t.is(isFormatDateValid('2020-05-01 09:35:20Z'), true); + t.is(isFormatDateValid('2020-05-01T09:35:20'), true); + t.is(isFormatDateValid('2020-05-01T09:35:20Z'), true); + + t.is(isFormatDateValid('2020-05-01 09:35:20.000'), true); + t.is(isFormatDateValid('2020-05-01 09:35:20.000Z'), true); + t.is(isFormatDateValid('2020-05-01T09:35:20.000'), true); + t.is(isFormatDateValid('2020-05-01T09:35:20.000Z'), true); + t.is(isFormatDateValid('2020-05-13T08:24:45.701Z'), true); + t.is(isFormatDateValid(new Date().toISOString()), true); + + t.is(isFormatDateValid('2016-01-01 11:31:23 AM'), true); + t.is(isFormatDateValid('2016-01-01 11:31:23 am'), true); + t.is(isFormatDateValid('2016-01-01 23:31:23 pm'), true); + t.is(isFormatDateValid('17-02-2019', rightParser), true); t.is(isFormatDateValid('17-02-2019', wrongParser), true); // this shouldn't be right but we assume that if the user has written a parser, then the dates are in the correct format t.is(isFormatDateValid('17-02-2019', defaultParser), true); // this shouldn't be right but we assume that if the user has written a parser, then the dates are in the correct format - t.is(isFormatDateValid('2019-01-15 13:12:29.0'), false); + // t.is(isFormatDateValid('2013-02-08 09:00:00+01:00'), true); + // t.is(isFormatDateValid('2013-02-08 09:00:00+01:30'), true); + // t.is(isFormatDateValid('2013-02-08 09:00:00+0100'), true); + // t.is(isFormatDateValid('2013-02-08 09:00:00-01:00'), true); + // t.is(isFormatDateValid('2013-02-08 09:00:00-0100'), true); + + t.is(isFormatDateValid('cat-1'), false); + t.is(isFormatDateValid('2017-02-30'), false); + t.is(isFormatDateValid('2020-04-31'), false); + t.is(isFormatDateValid('2018-02-29'), false); + t.is(isFormatDateValid('2008-09-31'), false); t.is(isFormatDateValid('17-02-2019'), false); t.is(isFormatDateValid('0000-01-01'), false); t.is(isFormatDateValid('0100-10-23'), false); @@ -87,7 +117,20 @@ test('isDateValid', t => { t.is(isFormatDateValid('1942-11-0'), false); t.is(isFormatDateValid('1942-00-25'), false); t.is(isFormatDateValid('2000-10-00'), false); - t.is(isFormatDateValid('cat-1'), false); + t.is(isFormatDateValid('2019-01-15 24:00:00'), false); + t.is(isFormatDateValid('2019-01-15 23:60:00'), false); + t.is(isFormatDateValid('2019-01-15 23:59:60'), false); + t.is(isFormatDateValid('2019-01-15 13:12:29.0'), false); + t.is(isFormatDateValid('2020-05-01 01:01:01.12'), false); + t.is(isFormatDateValid('2016-01-01 11:31:23 PM'), false); + t.is(isFormatDateValid('2020-05-01 00'), false); + t.is(isFormatDateValid('2020-02-31'), false); + t.is(isFormatDateValid('2016/01/01'), false); + t.is(isFormatDateValid('2020-05-01 01:60:00'), false); + t.is(isFormatDateValid('2020-05-01 60'), false); + t.is(isFormatDateValid('2020-05-01 '), false); + t.is(isFormatDateValid(new Date().toString()), false); + t.is(isFormatDateValid('Wed May 13 2020 10:25:23 GMT+0200 (Central European Summer Time)'), false); }); // ---------------------------------------------------------- diff --git a/tslint.json b/tslint.json index ad9677a..6452cf4 100644 --- a/tslint.json +++ b/tslint.json @@ -25,12 +25,14 @@ "no-class": true, "no-mixed-interface": false, "no-expression-statement": [ - true, + false, { "ignore-prefix": ["console.", "process.exit"] } ], "no-if-statement": false, /* end tslint-immutable rules */ - "ordered-imports": false + "ordered-imports": false, + "curly": false, + "no-shadowed-variable": false } } diff --git a/yarn.lock b/yarn.lock index 5ad7d5d..669dd58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1603,9 +1603,9 @@ dateformat@^3.0.0: integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== dayjs@^1.8.13: - version "1.8.13" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.8.13.tgz#51b5cdad23ba508bcea939a853b492fefb7fdc47" - integrity sha512-JZ01l/PMU8OqwuUs2mOQ/CTekMtoXOUSylfjqjgDzbhRSxpFIrPnHn8Y8a0lfocNgAdBNZb8y0/gbzJ2riQ4WQ== + version "1.8.27" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.8.27.tgz#a8ae63ee990af28c05c430f0e160ae835a0fbbf8" + integrity sha512-Jpa2acjWIeOkg8KURUHICk0EqnEFSSF5eMEscsOgyJ92ZukXwmpmRkPSUka7KHSfbj5eKH30ieosYip+ky9emQ== debug@^2.1.2, debug@^2.2.0, debug@^2.3.3: version "2.6.9"