Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test(date): Add Intl-Based Tests for Date Definitions to Ensure Completeness of Weekdays and Months #2912

Open
wants to merge 27 commits into
base: next
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
3097485
test(date): create Intl based tests for date definitions
sossost May 18, 2024
4d3055f
test: create Intl based test for date definition
sossost May 18, 2024
fe8ce70
test : add condition filtering valid Itnl locales
sossost May 19, 2024
fa73f0a
chore : add generate:date command
sossost May 19, 2024
efb206f
feat: auto generate/update date files
sossost May 19, 2024
49f08f6
fix : Add condition that exception case 'vd'
sossost May 19, 2024
1e67eb6
Merge branch 'next' into test/create-Intl-based-tests-for-date-defini…
ST-DDT May 19, 2024
9a07a19
test : restore existing test and remove month, weekday test about cou…
sossost May 19, 2024
16f9994
Merge branch 'test/create-Intl-based-tests-for-date-definitions' of h…
sossost May 19, 2024
5005fcb
test : Refactoring to locale data instead of using a new Faker instance
sossost May 19, 2024
80ee22a
refactor : integrate function that update month, weekDay locale data …
sossost May 19, 2024
aff49d5
fix : rollback generate scripts
sossost May 19, 2024
7263f0d
docs : add comments to explain why 'dv' locale is excluded
sossost May 19, 2024
3488e11
refactor : refactor generate-locales.ts reflecting code review
ynnsuis May 20, 2024
cb80f8d
chore: review and reduce diff
ST-DDT May 20, 2024
8f92366
chore: generate files
ST-DDT May 20, 2024
2a84b19
fix: Replace underscores with dashes in Intl.DateTimeFormat locale
sossost May 23, 2024
9ccf7dc
feat: Skip generating locale date files if identical to parent
sossost May 23, 2024
c6ffe2a
Merge branch 'faker-js:next' into test/create-Intl-based-tests-for-da…
sossost May 24, 2024
7ab5dd2
refactor : Move invalidLocale function inside generateDateModule and …
sossost May 24, 2024
18d9975
Merge branch 'next' of https://github.com/sossost/faker into test/cre…
sossost May 24, 2024
08f49ef
Merge branch 'test/create-Intl-based-tests-for-date-definitions' of h…
sossost May 24, 2024
93f6dbb
Merge branch 'next' into test/create-Intl-based-tests-for-date-defini…
ST-DDT May 27, 2024
3e9d64e
Merge branch 'next' into pr/sossost/2912
ST-DDT Jun 18, 2024
a09f70d
chore: generate files only when needed
ST-DDT Jun 18, 2024
aeb62d6
chore: generate missing files
ST-DDT Jun 18, 2024
78c7183
fix: parentIntl detection
ST-DDT Jun 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 194 additions & 26 deletions scripts/generate-locales.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
#!/usr/bin/env node

/* eslint-disable no-restricted-globals */

/**
* This file contains a script that can be used to update the following files:
*
* - `src/locale/<locale>.ts`
* - `src/locales/<locale>/date/month.ts`
* - `src/locales/<locale>/date/weekday.ts`
* - `src/locales/<locale>/index.ts`
* - `src/locales/<locale>/<module...>/index.ts`
* - `src/docs/guide/localization.md`
Expand All @@ -15,10 +19,21 @@
* Run this script using `pnpm run generate:locales`
*/
import { constants } from 'node:fs';
import { access, readFile, readdir, stat, writeFile } from 'node:fs/promises';
import {
access,
mkdir,
readFile,
readdir,
stat,
writeFile,
} from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import type { LocaleDefinition, MetadataDefinition } from '../src/definitions';
import type {
DateEntryDefinition,
LocaleDefinition,
MetadataDefinition,
} from '../src/definitions';
import { keys } from '../src/internal/keys';
import { formatMarkdown, formatTypescript } from './apidocs/utils/format';

Expand Down Expand Up @@ -111,6 +126,41 @@ function escapeField(parent: string, module: string): string {
return module;
}

function isValidLocale(locale: string): boolean {
ST-DDT marked this conversation as resolved.
Show resolved Hide resolved
// 'dv' (Dhivehi) locale is excluded because it may not be fully supported
if (locale === 'dv') return false;
try {
new Intl.DateTimeFormat(locale);
ST-DDT marked this conversation as resolved.
Show resolved Hide resolved
return true;
} catch {
return false;
}
}

function normalizeDataRecursive<T>(localeData: T): T {
if (typeof localeData !== 'object' || localeData === null) {
// we can only traverse object-like structs
return localeData;
}

if (Array.isArray(localeData)) {
return (
[...new Set(localeData)]
// limit entries to 1k
.slice(0, 1000)
// sort entries alphabetically
.sort() as T
);
}

const result = {} as T;
for (const key of keys(localeData)) {
result[key] = normalizeDataRecursive(localeData[key]);
}

return result;
}

async function generateLocaleFile(locale: string): Promise<void> {
const parts = locale.split('_');
const locales = [locale];
Expand Down Expand Up @@ -286,30 +336,6 @@ async function updateLocaleFileHook(
* @param definitionKey The definition key of the current file (ex. 'location').
*/
async function normalizeLocaleFile(filePath: string, definitionKey: string) {
function normalizeDataRecursive<T>(localeData: T): T {
if (typeof localeData !== 'object' || localeData === null) {
// we can only traverse object-like structs
return localeData;
}

if (Array.isArray(localeData)) {
return (
[...new Set(localeData)]
// limit entries to 1k
.slice(0, 1000)
// sort entries alphabetically
.sort() as T
);
}

const result = {} as T;
for (const key of keys(localeData)) {
result[key] = normalizeDataRecursive(localeData[key]);
}

return result;
}

const legacyDefinitions = ['app', 'cell_phone', 'team'];
const definitionsToSkip = [
'finance',
Expand Down Expand Up @@ -368,6 +394,145 @@ async function normalizeLocaleFile(filePath: string, definitionKey: string) {
return writeFile(filePath, await formatTypescript(newContent));
}

// `src/locales/<locale>/date/*
async function generateDateModule(locale: string) {
if (!isValidLocale(locale)) return;

const pathDate = resolve(pathLocales, locale, 'date');

// Function to check if it has the same data as the parent locale
async function isParentLocaleSame(
ST-DDT marked this conversation as resolved.
Show resolved Hide resolved
locale: string,
fileName: string
): Promise<boolean> {
const parentLocale = locale.split('_').slice(0, -1).join('_');
if (!parentLocale) return false;

const parentFilePath = resolve(pathLocales, parentLocale, 'date', fileName);
const currentFilePath = resolve(pathDate, fileName);

try {
const [parentFileContent, currentFileContent] = await Promise.all([
readFile(parentFilePath, 'utf8'),
readFile(currentFilePath, 'utf8'),
]);

return parentFileContent === currentFileContent;
} catch {
return false;
}
}

// `src/locales/<locale>/date/weekday.ts`
async function generateWeekdayFile(): Promise<void> {
const weekdayPath = resolve(pathDate, 'weekday.ts');
let storedWeekdays: Partial<DateEntryDefinition> = {};

// Skip creation if it has the same data as the parent locale
if (await isParentLocaleSame(locale, 'weekday.ts')) {
console.log(
`Skipping weekday file generation for locale ${locale} as it is identical to its parent.`
);
return;
}

// Import the current weekday values
try {
const imported = await import(`file:${weekdayPath}`);
storedWeekdays = imported.default;
} catch {
console.log(`Creating weekday.ts file for locale ${locale}.`);
}

// Generate correct weekday values
const wide: string[] = [];
const abbr: string[] = [];
const intlWeekdayLong = new Intl.DateTimeFormat(
locale.replaceAll('_', '-'),
{
weekday: 'long',
}
);
const intlWeekdayShort = new Intl.DateTimeFormat(
locale.replaceAll('_', '-'),
{
weekday: 'short',
}
);
for (let i = 0; i < 7; i++) {
const date: Date = new Date(2020, 0, i + 4);
// January 4-10, 2020 are Sunday to Saturday
wide.push(intlWeekdayLong.format(date));
abbr.push(intlWeekdayShort.format(date));
}

storedWeekdays.wide = wide;
storedWeekdays.abbr = abbr;

// Write updated values back to the file
const normalizedWeekdays = normalizeDataRecursive(storedWeekdays);
const updatedContent = `
${autoGeneratedCommentHeader}
export default ${JSON.stringify(normalizedWeekdays, null, 2)};
`;
return writeFile(weekdayPath, await formatTypescript(updatedContent));
}

// `src/locales/<locale>/date/month.ts`
async function generateMonthFile(): Promise<void> {
const monthPath = resolve(pathDate, 'month.ts');
let storedMonths: Partial<DateEntryDefinition> = {};

// Skip creation if it has the same data as the parent locale
if (await isParentLocaleSame(locale, 'month.ts')) {
console.log(
`Skipping month file generation for locale ${locale} as it is identical to its parent.`
);
return;
}

// Import the current month values
try {
const imported = await import(`file:${monthPath}`);
storedMonths = imported.default;
} catch {
console.log(`Creating month.ts file for locale ${locale}.`);
}

// Generate correct month values
const wide: string[] = [];
const abbr: string[] = [];
const intlMonthLong = new Intl.DateTimeFormat(locale.replaceAll('_', '-'), {
month: 'long',
});
const intlMonthShort = new Intl.DateTimeFormat(
locale.replaceAll('_', '-'),
{ month: 'short' }
);
for (let i = 0; i < 12; i++) {
const date: Date = new Date(2020, i, 1);
wide.push(intlMonthLong.format(date));
abbr.push(intlMonthShort.format(date));
}

storedMonths.wide = wide;
storedMonths.abbr = abbr;

// Write updated values back to the file
const normalizedMonths = normalizeDataRecursive(storedMonths);
const updatedContent = `
${autoGeneratedCommentHeader}
export default ${JSON.stringify(normalizedMonths, null, 2)};
`;
return writeFile(monthPath, await formatTypescript(updatedContent));
}

await mkdir(pathDate, { recursive: true });
await generateWeekdayFile();
await generateMonthFile();
return generateRecursiveModuleIndexes(pathDate, locale, 'DateDefinition', 2);
}

// Start of actual logic

const locales = await readdir(pathLocales);
Expand Down Expand Up @@ -411,6 +576,9 @@ for (const locale of locales) {
localesIndexImports += `import { default as ${locale} } from './${locale}';\n`;
localizationLocales += `| \`${locale}\` | ${localeTitle} | \`${localizedFaker}\` |\n`;

// We first need to generate the date module
await generateDateModule(locale);

promises.push(
// src/locale/<locale>.ts
// eslint-disable-next-line unicorn/prefer-top-level-await -- Disabled for performance
Expand Down
30 changes: 17 additions & 13 deletions src/locales/ar/date/month.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
/*
* This file is automatically generated.
* Run 'pnpm run generate:locales' to update.
*/
export default {
wide: [
'آب',
'آذَار',
'أَيَّار',
'أَيْلُول',
'تَمُّوز',
'تِشْرِين ٱلثَّانِي',
'تِشْرِين ٱلْأَوَّل',
'حَزِيرَان',
'شُبَاط',
'كَانُون ٱلثَّانِي',
'كَانُون ٱلْأَوَّل',
'نَيْسَان',
'أبريل',
'أغسطس',
'أكتوبر',
'ديسمبر',
'سبتمبر',
'فبراير',
'مارس',
'مايو',
'نوفمبر',
'يناير',
'يوليو',
'يونيو',
],
abbr: [
'أبريل',
'أغسطس',
'أكتوبر',
'إبريل',
'ديسمبر',
'سبتمبر',
'فبراير',
Expand Down
18 changes: 15 additions & 3 deletions src/locales/ar/date/weekday.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,21 @@
/*
* This file is automatically generated.
* Run 'pnpm run generate:locales' to update.
*/
export default {
abbr: null,
abbr: [
'الأحد',
'الأربعاء',
'الاثنين',
'الثلاثاء',
'الجمعة',
'الخميس',
'السبت',
],
wide: [
'الأحَد',
'الأحد',
'الأربعاء',
'الإثنين',
'الاثنين',
'الثلاثاء',
'الجمعة',
'الخميس',
Expand Down
28 changes: 16 additions & 12 deletions src/locales/az/date/month.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
/*
* This file is automatically generated.
* Run 'pnpm run generate:locales' to update.
*/
export default {
wide: [
'aprel',
Expand Down Expand Up @@ -28,18 +32,18 @@ export default {
'января',
],
abbr: [
'авг.',
'апр.',
'дек.',
'июль',
'июнь',
'май',
'март',
'нояб.',
'окт.',
'сент.',
'февр.',
'янв.',
'apr',
'avq',
'dek',
'fev',
'iyl',
'iyn',
'mar',
'may',
'noy',
'okt',
'sen',
'yan',
],
abbr_context: [
'авг.',
Expand Down
Loading