perf: cleanup unused attribute in themes and langs

.gitignore

@@ -10,5 +10,6 @@ logs
 node_modules
 temp
 packages/shikiji/src/assets/langs
+packages/shikiji/src/assets/themes
 packages/shikiji/src/assets/*.json
 cache

packages/shikiji-core/src/types.ts

@@ -198,6 +198,9 @@ export interface LanguageRegistration extends RawGrammar {
   balancedBracketSelectors?: string[]
   unbalancedBracketSelectors?: string[]
 
+  foldingStopMarker?: string
+  foldingStartMarker?: string
+
   /**
    * Inject this language to other scopes.
    * Same as `injectTo` in VSCode's `contributes.grammars`.
@@ -354,40 +357,48 @@ export interface ThemeRegistrationRaw extends RawTheme {}
 
 export interface ThemeRegistration extends ThemeRegistrationRaw {
   /**
-   * @description theme name
+   * Theme name
    */
   name: string
 
   /**
-   * @description light/dark theme
+   * Display name
    */
-  type: 'light' | 'dark' | 'css'
+  displayName?: string
 
   /**
-   * @description tokenColors of the theme file
+   * Light/dark theme
+   */
+  type: 'light' | 'dark'
+
+  /**
+   * TokenColors of the theme file
    */
   settings: IRawThemeSetting[]
 
   /**
-   * @description text default foreground color
+   * Text default foreground color
    */
   fg: string
 
   /**
-   * @description text default background color
+   * Text default background color
    */
   bg: string
 
   /**
-   * @description relative path of included theme
+   * Relative path of included theme
    */
   include?: string
 
   /**
-   *
-   * @description color map of the theme file
+   * Color map of the theme file
    */
   colors?: Record<string, string>
+
+  tokenColors?: any[]
+  semanticHighlighting?: boolean
+  semanticTokenColors?: any[]
 }
 
 export interface ShikijiTransformerContextMeta {}

packages/shikiji/scripts/prepare.ts

@@ -4,5 +4,7 @@ import { prepareTheme } from './prepare/themes'
 
 await fs.ensureDir('./src/assets/langs')
 await fs.emptyDir('./src/assets/langs')
+await fs.ensureDir('./src/assets/themes')
+await fs.emptyDir('./src/assets/themes')
 await prepareLangs()
 await prepareTheme()

packages/shikiji/scripts/prepare/injections.ts

@@ -3,6 +3,7 @@
 import fs from 'fs-extra'
 import { fetch } from 'ofetch'
 import { COMMENT_HEAD } from './constants'
+import { cleanupLanguageReg } from './utils'
 
 interface Injection {
   name: string
@@ -24,7 +25,9 @@ export async function prepareInjections() {
       `${COMMENT_HEAD}
 import type { LanguageRegistration } from 'shikiji-core'
 
-export default ${JSON.stringify(injection.contents, null, 2)} as unknown as LanguageRegistration[]
+export default [
+  ${injection.contents.map(i => `Object.freeze(${JSON.stringify(i, null, 2)})`).join(',\n')}
+] as unknown as LanguageRegistration[]
 `,
       'utf-8',
     )
@@ -41,11 +44,11 @@ export async function prepareVueInjections(): Promise<Injection> {
     .filter(i => i.injectTo)
     .map(async (i) => {
       const content = await fetchJson(`${new URL(i.path, base).href}?raw=true`)
-      return {
+      return cleanupLanguageReg({
         ...content,
         name: i.language,
         injectTo: i.injectTo,
-      }
+      })
     }),
   )
 

packages/shikiji/scripts/prepare/langs.ts

@@ -3,6 +3,7 @@ import { BUNDLED_LANGUAGES } from 'shiki'
 import fg from 'fast-glob'
 import type { LanguageRegistration } from 'shikiji-core'
 import { COMMENT_HEAD } from './constants'
+import { cleanupLanguageReg } from './utils'
 import { prepareInjections } from './injections'
 
 export async function prepareLangs() {
@@ -24,7 +25,7 @@ export async function prepareLangs() {
       continue
     }
 
-    const json: LanguageRegistration = {
+    const json: LanguageRegistration = cleanupLanguageReg({
       ...content,
       name: content.name || lang.id,
       scopeName: content.scopeName || lang.scopeName,
@@ -33,7 +34,7 @@ export async function prepareLangs() {
       embeddedLangs: lang.embeddedLangs,
       balancedBracketSelectors: lang.balancedBracketSelectors,
       unbalancedBracketSelectors: lang.unbalancedBracketSelectors,
-    }
+    })
 
     // F# and Markdown has circular dependency
     if (lang.id === 'fsharp' && json.embeddedLangs)
@@ -49,7 +50,7 @@ import type { LanguageRegistration } from 'shikiji-core'
 
 ${deps.map(i => `import ${i.replace(/[^\w]/g, '_')} from './${i}'`).join('\n')}
 
-const lang = Object.freeze(${JSON.stringify(json)}) as unknown as LanguageRegistration
+const lang = Object.freeze(${JSON.stringify(json, null, 2)}) as unknown as LanguageRegistration
 
 export default [
 ${[
@@ -102,16 +103,6 @@ export const bundledLanguages = {
 `,
   'utf-8',
     )
-
-    await fs.writeJSON(
-    `src/assets/${fileName}.json`,
-    BUNDLED_LANGUAGES.map(i => ({
-      id: i.id,
-      name: i.displayName,
-      aliases: i.aliases,
-    })),
-    { spaces: 2 },
-    )
   }
 
   await writeLanguageBundleIndex('langs', BUNDLED_LANGUAGES.map(i => i.id))

packages/shikiji/scripts/prepare/themes.ts

@@ -1,23 +1,37 @@
 import fs from 'fs-extra'
 import { BUNDLED_THEMES } from 'shiki'
 import { COMMENT_HEAD } from './constants'
+import { cleanupThemeReg } from './utils'
 
 const allThemes = BUNDLED_THEMES
   .sort()
   .filter(i => i !== 'css-variables')
 
 export async function prepareTheme() {
-  const themes = allThemes
-    .map((id) => {
-      const theme = fs.readJSONSync(`./node_modules/shiki/themes/${id}.json`)
+  const themes = await Promise.all(allThemes
+    .map(async (id) => {
+      const theme = cleanupThemeReg(await fs.readJSON(`./node_modules/shiki/themes/${id}.json`))
+
+      theme.displayName = guessThemeName(id, theme)
+      theme.type = guessThemeType(id, theme)
+
+      await fs.writeFile(
+        `./src/assets/themes/${id}.ts`,
+        `${COMMENT_HEAD}
+import type { ThemeRegistration } from 'shikiji-core'
+
+export default Object.freeze(${JSON.stringify(theme, null, 2)}) as unknown as ThemeRegistration
+`,
+        'utf-8',
+      )
 
       return {
         id,
-        name: guessThemeName(id, theme),
-        type: guessThemeType(id, theme),
-        import: `__(() => import('shiki/themes/${id}.json')) as unknown as DynamicThemeReg__`,
+        displayName: theme.displayName,
+        type: theme.type,
+        import: `__(() => import('./themes/${id}')) as unknown as DynamicThemeReg__`,
       }
-    })
+    }))
   await fs.writeFile(
     'src/assets/themes.ts',
 `${COMMENT_HEAD}
@@ -27,7 +41,7 @@ type DynamicThemeReg = () => Promise<{ default: ThemeRegistrationRaw }>
 
 export interface BundledThemeInfo {
   id: string
-  name: string
+  displayName: string
   type: 'light' | 'dark'
   import: DynamicThemeReg
 }
@@ -40,12 +54,6 @@ export const bundledThemes = Object.fromEntries(bundledThemesInfo.map(i => [i.id
 `,
 'utf-8',
   )
-  await fs.writeJSON(
-    'src/assets/themes.json',
-    allThemes
-      .map(i => ({ id: i })),
-    { spaces: 2 },
-  )
 }
 
 function isLightColor(hex: string) {
@@ -54,6 +62,8 @@ function isLightColor(hex: string) {
 }
 
 function guessThemeType(id: string, theme: any) {
+  if ('type' in theme)
+    return theme.type
   let color
   if (id.includes('dark') || id.includes('dimmed') || id.includes('black'))
     color = 'dark'

packages/shikiji/scripts/prepare/utils.ts

@@ -0,0 +1,64 @@
+import type { LanguageRegistration, ThemeRegistration } from 'shikiji-core'
+
+export function objectPick<T extends Record<any, any>>(
+  obj: T,
+  keys: (keyof T)[],
+  onRemoval?: (obj: T, key: string, value: any) => void,
+): T {
+  return Object.fromEntries(
+    Array.from(
+      Object.entries(obj)
+        .filter((i) => {
+          if (keys.includes(i[0]))
+            return true
+          onRemoval?.(obj, ...i)
+          return false
+        }),
+    ),
+  ) as T
+}
+
+export function cleanupLanguageReg(lang: LanguageRegistration) {
+  return objectPick(
+    lang,
+    [
+      'aliases',
+      'balancedBracketSelectors',
+      'displayName',
+      'embeddedLangs',
+      'fileTypes',
+      'firstLineMatch',
+      'injections',
+      'injectionSelector',
+      'injectTo',
+      'name',
+      'patterns',
+      'repository',
+      'scopeName',
+      'foldingStopMarker',
+      'foldingStartMarker',
+      'unbalancedBracketSelectors',
+    ],
+    // (_, key, value) => console.log('lang key removal', key, '|', value),
+  )
+}
+
+export function cleanupThemeReg(theme: ThemeRegistration) {
+  return objectPick(
+    theme,
+    [
+      'name',
+      'type',
+      'colors',
+      'fg',
+      'bg',
+      'include',
+      'settings',
+
+      'tokenColors',
+      'semanticHighlighting',
+      'semanticTokenColors',
+    ],
+    // (_, key, value) => console.log('theme key removal', key, value),
+  )
+}