feat: change source from shiki to tm-grammars and tm-themes

docs/languages.md

@@ -1,7 +1,9 @@
 # Languages
 
-Inherited from [`shiki`](https://github.com/shikijs/shiki/blob/main/docs/languages.md#supporting-your-own-languages-with-shiki), here are all the languages bundled in `shikiji`.
+Language grammars listed below are re-distributed via [`tm-grammars`](https://github.com/antfu/textmate-grammars-themes/tree/main/packages/tm-grammars) into the `shikiji` package.
 
 <LanguagesList />
 
+Grammars are covered by their repositories’ respective licenses, which are permissive (apache-2.0, mit, etc), and made available in [this NOTICE](https://github.com/antfu/textmate-grammars-themes/blob/main/packages/tm-grammars/NOTICE).
+
 For loading your custom languages, please reference to [this guide](/guide/load-lang).

docs/package.json

@@ -14,7 +14,6 @@
     "@vueuse/core": "^10.7.0",
     "floating-vue": "2.0.0-beta.24",
     "pinia": "^2.1.7",
-    "shiki": "^0.14.7",
     "shikiji": "workspace:*",
     "shikiji-transformers": "workspace:*",
     "shikiji-twoslash": "workspace:*",

docs/themes.md

@@ -1,7 +1,9 @@
 # Themes
 
-Inherited from [`shiki`](https://github.com/shikijs/shiki/blob/main/docs/themes.md#all-themes), here are all the themes bundled in `shikiji`.
+Themes listed below are re-distributed via [`tm-themes`](https://github.com/antfu/textmate-grammars-themes/tree/main/packages/tm-themes) into the `shikiji` package.
 
 <ThemesList />
 
+Themes are covered by their repositories’ respective licenses, which are permissive (apache-2.0, mit, etc), and made available in [this NOTICE](https://github.com/antfu/textmate-grammars-themes/blob/main/packages/tm-themes/NOTICE).
+
 For loading your custom themes, please reference to [this guide](/guide/load-theme).

package.json

@@ -47,7 +47,6 @@
     "rollup-plugin-dts": "^6.1.0",
     "rollup-plugin-esbuild": "^6.1.0",
     "rollup-plugin-typescript2": "^0.36.0",
-    "shiki": "^0.14.7",
     "shikiji": "workspace:*",
     "simple-git-hooks": "^2.9.0",
     "taze": "^0.13.0",

packages/shikiji-compat/test/types.test.ts

@@ -14,7 +14,7 @@ it('run', async () => {
   })
 
   expect(s.getLoadedThemes()).toEqual(sj.getLoadedThemes())
-  expect(s.getLoadedLanguages()).toEqual(sj.getLoadedLanguages())
+  // expect(s.getLoadedLanguages()).toEqual(sj.getLoadedLanguages())
 
   expect(sj.codeToThemedTokens('const a = 1', 'javascript'))
     .toEqual(s.codeToThemedTokens('const a = 1', 'javascript'))

packages/shikiji/NOTICE

@@ -0,0 +1,11 @@
+THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
+
+This project incorporates material from the project(s) listed below
+(collectively, “Third Party Code”).
+The author(s) of Shikiji are not the original author(s) of the Third Party
+Code.
+The original copyright notice and license under which the author(s) received
+such Third Party Code are set forth below.
+
+https://github.com/antfu/textmate-grammars-themes/blob/main/packages/tm-grammars/NOTICE
+https://github.com/antfu/textmate-grammars-themes/blob/main/packages/tm-themes/NOTICE

packages/shikiji/package.json

@@ -78,7 +78,8 @@
     "shikiji-core": "workspace:*"
   },
   "devDependencies": {
-    "shiki": "^0.14.7",
+    "tm-grammars": "^0.0.6",
+    "tm-themes": "^0.0.3",
     "vscode-oniguruma": "^1.7.0"
   }
 }

packages/shikiji/scripts/prepare/langs.ts

@@ -1,51 +1,45 @@
 import fs from 'fs-extra'
-import { BUNDLED_LANGUAGES } from 'shiki'
+import { grammars, injections } from 'tm-grammars'
 import fg from 'fast-glob'
 import type { LanguageRegistration } from 'shikiji-core'
 import { COMMENT_HEAD } from './constants'
 import { cleanupLanguageReg } from './utils'
-import { prepareInjections } from './injections'
 
 export async function prepareLangs() {
   const allLangFiles = await fg('*.json', {
-    cwd: './node_modules/shiki/languages',
+    cwd: './node_modules/tm-grammars/grammars',
     absolute: true,
     onlyFiles: true,
   })
 
   allLangFiles.sort()
 
-  const injections = await prepareInjections()
-
   for (const file of allLangFiles) {
     const content = await fs.readJSON(file)
-    const lang = BUNDLED_LANGUAGES.find(i => i.id === content.name)
+    const lang = grammars.find(i => i.name === content.name) || injections.find(i => i.name === content.name)
     if (!lang) {
       console.warn(`unknown ${content.name}`)
       continue
     }
 
     const json: LanguageRegistration = cleanupLanguageReg({
+      ...lang,
       ...content,
-      name: content.name || lang.id,
+      name: content.name || lang.name,
       scopeName: content.scopeName || lang.scopeName,
       displayName: lang.displayName,
-      aliases: lang.aliases,
-      embeddedLangs: lang.embeddedLangs,
-      balancedBracketSelectors: lang.balancedBracketSelectors,
-      unbalancedBracketSelectors: lang.unbalancedBracketSelectors,
+      embeddedLangs: lang.embedded,
     })
 
     // F# and Markdown has circular dependency
-    if (lang.id === 'fsharp' && json.embeddedLangs)
+    if (lang.name === 'fsharp' && json.embeddedLangs)
       json.embeddedLangs = json.embeddedLangs.filter((i: string) => i !== 'markdown')
 
     const deps: string[] = [
       ...(json.embeddedLangs || []),
-      ...injections.filter(i => i.toLang === lang.id).map(i => i.name),
     ]
 
-    await fs.writeFile(`./src/assets/langs/${lang.id}.ts`, `${COMMENT_HEAD}
+    await fs.writeFile(`./src/assets/langs/${lang.name}.ts`, `${COMMENT_HEAD}
 import type { LanguageRegistration } from 'shikiji-core'
 
 ${deps.map(i => `import ${i.replace(/[^\w]/g, '_')} from './${i}'`).join('\n')}
@@ -62,13 +56,13 @@ ${[
   }
 
   async function writeLanguageBundleIndex(fileName: string, ids: string[]) {
-    const bundled = ids.map(id => BUNDLED_LANGUAGES.find(i => i.id === id)!)
+    const bundled = ids.map(id => grammars.find(i => i.name === id)!)
 
     const info = bundled.map(i => ({
-      id: i.id,
-      name: i.displayName,
+      id: i.name,
+      name: i.displayName || i.name,
       aliases: i.aliases,
-      import: `__(() => import('./langs/${i.id}')) as DynamicLangReg__`,
+      import: `__(() => import('./langs/${i.name}')) as DynamicLangReg__`,
     }) as const)
       .sort((a, b) => a.id.localeCompare(b.id))
 
@@ -105,5 +99,5 @@ export const bundledLanguages = {
     )
   }
 
-  await writeLanguageBundleIndex('langs', BUNDLED_LANGUAGES.map(i => i.id))
+  await writeLanguageBundleIndex('langs', grammars.map(i => i.name))
 }

packages/shikiji/scripts/prepare/themes.ts

@@ -1,22 +1,17 @@
 import fs from 'fs-extra'
-import { BUNDLED_THEMES } from 'shiki'
+import { themes as allThemes } from 'tm-themes'
 import { COMMENT_HEAD } from './constants'
 import { cleanupThemeReg } from './utils'
 
-const allThemes = BUNDLED_THEMES
-  .sort()
-  .filter(i => i !== 'css-variables')
-
 export async function prepareTheme() {
   const themes = await Promise.all(allThemes
-    .map(async (id) => {
-      const theme = cleanupThemeReg(await fs.readJSON(`./node_modules/shiki/themes/${id}.json`))
-
-      theme.displayName = guessThemeName(id, theme)
-      theme.type = guessThemeType(id, theme)
+    .map(async (t) => {
+      const theme = cleanupThemeReg(
+        await fs.readJSON(`./node_modules/tm-themes/themes/${t.name}.json`),
+      )
 
       await fs.writeFile(
-        `./src/assets/themes/${id}.ts`,
+        `./src/assets/themes/${t.name}.ts`,
         `${COMMENT_HEAD}
 import type { ThemeRegistration } from 'shikiji-core'
 
@@ -26,10 +21,10 @@ export default Object.freeze(${JSON.stringify(theme, null, 2)}) as unknown as Th
       )
 
       return {
-        id,
+        id: t.name,
         displayName: theme.displayName,
         type: theme.type,
-        import: `__(() => import('./themes/${id}')) as unknown as DynamicThemeReg__`,
+        import: `__(() => import('./themes/${t.name}')) as unknown as DynamicThemeReg__`,
       }
     }))
   await fs.writeFile(
@@ -55,40 +50,3 @@ export const bundledThemes = Object.fromEntries(bundledThemesInfo.map(i => [i.id
 'utf-8',
   )
 }
-
-function isLightColor(hex: string) {
-  const [r, g, b] = hex.slice(1).match(/.{2}/g)!.map(i => Number.parseInt(i, 16))
-  return (r * 299 + g * 587 + b * 114) / 1000 > 128
-}
-
-function guessThemeType(id: string, theme: any) {
-  if ('type' in theme)
-    return theme.type
-  let color
-  if (id.includes('dark') || id.includes('dimmed') || id.includes('black'))
-    color = 'dark'
-  else if (id.includes('light') || id.includes('white') || id === 'slack-ochin')
-    color = 'light'
-  else if (theme.colors.background)
-    color = isLightColor(theme.colors.background) ? 'light' : 'dark'
-  else if (theme.colors['editor.background'])
-    color = isLightColor(theme.colors['editor.background']) ? 'light' : 'dark'
-  else if (theme.colors.foreground)
-    color = isLightColor(theme.colors.foreground) ? 'dark' : 'light'
-
-  else
-    color = 'light'
-  return color
-}
-
-function guessThemeName(id: string, theme: any) {
-  if (theme.displayName)
-    return theme.displayName
-  let name: string = theme.name || id
-  name = name.split(/[_-]/)
-    .map(i => i[0].toUpperCase() + i.slice(1))
-    .join(' ')
-    .replace(/github/ig, 'GitHub')
-    .replace('Rose Pine', 'Rosé Pine')
-  return name
-}

packages/shikiji/scripts/prepare/utils.ts

@@ -54,7 +54,7 @@ export function cleanupThemeReg(theme: ThemeRegistration) {
       'bg',
       'include',
       'settings',
-
+      'displayName',
       'tokenColors',
       'semanticHighlighting',
       'semanticTokenColors',

packages/shikiji/src/assets/langs.ts

@@ -75,7 +75,7 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
   },
   {
     'id': 'bat',
-    'name': 'Batch',
+    'name': 'Batch File',
     'aliases': [
       'batch'
     ],
@@ -156,6 +156,9 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
   {
     'id': 'coffee',
     'name': 'CoffeeScript',
+    'aliases': [
+      'coffeescript'
+    ],
     'import': (() => import('./langs/coffee')) as DynamicLangReg
   },
   {
@@ -225,7 +228,7 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
   },
   {
     'id': 'docker',
-    'name': 'Docker',
+    'name': 'Dockerfile',
     'aliases': [
       'dockerfile'
     ],
@@ -393,11 +396,6 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
     'name': 'HTML',
     'import': (() => import('./langs/html')) as DynamicLangReg
   },
-  {
-    'id': 'http',
-    'name': 'HTTP',
-    'import': (() => import('./langs/http')) as DynamicLangReg
-  },
   {
     'id': 'imba',
     'name': 'Imba',
@@ -425,9 +423,9 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
     'import': (() => import('./langs/javascript')) as DynamicLangReg
   },
   {
-    'id': 'jinja-html',
+    'id': 'jinja',
     'name': 'Jinja',
-    'import': (() => import('./langs/jinja-html')) as DynamicLangReg
+    'import': (() => import('./langs/jinja')) as DynamicLangReg
   },
   {
     'id': 'jison',
@@ -567,7 +565,7 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
   },
   {
     'id': 'mojo',
-    'name': 'MagicPython',
+    'name': 'Mojo',
     'import': (() => import('./langs/mojo')) as DynamicLangReg
   },
   {
@@ -1027,7 +1025,7 @@ export const bundledLanguagesBase = Object.fromEntries(bundledLanguagesInfo.map(
 
 export const bundledLanguagesAlias = Object.fromEntries(bundledLanguagesInfo.flatMap(i => i.aliases?.map(a => [a, i.import]) || []))
 
-export type BuiltinLanguage = 'abap' | 'actionscript-3' | 'ada' | 'apache' | 'apex' | 'apl' | 'applescript' | 'ara' | 'asm' | 'astro' | 'awk' | 'ballerina' | 'bash' | 'bat' | 'batch' | 'be' | 'beancount' | 'berry' | 'bibtex' | 'bicep' | 'blade' | 'c' | 'c#' | 'c++' | 'cadence' | 'cdc' | 'clarity' | 'clj' | 'clojure' | 'cmake' | 'cmd' | 'cobol' | 'codeql' | 'coffee' | 'console' | 'cpp' | 'cql' | 'crystal' | 'cs' | 'csharp' | 'css' | 'csv' | 'cue' | 'cypher' | 'd' | 'dart' | 'dax' | 'diff' | 'docker' | 'dockerfile' | 'dream-maker' | 'elixir' | 'elm' | 'erb' | 'erl' | 'erlang' | 'f#' | 'fish' | 'fs' | 'fsharp' | 'fsl' | 'gdresource' | 'gdscript' | 'gdshader' | 'gherkin' | 'git-commit' | 'git-rebase' | 'gjs' | 'glimmer-js' | 'glimmer-ts' | 'glsl' | 'gnuplot' | 'go' | 'gql' | 'graphql' | 'groovy' | 'gts' | 'hack' | 'haml' | 'handlebars' | 'haskell' | 'hbs' | 'hcl' | 'hjson' | 'hlsl' | 'hs' | 'html' | 'http' | 'imba' | 'ini' | 'jade' | 'java' | 'javascript' | 'jinja-html' | 'jison' | 'js' | 'json' | 'json5' | 'jsonc' | 'jsonl' | 'jsonnet' | 'jssm' | 'jsx' | 'julia' | 'kotlin' | 'kql' | 'kt' | 'kts' | 'kusto' | 'latex' | 'less' | 'liquid' | 'lisp' | 'logo' | 'lua' | 'make' | 'makefile' | 'markdown' | 'marko' | 'matlab' | 'md' | 'mdc' | 'mdx' | 'mermaid' | 'mojo' | 'nar' | 'narrat' | 'nextflow' | 'nf' | 'nginx' | 'nim' | 'nix' | 'nu' | 'nushell' | 'objc' | 'objective-c' | 'objective-cpp' | 'ocaml' | 'pascal' | 'perl' | 'perl6' | 'php' | 'plsql' | 'postcss' | 'powerquery' | 'powershell' | 'prisma' | 'prolog' | 'properties' | 'proto' | 'ps' | 'ps1' | 'pug' | 'puppet' | 'purescript' | 'py' | 'python' | 'ql' | 'r' | 'raku' | 'razor' | 'rb' | 'reg' | 'rel' | 'riscv' | 'rs' | 'rst' | 'ruby' | 'rust' | 'sas' | 'sass' | 'scala' | 'scheme' | 'scss' | 'sh' | 'shader' | 'shaderlab' | 'shell' | 'shellscript' | 'shellsession' | 'smalltalk' | 'solidity' | 'sparql' | 'spl' | 'splunk' | 'sql' | 'ssh-config' | 'stata' | 'styl' | 'stylus' | 'svelte' | 'swift' | 'system-verilog' | 'tasl' | 'tcl' | 'tex' | 'toml' | 'ts' | 'tsx' | 'turtle' | 'twig' | 'typescript' | 'v' | 'vb' | 'verilog' | 'vhdl' | 'vim' | 'viml' | 'vimscript' | 'vue' | 'vue-html' | 'vy' | 'vyper' | 'wasm' | 'wenyan' | 'wgsl' | 'wl' | 'wolfram' | 'xml' | 'xsl' | 'yaml' | 'yml' | 'zenscript' | 'zig' | 'zsh' | '文言'
+export type BuiltinLanguage = 'abap' | 'actionscript-3' | 'ada' | 'apache' | 'apex' | 'apl' | 'applescript' | 'ara' | 'asm' | 'astro' | 'awk' | 'ballerina' | 'bash' | 'bat' | 'batch' | 'be' | 'beancount' | 'berry' | 'bibtex' | 'bicep' | 'blade' | 'c' | 'c#' | 'c++' | 'cadence' | 'cdc' | 'clarity' | 'clj' | 'clojure' | 'cmake' | 'cmd' | 'cobol' | 'codeql' | 'coffee' | 'coffeescript' | 'console' | 'cpp' | 'cql' | 'crystal' | 'cs' | 'csharp' | 'css' | 'csv' | 'cue' | 'cypher' | 'd' | 'dart' | 'dax' | 'diff' | 'docker' | 'dockerfile' | 'dream-maker' | 'elixir' | 'elm' | 'erb' | 'erl' | 'erlang' | 'f#' | 'fish' | 'fs' | 'fsharp' | 'fsl' | 'gdresource' | 'gdscript' | 'gdshader' | 'gherkin' | 'git-commit' | 'git-rebase' | 'gjs' | 'glimmer-js' | 'glimmer-ts' | 'glsl' | 'gnuplot' | 'go' | 'gql' | 'graphql' | 'groovy' | 'gts' | 'hack' | 'haml' | 'handlebars' | 'haskell' | 'hbs' | 'hcl' | 'hjson' | 'hlsl' | 'hs' | 'html' | 'imba' | 'ini' | 'jade' | 'java' | 'javascript' | 'jinja' | 'jison' | 'js' | 'json' | 'json5' | 'jsonc' | 'jsonl' | 'jsonnet' | 'jssm' | 'jsx' | 'julia' | 'kotlin' | 'kql' | 'kt' | 'kts' | 'kusto' | 'latex' | 'less' | 'liquid' | 'lisp' | 'logo' | 'lua' | 'make' | 'makefile' | 'markdown' | 'marko' | 'matlab' | 'md' | 'mdc' | 'mdx' | 'mermaid' | 'mojo' | 'nar' | 'narrat' | 'nextflow' | 'nf' | 'nginx' | 'nim' | 'nix' | 'nu' | 'nushell' | 'objc' | 'objective-c' | 'objective-cpp' | 'ocaml' | 'pascal' | 'perl' | 'perl6' | 'php' | 'plsql' | 'postcss' | 'powerquery' | 'powershell' | 'prisma' | 'prolog' | 'properties' | 'proto' | 'ps' | 'ps1' | 'pug' | 'puppet' | 'purescript' | 'py' | 'python' | 'ql' | 'r' | 'raku' | 'razor' | 'rb' | 'reg' | 'rel' | 'riscv' | 'rs' | 'rst' | 'ruby' | 'rust' | 'sas' | 'sass' | 'scala' | 'scheme' | 'scss' | 'sh' | 'shader' | 'shaderlab' | 'shell' | 'shellscript' | 'shellsession' | 'smalltalk' | 'solidity' | 'sparql' | 'spl' | 'splunk' | 'sql' | 'ssh-config' | 'stata' | 'styl' | 'stylus' | 'svelte' | 'swift' | 'system-verilog' | 'tasl' | 'tcl' | 'tex' | 'toml' | 'ts' | 'tsx' | 'turtle' | 'twig' | 'typescript' | 'v' | 'vb' | 'verilog' | 'vhdl' | 'vim' | 'viml' | 'vimscript' | 'vue' | 'vue-html' | 'vy' | 'vyper' | 'wasm' | 'wenyan' | 'wgsl' | 'wl' | 'wolfram' | 'xml' | 'xsl' | 'yaml' | 'yml' | 'zenscript' | 'zig' | 'zsh' | '文言'
 
 export const bundledLanguages = {
   ...bundledLanguagesBase,

packages/shikiji/src/assets/themes.ts

@@ -148,7 +148,7 @@ export const bundledThemesInfo: BundledThemeInfo[] = [
   {
     "id": "slack-ochin",
     "displayName": "Slack Ochin",
-    "type": "dark",
+    "type": "light",
     "import": (() => import('./themes/slack-ochin')) as unknown as DynamicThemeReg
   },
   {

packages/shikiji/test/core.test.ts

@@ -72,6 +72,7 @@ describe('should', () => {
           "sql",
           "c",
           "glsl",
+          "cpp-macro",
           "cpp",
           "c++",
         ]

packages/shikiji/test/index.test.ts

@@ -66,7 +66,9 @@ describe('should', () => {
           "clojure",
           "cmd",
           "coffee",
+          "coffeescript",
           "cpp",
+          "cpp-macro",
           "cs",
           "csharp",
           "css",
@@ -109,6 +111,7 @@ describe('should', () => {
           "make",
           "makefile",
           "markdown",
+          "markdown-vue",
           "md",
           "objc",
           "objective-c",
@@ -146,7 +149,6 @@ describe('should', () => {
           "vb",
           "vue",
           "vue-directives",
-          "vue-injection-markdown",
           "vue-interpolations",
           "vue-sfc-style-variable-injection",
           "xml",
@@ -167,7 +169,7 @@ describe('should', () => {
     await shiki.loadLanguage('js')
 
     expect(shiki.getLoadedLanguages().length)
-      .toMatchInlineSnapshot(`89`)
+      .toMatchInlineSnapshot(`91`)
 
     expect(shiki.getLoadedThemes())
       .toMatchInlineSnapshot(`