Skip to content

Commit 8aa5e85

Browse files
ricardonunez-ioricardonunez-io
committed
Lazy load languages other than base default ones
1 parent b1d226e commit 8aa5e85

File tree

5 files changed

+147
-67
lines changed

5 files changed

+147
-67
lines changed

‎.yarn/install-state.gz‎

3.62 KB
Binary file not shown.

‎packages/mdx/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
"main": "./dist/index.js",
66
"types": "./dist/index.d.ts",
77
"sideEffects": false,
8-
"files": ["dist"],
8+
"files": [
9+
"dist"
10+
],
911
"exports": {
1012
".": {
1113
"import": "./dist/index.js",
@@ -67,6 +69,7 @@
6769
},
6870
"dependencies": {
6971
"hast-util-to-string": "^3.0.1",
72+
"mdast-util-mdx-jsx": "^3.2.0",
7073
"next-mdx-remote-client": "^1.0.3",
7174
"rehype-katex": "^7.0.1",
7275
"remark-gfm": "^4.0.0",

‎packages/mdx/src/plugins/rehype/rehypeSyntaxHighlighting.ts‎

Lines changed: 89 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { Element, Root } from 'hast';
22
import { toString } from 'hast-util-to-string';
3+
import type { MdxJsxFlowElementHast, MdxJsxTextElementHast } from 'mdast-util-mdx-jsx';
34
import { createHighlighter, type Highlighter } from 'shiki';
45
import type { Plugin } from 'unified';
56
import { visit } from 'unist-util-visit';
@@ -15,6 +16,7 @@ import {
1516
DEFAULT_DARK_THEME,
1617
DEFAULT_LIGHT_THEME,
1718
DEFAULT_THEMES,
19+
DEFAULT_LANGS,
1820
} from './shiki-constants.js';
1921
import {
2022
getLanguage,
@@ -35,7 +37,7 @@ async function getHighlighter(): Promise<Highlighter> {
3537
if (!highlighterPromise) {
3638
highlighterPromise = createHighlighter({
3739
themes: DEFAULT_THEMES,
38-
langs: UNIQUE_LANGS,
40+
langs: DEFAULT_LANGS,
3941
});
4042
}
4143
return highlighterPromise;
@@ -45,6 +47,7 @@ export const rehypeSyntaxHighlighting: Plugin<[RehypeSyntaxHighlightingOptions?]
4547
options = {}
4648
) => {
4749
return async (tree) => {
50+
const asyncNodesToProcess: Promise<void>[] = [];
4851
const themesToLoad: ShikiTheme[] = [];
4952
if (options.themes) {
5053
themesToLoad.push(options.themes.dark);
@@ -91,74 +94,94 @@ export const rehypeSyntaxHighlighting: Plugin<[RehypeSyntaxHighlightingOptions?]
9194
getLanguage(child, DEFAULT_LANG_ALIASES) ??
9295
DEFAULT_LANG;
9396

94-
try {
95-
const code = toString(node);
96-
const lines = code.split('\n');
97-
let linesToHighlight = getLinesToHighlight(node, lines.length);
98-
99-
const hast = highlighter.codeToHast(code, {
100-
lang: lang ?? DEFAULT_LANG,
101-
themes: {
102-
light:
103-
options.themes?.light ??
104-
options.theme ??
105-
(options.codeStyling === 'dark' ? DEFAULT_DARK_THEME : DEFAULT_LIGHT_THEME),
106-
dark: options.themes?.dark ?? options.theme ?? DEFAULT_DARK_THEME,
107-
},
108-
colorReplacements: shikiColorReplacements,
109-
tabindex: false,
110-
tokenizeMaxLineLength: 1000,
111-
});
112-
113-
const codeElement = hast.children[0] as Element;
114-
if (!codeElement) return;
115-
116-
let lineNumber = 0;
117-
visit(codeElement, 'element', (span, spanIndex, spanParent) => {
118-
if (
119-
!spanParent ||
120-
spanParent.type !== 'element' ||
121-
spanParent.tagName !== 'code' ||
122-
span.tagName !== 'span' ||
123-
(!span.children.length && spanIndex === spanParent.children.length - 1) ||
124-
(typeof span.properties.class !== 'string' && !Array.isArray(span.properties.class)) ||
125-
!span.properties.class.includes('line')
126-
) {
127-
return;
128-
}
129-
130-
lineNumber++;
131-
if (linesToHighlight.includes(lineNumber)) {
132-
if (typeof span.properties.class === 'string') {
133-
span.properties.class += ' ' + LINE_HIGHLIGHT_CLASS;
134-
} else {
135-
span.properties.class = [...span.properties.class, LINE_HIGHLIGHT_CLASS];
136-
}
137-
}
138-
});
139-
140-
const preChild = codeElement.children[0] as Element;
141-
const numberOfLines = lineNumber;
142-
143-
node.data = node.data ?? {};
144-
if (node.data.meta) {
145-
node.data.meta = node.data.meta.replace(lineHighlightPattern, '').trim();
146-
}
147-
codeElement.data = node.data;
148-
codeElement.properties.numberOfLines = numberOfLines;
149-
if (preChild) {
150-
preChild.data = node.data;
151-
preChild.properties.numberOfLines = numberOfLines;
152-
}
153-
parent.children.splice(index, 1, codeElement);
154-
} catch (err) {
155-
if (err instanceof Error && /Unknown language/.test(err.message)) {
156-
return;
157-
}
158-
throw err;
97+
if (!DEFAULT_LANGS.includes(lang)) {
98+
asyncNodesToProcess.push(
99+
highlighter.loadLanguage(lang).then(() => {
100+
traverseNode(node, index, parent, highlighter, lang, options);
101+
})
102+
);
103+
} else {
104+
traverseNode(node, index, parent, highlighter, lang, options);
159105
}
160106
});
107+
await Promise.all(asyncNodesToProcess);
161108
};
162109
};
163110

111+
const traverseNode = (
112+
node: Element,
113+
index: number,
114+
parent: Element | Root | MdxJsxTextElementHast | MdxJsxFlowElementHast,
115+
highlighter: Highlighter,
116+
lang: ShikiLang,
117+
options: RehypeSyntaxHighlightingOptions
118+
) => {
119+
try {
120+
const code = toString(node);
121+
const lines = code.split('\n');
122+
let linesToHighlight = getLinesToHighlight(node, lines.length);
123+
124+
const hast = highlighter.codeToHast(code, {
125+
lang: lang ?? DEFAULT_LANG,
126+
themes: {
127+
light:
128+
options.themes?.light ??
129+
options.theme ??
130+
(options.codeStyling === 'dark' ? DEFAULT_DARK_THEME : DEFAULT_LIGHT_THEME),
131+
dark: options.themes?.dark ?? options.theme ?? DEFAULT_DARK_THEME,
132+
},
133+
colorReplacements: shikiColorReplacements,
134+
tabindex: false,
135+
tokenizeMaxLineLength: 1000,
136+
});
137+
138+
const codeElement = hast.children[0] as Element;
139+
if (!codeElement) return;
140+
141+
let lineNumber = 0;
142+
visit(codeElement, 'element', (span, spanIndex, spanParent) => {
143+
if (
144+
!spanParent ||
145+
spanParent.type !== 'element' ||
146+
spanParent.tagName !== 'code' ||
147+
span.tagName !== 'span' ||
148+
(!span.children.length && spanIndex === spanParent.children.length - 1) ||
149+
(typeof span.properties.class !== 'string' && !Array.isArray(span.properties.class)) ||
150+
!span.properties.class.includes('line')
151+
) {
152+
return;
153+
}
154+
155+
lineNumber++;
156+
if (linesToHighlight.includes(lineNumber)) {
157+
if (typeof span.properties.class === 'string') {
158+
span.properties.class += ' ' + LINE_HIGHLIGHT_CLASS;
159+
} else {
160+
span.properties.class = [...span.properties.class, LINE_HIGHLIGHT_CLASS];
161+
}
162+
}
163+
});
164+
165+
const preChild = codeElement.children[0] as Element;
166+
const numberOfLines = lineNumber;
167+
168+
node.data = node.data ?? {};
169+
if (node.data.meta) {
170+
node.data.meta = node.data.meta.replace(lineHighlightPattern, '').trim();
171+
}
172+
codeElement.data = node.data;
173+
codeElement.properties.numberOfLines = numberOfLines;
174+
if (preChild) {
175+
preChild.data = node.data;
176+
preChild.properties.numberOfLines = numberOfLines;
177+
}
178+
parent.children.splice(index, 1, codeElement);
179+
} catch (err) {
180+
if (err instanceof Error && /Unknown language/.test(err.message)) {
181+
return;
182+
}
183+
throw err;
184+
}
185+
};
186+
164187
export { UNIQUE_LANGS, DEFAULT_LANG_ALIASES, SHIKI_THEMES, ShikiLang, ShikiTheme };

‎packages/mdx/src/plugins/rehype/shiki-constants.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,3 +460,36 @@ export const SHIKI_THEMES = [
460460

461461
'css-variables', // for users who want to use custom CSS to style their code blocks
462462
] as const;
463+
464+
export const DEFAULT_LANGS = [
465+
'bash',
466+
'blade',
467+
'c',
468+
'css',
469+
'c#',
470+
'c++',
471+
'dart',
472+
'diff',
473+
'go',
474+
'html',
475+
'java',
476+
'javascript',
477+
'jsx',
478+
'json',
479+
'kotlin',
480+
'log',
481+
'lua',
482+
'markdown',
483+
'mdx',
484+
'php',
485+
'powershell',
486+
'python',
487+
'ruby',
488+
'rust',
489+
'solidity',
490+
'swift',
491+
'toml',
492+
'typescript',
493+
'tsx',
494+
'yaml',
495+
];

‎yarn.lock‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ __metadata:
445445
eslint-config-prettier: "npm:8.x"
446446
eslint-plugin-unused-imports: "npm:^3.x"
447447
hast-util-to-string: "npm:^3.0.1"
448+
mdast-util-mdx-jsx: "npm:^3.2.0"
448449
next-mdx-remote-client: "npm:^1.0.3"
449450
prettier: "npm:^3.1.1"
450451
prettier-plugin-tailwindcss: "npm:^0.6.8"
@@ -4203,6 +4204,26 @@ __metadata:
42034204
languageName: node
42044205
linkType: hard
42054206

4207+
"mdast-util-mdx-jsx@npm:^3.2.0":
4208+
version: 3.2.0
4209+
resolution: "mdast-util-mdx-jsx@npm:3.2.0"
4210+
dependencies:
4211+
"@types/estree-jsx": "npm:^1.0.0"
4212+
"@types/hast": "npm:^3.0.0"
4213+
"@types/mdast": "npm:^4.0.0"
4214+
"@types/unist": "npm:^3.0.0"
4215+
ccount: "npm:^2.0.0"
4216+
devlop: "npm:^1.1.0"
4217+
mdast-util-from-markdown: "npm:^2.0.0"
4218+
mdast-util-to-markdown: "npm:^2.0.0"
4219+
parse-entities: "npm:^4.0.0"
4220+
stringify-entities: "npm:^4.0.0"
4221+
unist-util-stringify-position: "npm:^4.0.0"
4222+
vfile-message: "npm:^4.0.0"
4223+
checksum: 10c0/3acadaf3b962254f7ad2990fed4729961dc0217ca31fde9917986e880843f3ecf3392b1f22d569235cacd180d50894ad266db7af598aedca69d330d33c7ac613
4224+
languageName: node
4225+
linkType: hard
4226+
42064227
"mdast-util-mdx@npm:^3.0.0":
42074228
version: 3.0.0
42084229
resolution: "mdast-util-mdx@npm:3.0.0"

0 commit comments

Comments
 (0)