feat: generate preprocessor sourcemaps (#101)

* feat: generate preprocessor sourcemaps

* fix: lint error after format

* fix: set filename as source instead of output name for sourcemap

Author: dominikg

.changeset/plenty-pumpkins-buy.md

@@ -0,0 +1,5 @@
+---
+'@sveltejs/vite-plugin-svelte': patch
+---
+
+reduce log output with log.once function to filter repetetive messages

.changeset/proud-dragons-shout.md

@@ -0,0 +1,7 @@
+---
+'@sveltejs/vite-plugin-svelte': minor
+---
+
+Experimental: Generate sourcemaps for preprocessors that lack them
+
+enable option `experimental.generateMissingPreprocessorSourcemaps` to use it

packages/vite-plugin-svelte/package.json

@@ -46,6 +46,7 @@
   "dependencies": {
     "@rollup/pluginutils": "^4.1.0",
     "debug": "^4.3.2",
+    "diff-match-patch": "^1.0.5",
     "kleur": "^4.1.4",
     "magic-string": "^0.25.7",
     "require-relative": "^0.8.7",
@@ -57,6 +58,7 @@
   },
   "devDependencies": {
     "@types/debug": "^4.1.6",
+    "@types/diff-match-patch": "^1.0.32",
     "esbuild": "^0.12.15",
     "rollup": "^2.53.1",
     "svelte": "^3.38.3",

packages/vite-plugin-svelte/src/utils/__tests__/sourcemap.spec.ts

@@ -0,0 +1,24 @@
+import { buildMagicString, buildSourceMap } from '../sourcemap';
+
+describe('sourcemap', () => {
+	describe('buildMagicString', () => {
+		it('should return a valid magic string', () => {
+			const from = 'h1{color: blue}\nh2{color: green}\nh3{color: red}\n';
+			const to = 'h1{color: blue}\ndiv{color: white}\nh3{color: red}\nh2{color: green}\n';
+			const m = buildMagicString(from, to);
+			expect(m).toBeDefined();
+			expect(m.original).toBe(from);
+			expect(m.toString()).toBe(to);
+		});
+	});
+	describe('buildSourceMap', () => {
+		it('should return a map with mappings and filename', () => {
+			const map = buildSourceMap('foo', 'bar', 'foo.txt');
+			expect(map).toBeDefined();
+			expect(map.mappings).toBeDefined();
+			expect(map.mappings[0]).toBeDefined();
+			expect(map.mappings[0][0]).toBeDefined();
+			expect(map.sources[0]).toBe('foo.txt');
+		});
+	});
+});

packages/vite-plugin-svelte/src/utils/log.ts

@@ -64,16 +64,30 @@ function _log(logger: any, message: string, payload?: any) {
 export interface LogFn {
 	(message: string, payload?: any): void;
 	enabled: boolean;
+	once: (message: string, payload?: any) => void;
 }
 
 function createLogger(level: string): LogFn {
 	const logger = loggers[level];
 	const logFn: LogFn = _log.bind(null, logger) as LogFn;
+	const logged = new Set<String>();
+	const once = function (message: string, payload?: any) {
+		if (logged.has(message)) {
+			return;
+		}
+		logged.add(message);
+		logFn.apply(null, [message, payload]);
+	};
 	Object.defineProperty(logFn, 'enabled', {
 		get() {
 			return logger.enabled;
 		}
 	});
+	Object.defineProperty(logFn, 'once', {
+		get() {
+			return once;
+		}
+	});
 	return logFn;
 }
 

packages/vite-plugin-svelte/src/utils/options.ts

@@ -405,6 +405,11 @@ export interface ExperimentalOptions {
 	 * @default false
 	 */
 	useVitePreprocess?: boolean;
+
+	/**
+	 * wrap all preprocessors in with a function that adds a sourcemap to the output if missing
+	 */
+	generateMissingPreprocessorSourcemaps?: boolean;
 }
 
 export interface ResolvedOptions extends Options {

packages/vite-plugin-svelte/src/utils/preprocess.ts

@@ -1,8 +1,9 @@
 import { ResolvedConfig, TransformResult, Plugin } from 'vite';
 import MagicString from 'magic-string';
-import { Preprocessor, PreprocessorGroup, ResolvedOptions } from './options';
+import { Preprocessor, PreprocessorGroup, Processed, ResolvedOptions } from './options';
 import { TransformPluginContext } from 'rollup';
 import { log } from './log';
+import { buildSourceMap } from './sourcemap';
 
 const supportedStyleLangs = ['css', 'less', 'sass', 'scss', 'styl', 'stylus', 'postcss'];
 
@@ -38,13 +39,13 @@ function createPreprocessorFromVitePlugin(
 			moduleId
 		)) as TransformResult;
 		// TODO vite:css transform currently returns an empty mapping that would kill svelte compiler.
-		const hasMap = !!transformResult.map?.mappings;
+		const hasMap = transformResult.map && transformResult.map?.mappings !== '';
 		if (transformResult.map?.sources?.[0] === moduleId) {
 			transformResult.map.sources[0] = filename as string;
 		}
 		return {
 			code: transformResult.code,
-			map: hasMap ? (transformResult.map as object) : null,
+			map: hasMap ? (transformResult.map as object) : undefined,
 			dependencies: transformResult.deps
 		};
 	};
@@ -73,7 +74,7 @@ function createInjectScopeEverythingRulePreprocessorGroup(): PreprocessorGroup {
 			s.append(' *{}');
 			return {
 				code: s.toString(),
-				map: s.generateDecodedMap({ file: filename })
+				map: s.generateDecodedMap({ source: filename, hires: true })
 			};
 		}
 	};
@@ -158,4 +159,62 @@ export function addExtraPreprocessors(options: ResolvedOptions, config: Resolved
 			options.preprocess = [options.preprocess, ...extra];
 		}
 	}
+	const generateMissingSourceMaps = !!options.experimental?.generateMissingPreprocessorSourcemaps;
+	if (options.preprocess && generateMissingSourceMaps) {
+		options.preprocess = Array.isArray(options.preprocess)
+			? options.preprocess.map((p, i) => validateSourceMapOutputWrapper(p, i))
+			: validateSourceMapOutputWrapper(options.preprocess, 0);
+	}
+}
+
+function validateSourceMapOutputWrapper(group: PreprocessorGroup, i: number): PreprocessorGroup {
+	const wrapper: PreprocessorGroup = {};
+
+	for (const [processorType, processorFn] of Object.entries(group) as Array<
+		// eslint-disable-next-line no-unused-vars
+		[keyof PreprocessorGroup, (options: { filename?: string; content: string }) => Processed]
+	>) {
+		wrapper[processorType] = async (options) => {
+			const result = await processorFn(options);
+
+			if (result && result.code !== options.content) {
+				let invalidMap = false;
+				if (!result.map) {
+					invalidMap = true;
+					log.warn.enabled &&
+						log.warn.once(
+							`preprocessor at index ${i} did not return a sourcemap for ${processorType} transform`,
+							{
+								filename: options.filename,
+								type: processorType,
+								processor: processorFn.toString()
+							}
+						);
+				} else if ((result.map as any)?.mappings === '') {
+					invalidMap = true;
+					log.warn.enabled &&
+						log.warn.once(
+							`preprocessor at index ${i} returned an invalid empty sourcemap for ${processorType} transform`,
+							{
+								filename: options.filename,
+								type: processorType,
+								processor: processorFn.toString()
+							}
+						);
+				}
+				if (invalidMap) {
+					try {
+						const map = buildSourceMap(options.content, result.code, options.filename);
+						log.warn.once('adding generated sourcemap to preprocesor result');
+						result.map = map;
+					} catch (e) {
+						log.error(`failed to build sourcemap`, e);
+					}
+				}
+			}
+			return result;
+		};
+	}
+
+	return wrapper;
 }

packages/vite-plugin-svelte/src/utils/sourcemap.ts

@@ -0,0 +1,45 @@
+import MagicString, { MagicStringOptions } from 'magic-string';
+import { diff_match_patch, DIFF_DELETE, DIFF_INSERT } from 'diff-match-patch';
+
+export function buildMagicString(
+	from: string,
+	to: string,
+	options?: MagicStringOptions
+): MagicString {
+	const dmp = new diff_match_patch();
+	const diffs = dmp.diff_main(from, to);
+	dmp.diff_cleanupSemantic(diffs);
+	const m = new MagicString(from, options);
+	let pos = 0;
+	for (let i = 0; i < diffs.length; i++) {
+		const diff = diffs[i];
+		const nextDiff = diffs[i + 1];
+		if (diff[0] === DIFF_DELETE) {
+			if (nextDiff?.[0] === DIFF_INSERT) {
+				// delete followed by insert, use overwrite and skip ahead
+				m.overwrite(pos, pos + diff[1].length, nextDiff[1]);
+				i++;
+			} else {
+				m.remove(pos, pos + diff[1].length);
+			}
+			pos += diff[1].length;
+		} else if (diff[0] === DIFF_INSERT) {
+			if (nextDiff) {
+				m.appendRight(pos, diff[1]);
+			} else {
+				m.append(diff[1]);
+			}
+		} else {
+			// unchanged block, advance pos
+			pos += diff[1].length;
+		}
+	}
+	// at this point m.toString() === to
+	return m;
+}
+
+export function buildSourceMap(from: string, to: string, filename?: string) {
+	// @ts-ignore
+	const m = buildMagicString(from, to, { filename });
+	return m.generateDecodedMap({ source: filename, hires: true, includeContent: false });
+}

pnpm-lock.yaml

@@ -156,25 +156,26 @@ afterAll(async () => {
 			err = e;
 		}
 	}
-
-	// unlink node modules to prevent removal of linked modules on cleanup
-	const temp_node_modules = path.join(tempDir, 'node_modules');
-	try {
-		await fs.unlink(temp_node_modules);
-	} catch (e) {
-		console.error(`failed to unlink ${temp_node_modules}`);
-		if (!err) {
-			err = e;
+	if (tempDir) {
+		// unlink node modules to prevent removal of linked modules on cleanup
+		const temp_node_modules = path.join(tempDir, 'node_modules');
+		try {
+			await fs.unlink(temp_node_modules);
+		} catch (e) {
+			console.error(`failed to unlink ${temp_node_modules}`);
+			if (!err) {
+				err = e;
+			}
 		}
-	}
-	const logDir = path.join(testDir(), 'logs');
-	const logFile = path.join(logDir, 'browser.log');
-	try {
-		await fs.writeFile(logFile, logs.join('\n'));
-	} catch (e) {
-		console.error(`failed to write browserlogs in ${logFile}`, e);
-		if (!err) {
-			err = e;
+		const logDir = path.join(testDir(), 'logs');
+		const logFile = path.join(logDir, 'browser.log');
+		try {
+			await fs.writeFile(logFile, logs.join('\n'));
+		} catch (e) {
+			console.error(`failed to write browserlogs in ${logFile}`, e);
+			if (!err) {
+				err = e;
+			}
 		}
 	}