From 159f231e383e2a6a075ee752e2e1deadb605f7fc Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 12 Apr 2021 22:35:19 -0700 Subject: [PATCH 01/20] First pass, add Interpreter, clean out compiler, update deps. --- package.json | 2 +- src/compiler.js | 230 ++++-------------------------------- src/index.js | 1 + src/interpreter.js | 282 +++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 20 ++-- 5 files changed, 318 insertions(+), 217 deletions(-) create mode 100644 src/interpreter.js diff --git a/package.json b/package.json index 7699815..1b641c6 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "compiler" ], "dependencies": { - "@observablehq/parser": "^3.0.0", + "@observablehq/parser": "4.2", "acorn-walk": "^7.0.0" }, "devDependencies": { diff --git a/src/compiler.js b/src/compiler.js index 9e80d50..d4b50c9 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -2,11 +2,6 @@ import { parseCell, parseModule, walk } from "@observablehq/parser"; import { simple } from "acorn-walk"; import { extractPath } from "./utils"; -const AsyncFunction = Object.getPrototypeOf(async function() {}).constructor; -const GeneratorFunction = Object.getPrototypeOf(function*() {}).constructor; -const AsyncGeneratorFunction = Object.getPrototypeOf(async function*() {}) - .constructor; - const setupRegularCell = cell => { let name = null; if (cell.id && cell.id.name) name = cell.id.name; @@ -68,31 +63,6 @@ const setupRegularCell = cell => { return { cellName: name, references, bodyText, cellReferences }; }; -const createRegularCellDefintion = cell => { - const { cellName, references, bodyText, cellReferences } = setupRegularCell( - cell - ); - - let code; - if (cell.body.type !== "BlockStatement") { - if (cell.async) - code = `return (async function(){ return (${bodyText});})()`; - else code = `return (function(){ return (${bodyText});})()`; - } else code = bodyText; - - let f; - if (cell.generator && cell.async) - f = new AsyncGeneratorFunction(...references, code); - else if (cell.async) f = new AsyncFunction(...references, code); - else if (cell.generator) f = new GeneratorFunction(...references, code); - else f = new Function(...references, code); - return { - cellName, - cellFunction: f, - cellReferences - }; -}; - const setupImportCell = cell => { const specifiers = []; if (cell.body.specifiers) @@ -149,124 +119,7 @@ const setupImportCell = cell => { return { specifiers, hasInjections, injections, importString }; }; -const createCellDefinition = ( - cell, - main, - observer, - dependencyMap, - define = true -) => { - if (cell.body.type === "ImportDeclaration") { - const { - specifiers, - hasInjections, - injections, - importString - } = setupImportCell(cell); - // this will display extra names for viewof / mutable imports (for now?) - main.variable(observer()).define( - null, - ["md"], - md => md`~~~javascript -${importString} -~~~` - ); - - const other = main._runtime.module( - dependencyMap.get(cell.body.source.value) - ); - - if (hasInjections) { - const child = other.derive(injections, main); - for (const { name, alias } of specifiers) main.import(name, alias, child); - } else { - for (const { name, alias } of specifiers) main.import(name, alias, other); - } - } else { - const { - cellName, - cellFunction, - cellReferences - } = createRegularCellDefintion(cell); - if (cell.id && cell.id.type === "ViewExpression") { - const reference = `viewof ${cellName}`; - if (define) { - main - .variable(observer(reference)) - .define(reference, cellReferences, cellFunction); - main - .variable(observer(cellName)) - .define(cellName, ["Generators", reference], (G, _) => G.input(_)); - } else { - main.redefine(reference, cellReferences, cellFunction); - main.redefine(cellName, ["Generators", reference], (G, _) => - G.input(_) - ); - } - } else if (cell.id && cell.id.type === "MutableExpression") { - const initialName = `initial ${cellName}`; - const mutableName = `mutable ${cellName}`; - if (define) { - main.variable(null).define(initialName, cellReferences, cellFunction); - main - .variable(observer(mutableName)) - .define(mutableName, ["Mutable", initialName], (M, _) => new M(_)); - main - .variable(observer(cellName)) - .define(cellName, [mutableName], _ => _.generator); - } else { - main.redefine(initialName, cellReferences, cellFunction); - main.redefine( - mutableName, - ["Mutable", initialName], - (M, _) => new M(_) - ); - main.redefine(cellName, [mutableName], _ => _.generator); - } - } else { - if (define) - main - .variable(observer(cellName)) - .define(cellName, cellReferences, cellFunction); - else main.redefine(cellName, cellReferences, cellFunction); - } - } -}; -const createModuleDefintion = async ( - moduleObject, - resolveModule, - resolveFileAttachments -) => { - const filteredImportCells = new Set(); - const importCells = moduleObject.cells.filter(({ body }) => { - if ( - body.type !== "ImportDeclaration" || - filteredImportCells.has(body.source.value) - ) - return false; - filteredImportCells.add(body.source.value); - return true; - }); - - const dependencyMap = new Map(); - const importCellsPromise = importCells.map(async ({ body }) => { - const fromModule = await resolveModule(body.source.value); - dependencyMap.set(body.source.value, fromModule); - }); - await Promise.all(importCellsPromise); - - return function define(runtime, observer) { - const main = runtime.module(); - main.builtin( - "FileAttachment", - runtime.fileAttachments(resolveFileAttachments) - ); - for (const cell of moduleObject.cells) - createCellDefinition(cell, main, observer, dependencyMap); - }; -}; - -const ESMImports = (moduleObject, resolvePath) => { +const ESMImports = (moduleObject, resolveImportPath) => { const importMap = new Map(); let importSrc = ""; let j = 0; @@ -276,7 +129,7 @@ const ESMImports = (moduleObject, resolvePath) => { continue; const defineName = `define${++j}`; - const fromPath = resolvePath(body.source.value); + const fromPath = resolveImportPath(body.source.value); importMap.set(body.source.value, { defineName, fromPath }); importSrc += `import ${defineName} from "${fromPath}";\n`; } @@ -386,8 +239,12 @@ ${bodyText} }) .join("\n"); }; -const createESModule = (moduleObject, resolvePath, resolveFileAttachments) => { - const { importSrc, importMap } = ESMImports(moduleObject, resolvePath); +const createESModule = ( + moduleObject, + resolveImportPath, + resolveFileAttachments +) => { + const { importSrc, importMap } = ESMImports(moduleObject, resolveImportPath); return `${importSrc}export default function define(runtime, observer) { const main = runtime.module(); ${ESMAttachments(moduleObject, resolveFileAttachments)} @@ -396,71 +253,32 @@ ${ESMVariables(moduleObject, importMap) || ""} }`; }; -const defaultResolver = async path => { - const source = extractPath(path); - return import(`https://api.observablehq.com/${source}.js?v=3`).then( - m => m.default - ); -}; -const defaultResolvePath = path => { +function defaultResolveImportPath(path) { const source = extractPath(path); return `https://api.observablehq.com/${source}.js?v=3`; -}; +} +function defaultResolveFileAttachments(name) { + return name; +} export class Compiler { - constructor( - resolve = defaultResolver, - resolveFileAttachments = name => name, - resolvePath = defaultResolvePath - ) { - this.resolve = resolve; + constructor(params = {}) { + const { + resolveFileAttachments = defaultResolveFileAttachments, + resolveImportPath = defaultResolveImportPath + } = params; this.resolveFileAttachments = resolveFileAttachments; - this.resolvePath = resolvePath; - } - async cell(text) { - const cell = parseCell(text); - cell.input = text; - const dependencyMap = new Map(); - if (cell.body.type === "ImportDeclaration") { - const fromModule = await this.resolve(cell.body.source.value); - dependencyMap.set(cell.body.source.value, fromModule); - } - return { - define(module, observer) { - createCellDefinition(cell, module, observer, dependencyMap, true); - }, - redefine(module, observer) { - createCellDefinition(cell, module, observer, dependencyMap, false); - } - }; + this.resolveImportPath = resolveImportPath; } - - async module(text) { + module(text) { const m1 = parseModule(text); - return await createModuleDefintion( + return createESModule( m1, - this.resolve, - this.resolveFileAttachments - ); - } - async notebook(obj) { - const cells = obj.nodes.map(({ value }) => { - const cell = parseCell(value); - cell.input = value; - return cell; - }); - return await createModuleDefintion( - { cells }, - this.resolve, + this.resolveImportPath, this.resolveFileAttachments ); } - - moduleToESModule(text) { - const m1 = parseModule(text); - return createESModule(m1, this.resolvePath, this.resolveFileAttachments); - } - notebookToESModule(obj) { + notebook(obj) { const cells = obj.nodes.map(({ value }) => { const cell = parseCell(value); cell.input = value; @@ -468,7 +286,7 @@ export class Compiler { }); return createESModule( { cells }, - this.resolvePath, + this.resolveImportPath, this.resolveFileAttachments ); } diff --git a/src/index.js b/src/index.js index 1a60bf6..05395bb 100644 --- a/src/index.js +++ b/src/index.js @@ -1 +1,2 @@ export { Compiler } from "./compiler.js"; +export { Interpreter } from "./interpreter.js"; diff --git a/src/interpreter.js b/src/interpreter.js new file mode 100644 index 0000000..4b0c3d1 --- /dev/null +++ b/src/interpreter.js @@ -0,0 +1,282 @@ +import { parseCell, parseModule, walk } from "@observablehq/parser"; +import { simple } from "acorn-walk"; +import { extractPath } from "./utils"; + +function setupImportCell(cell) { + const specifiers = []; + + if (cell.body.specifiers) + for (const specifier of cell.body.specifiers) { + if (specifier.view) { + specifiers.push({ + name: "viewof " + specifier.imported.name, + alias: "viewof " + specifier.local.name + }); + } else if (specifier.mutable) { + specifiers.push({ + name: "mutable " + specifier.imported.name, + alias: "mutable " + specifier.local.name + }); + } + specifiers.push({ + name: specifier.imported.name, + alias: specifier.local.name + }); + } + // If injections is undefined, do not derive! + const hasInjections = cell.body.injections !== undefined; + const injections = []; + if (hasInjections) + for (const injection of cell.body.injections) { + // This currently behaves like notebooks on observablehq.com + // Commenting out the if & else if blocks result in behavior like Example 3 here: https://observablehq.com/d/7ccad009e4d89969 + if (injection.view) { + injections.push({ + name: "viewof " + injection.imported.name, + alias: "viewof " + injection.local.name + }); + } else if (injection.mutable) { + injections.push({ + name: "mutable " + injection.imported.name, + alias: "mutable " + injection.local.name + }); + } + injections.push({ + name: injection.imported.name, + alias: injection.local.name + }); + } + const importString = `import {${specifiers + .map(specifier => `${specifier.name} as ${specifier.alias}`) + .join(", ")}} ${ + hasInjections + ? `with {${injections + .map(injection => `${injection.name} as ${injection.alias}`) + .join(", ")}} ` + : `` + }from "${cell.body.source.value}"`; + + return { specifiers, hasInjections, injections, importString }; +} + +function setupRegularCell(cell) { + let name = null; + if (cell.id && cell.id.name) name = cell.id.name; + else if (cell.id && cell.id.id && cell.id.id.name) name = cell.id.id.name; + let bodyText = cell.input.substring(cell.body.start, cell.body.end); + const cellReferences = (cell.references || []).map(ref => { + if (ref.type === "ViewExpression") { + return "viewof " + ref.id.name; + } else if (ref.type === "MutableExpression") { + return "mutable " + ref.id.name; + } else return ref.name; + }); + let $count = 0; + let indexShift = 0; + const references = (cell.references || []).map(ref => { + if (ref.type === "ViewExpression") { + const $string = "$" + $count; + $count++; + // replace "viewof X" in bodyText with "$($count)" + simple( + cell.body, + { + ViewExpression(node) { + const start = node.start - cell.body.start; + const end = node.end - cell.body.start; + bodyText = + bodyText.slice(0, start + indexShift) + + $string + + bodyText.slice(end + indexShift); + indexShift += $string.length - (end - start); + } + }, + walk + ); + return $string; + } else if (ref.type === "MutableExpression") { + const $string = "$" + $count; + const $stringValue = $string + ".value"; + $count++; + // replace "mutable Y" in bodyText with "$($count).value" + simple( + cell.body, + { + MutableExpression(node) { + const start = node.start - cell.body.start; + const end = node.end - cell.body.start; + bodyText = + bodyText.slice(0, start + indexShift) + + $stringValue + + bodyText.slice(end + indexShift); + indexShift += $stringValue.length - (end - start); + } + }, + walk + ); + return $string; + } else return ref.name; + }); + return { cellName: name, references, bodyText, cellReferences }; +} + +function createRegularCellDefinition(cell) { + const { cellName, references, bodyText, cellReferences } = setupRegularCell( + cell + ); + + let code; + if (cell.body.type !== "BlockStatement") { + if (cell.async) + code = `return (async function(){ return (${bodyText});})()`; + else code = `return (function(){ return (${bodyText});})()`; + } else code = bodyText; + + let f; + if (cell.generator && cell.async) + f = new AsyncGeneratorFunction(...references, code); + else if (cell.async) f = new AsyncFunction(...references, code); + else if (cell.generator) f = new GeneratorFunction(...references, code); + else f = new Function(...references, code); + return { + cellName, + cellFunction: f, + cellReferences + }; +} + +function defaultResolveImportPath(path) { + const source = extractPath(path); + return import(`https://api.observablehq.com/${source}.js?v=3`).then( + m => m.default + ); +} + +function defaultResolveFileAttachments(name) { + return name; +} + +export class Interpreter { + constructor(params = {}) { + const { + module = null, + observer = null, + resolveImportPath = defaultResolveImportPath, + resolveFileAttachments = defaultResolveFileAttachments, + defineImportMarkdown = true, + observeViewofValues = true + } = params; + + // can't be this.module bc of async module(). + // so make defaultObserver follow same convention. + this.defaultModule = module; + this.defaultObserver = observer; + + this.resolveImportPath = resolveImportPath; + this.resolveFileAttachments = resolveFileAttachments; + this.defineImportMarkdown = defineImportMarkdown; + this.observeViewofValues = observeViewofValues; + } + + async module(input, module, observer) { + module = module || this.defaultModule; + observer = observer || this.defaultObserver; + + if (!module) throw Error("No module provided."); + + const parsedModule = parseModule(input); + const cellPromises = []; + for (const cell of parsedModule.cells) { + cell.input = input; + cellPromises.push(this.cell(cell, module, observer)); + } + return Promise.all(cellPromises); + } + + async cell(input, module, observer) { + module = module || this.defaultModule; + observer = observer || this.defaultObserver; + + if (!module) throw Error("No module provided."); + + let cell; + if (typeof input === "string") { + cell = parseCell(input); + cell.input = input; + } else { + cell = input; + } + + if (cell.body.type === "ImportDeclaration") { + const path = cell.body.source.value; + const fromModule = await this.resolveImportPath(path); + let mdVariable, vars; + + const { + specifiers, + hasInjections, + injections, + importString + } = setupImportCell(cell); + + const other = module._runtime.module(fromModule); + + if (this.defineImportMarkdown) + mdVariable = module.variable(observer()).define( + null, + ["md"], + md => md`~~~javascript + ${importString} + ~~~` + ); + if (hasInjections) { + const child = other.derive(injections, module); + vars = specifiers.map(({ name, alias }) => + module.import(name, alias, child) + ); + } else { + vars = specifiers.map(({ name, alias }) => + module.import(name, alias, other) + ); + } + return mdVariable ? [mdVariable, ...vars] : vars; + } else { + const { + cellName, + cellFunction, + cellReferences + } = createRegularCellDefinition(cell); + if (cell.id && cell.id.type === "ViewExpression") { + const reference = `viewof ${cellName}`; + return [ + module + .variable(observer(reference)) + .define(reference, cellReferences, cellFunction), + module + .variable(this.observeViewofValues ? observer(cellName) : null) + .define(cellName, ["Generators", reference], (G, _) => G.input(_)) + ]; + } else if (cell.id && cell.id.type === "MutableExpression") { + const initialName = `initial ${cellName}`; + const mutableName = `mutable ${cellName}`; + return [ + module + .variable(null) + .define(initialName, cellReferences, cellFunction), + module + .variable(observer(mutableName)) + .define(mutableName, ["Mutable", initialName], (M, _) => new M(_)), + module + .variable(observer(cellName)) + .define(cellName, [mutableName], _ => _.generator) + ]; + } else { + return [ + module + .variable(observer(cellName)) + .define(cellName, cellReferences, cellFunction) + ]; + } + } + } +} diff --git a/yarn.lock b/yarn.lock index 037194a..63f1df9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,12 +9,12 @@ dependencies: esm "^3.2.25" -"@observablehq/parser@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@observablehq/parser/-/parser-3.0.0.tgz#6332909fcff5680d994a581b77cadaba765a0d4a" - integrity sha512-BuIfvay+INd2kmUnXNbxYwZ/MnMTYjPEIhns7NuJ9YVIAV+TmxAkZlZdUeCkDpC2XET81BJM8hpOnIzJrqx+1w== +"@observablehq/parser@4.2": + version "4.2.0" + resolved "https://registry.yarnpkg.com/@observablehq/parser/-/parser-4.2.0.tgz#74f34b4fbed4b71e54c6f97e34d9209a6a745f18" + integrity sha512-i1v95BrUB4urKEBIL1CEPWCU8DovxKyMlWf03pEfpOfi5/DiESQ58QRQM6Ws+Sy/nwKXnIx8IMj4kX28Bif4xQ== dependencies: - acorn "^7.0.0" + acorn "^7.1.1" acorn-walk "^7.0.0" "@observablehq/runtime@^4.6.4": @@ -56,16 +56,16 @@ acorn-walk@^7.0.0: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.0.0.tgz#c8ba6f0f1aac4b0a9e32d1f0af12be769528f36b" integrity sha512-7Bv1We7ZGuU79zZbb6rRqcpxo3OY+zrdtloZWoyD8fmGX+FeXRjE+iuGkZjSXLVovLzrsvMGMy0EkwA0E0umxg== -acorn@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.0.0.tgz#26b8d1cd9a9b700350b71c0905546f64d1284e7a" - integrity sha512-PaF/MduxijYYt7unVGRuds1vBC9bFxbNf+VWqhOClfdgy7RlVkQqt610ig1/yxTgsDIfW1cWDel5EBbOy3jdtQ== - acorn@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.1.0.tgz#949d36f2c292535da602283586c2477c57eb2d6c" integrity sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ== +acorn@^7.1.1: + version "7.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" + integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== + async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" From f3a8c9fe7e94774f5e3d6ee43ec2985dbaea6a0f Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 12 Apr 2021 22:38:30 -0700 Subject: [PATCH 02/20] Change umd output name. --- rollup.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollup.config.js b/rollup.config.js index 941a2af..b3b9701 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -8,7 +8,7 @@ export default { compact: true, file: "dist/index.js", format: "umd", - name: "index.js" + name: "unofficial_observablehq_compiler" }, { compact: true, From 6df603a0ca1d686d205115eddfd412c57fbd5793 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 15 Apr 2021 09:55:26 -0700 Subject: [PATCH 03/20] cleanup, add tree shaking. --- package.json | 2 +- src/compiler.js | 205 +++++++++++++-------------------------------- src/interpreter.js | 123 +-------------------------- src/tree-shake.js | 73 ++++++++++++++++ src/utils.js | 122 +++++++++++++++++++++++++++ 5 files changed, 256 insertions(+), 269 deletions(-) create mode 100644 src/tree-shake.js diff --git a/package.json b/package.json index 1b641c6..332c088 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.5.0", + "version": "0.6.0-alpha.0", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", diff --git a/src/compiler.js b/src/compiler.js index d4b50c9..1439ff5 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -1,125 +1,8 @@ -import { parseCell, parseModule, walk } from "@observablehq/parser"; -import { simple } from "acorn-walk"; -import { extractPath } from "./utils"; +import { parseCell, parseModule } from "@observablehq/parser"; +import { setupRegularCell, setupImportCell, extractPath } from "./utils"; +import { computeShakenCells } from "./tree-shake"; -const setupRegularCell = cell => { - let name = null; - if (cell.id && cell.id.name) name = cell.id.name; - else if (cell.id && cell.id.id && cell.id.id.name) name = cell.id.id.name; - let bodyText = cell.input.substring(cell.body.start, cell.body.end); - const cellReferences = (cell.references || []).map(ref => { - if (ref.type === "ViewExpression") { - return "viewof " + ref.id.name; - } else if (ref.type === "MutableExpression") { - return "mutable " + ref.id.name; - } else return ref.name; - }); - let $count = 0; - let indexShift = 0; - const references = (cell.references || []).map(ref => { - if (ref.type === "ViewExpression") { - const $string = "$" + $count; - $count++; - // replace "viewof X" in bodyText with "$($count)" - simple( - cell.body, - { - ViewExpression(node) { - const start = node.start - cell.body.start; - const end = node.end - cell.body.start; - bodyText = - bodyText.slice(0, start + indexShift) + - $string + - bodyText.slice(end + indexShift); - indexShift += $string.length - (end - start); - } - }, - walk - ); - return $string; - } else if (ref.type === "MutableExpression") { - const $string = "$" + $count; - const $stringValue = $string + ".value"; - $count++; - // replace "mutable Y" in bodyText with "$($count).value" - simple( - cell.body, - { - MutableExpression(node) { - const start = node.start - cell.body.start; - const end = node.end - cell.body.start; - bodyText = - bodyText.slice(0, start + indexShift) + - $stringValue + - bodyText.slice(end + indexShift); - indexShift += $stringValue.length - (end - start); - } - }, - walk - ); - return $string; - } else return ref.name; - }); - return { cellName: name, references, bodyText, cellReferences }; -}; - -const setupImportCell = cell => { - const specifiers = []; - if (cell.body.specifiers) - for (const specifier of cell.body.specifiers) { - if (specifier.view) { - specifiers.push({ - name: "viewof " + specifier.imported.name, - alias: "viewof " + specifier.local.name - }); - } else if (specifier.mutable) { - specifiers.push({ - name: "mutable " + specifier.imported.name, - alias: "mutable " + specifier.local.name - }); - } - specifiers.push({ - name: specifier.imported.name, - alias: specifier.local.name - }); - } - // If injections is undefined, do not derive! - const hasInjections = cell.body.injections !== undefined; - const injections = []; - if (hasInjections) - for (const injection of cell.body.injections) { - // This currently behaves like notebooks on observablehq.com - // Commenting out the if & else if blocks result in behavior like Example 3 here: https://observablehq.com/d/7ccad009e4d89969 - if (injection.view) { - injections.push({ - name: "viewof " + injection.imported.name, - alias: "viewof " + injection.local.name - }); - } else if (injection.mutable) { - injections.push({ - name: "mutable " + injection.imported.name, - alias: "mutable " + injection.local.name - }); - } - injections.push({ - name: injection.imported.name, - alias: injection.local.name - }); - } - const importString = `import {${specifiers - .map(specifier => `${specifier.name} as ${specifier.alias}`) - .join(", ")}} ${ - hasInjections - ? `with {${injections - .map(injection => `${injection.name} as ${injection.alias}`) - .join(", ")}} ` - : `` - }from "${cell.body.source.value}"`; - - return { specifiers, hasInjections, injections, importString }; -}; - -const ESMImports = (moduleObject, resolveImportPath) => { +function ESMImports(moduleObject, resolveImportPath) { const importMap = new Map(); let importSrc = ""; let j = 0; @@ -136,9 +19,9 @@ const ESMImports = (moduleObject, resolveImportPath) => { if (importSrc.length) importSrc += "\n"; return { importSrc, importMap }; -}; +} -const ESMAttachments = (moduleObject, resolveFileAttachments) => { +function ESMAttachments(moduleObject, resolveFileAttachments) { const attachmentMapEntries = []; // loop over cells with fileAttachments for (const cell of moduleObject.cells) { @@ -154,9 +37,11 @@ const ESMAttachments = (moduleObject, resolveFileAttachments) => { attachmentMapEntries )}); main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name)));`; -}; +} + +function ESMVariables(moduleObject, importMap, params) { + const { defineImportMarkdown, observeViewofValues } = params; -const ESMVariables = (moduleObject, importMap) => { let childJ = 0; return moduleObject.cells .map(cell => { @@ -169,15 +54,17 @@ const ESMVariables = (moduleObject, importMap) => { injections, importString } = setupImportCell(cell); - // this will display extra names for viewof / mutable imports (for now?) - src += - ` main.variable(observer()).define( + + if (defineImportMarkdown) + src += + ` main.variable(observer()).define( null, ["md"], md => md\`~~~javascript ${importString} ~~~\` );` + "\n"; + // name imported notebook define functions const childName = `child${++childJ}`; src += ` const ${childName} = runtime.module(${ @@ -222,7 +109,9 @@ ${bodyText} if (cell.id && cell.id.type === "ViewExpression") { const reference = `"viewof ${cellName}"`; src += ` main.variable(observer(${reference})).define(${reference}, ${cellReferencesString}${cellFunction}); - main.variable(observer("${cellName}")).define("${cellName}", ["Generators", ${reference}], (G, _) => G.input(_));`; + main.variable(${ + observeViewofValues ? `observer("${cellName}")` : `null` + }).define("${cellName}", ["Generators", ${reference}], (G, _) => G.input(_));`; } else if (cell.id && cell.id.type === "MutableExpression") { const initialName = `"initial ${cellName}"`; const mutableName = `"mutable ${cellName}"`; @@ -238,20 +127,25 @@ ${bodyText} return src; }) .join("\n"); -}; -const createESModule = ( - moduleObject, - resolveImportPath, - resolveFileAttachments -) => { +} +function createESModule(moduleObject, params = {}) { + const { + resolveImportPath, + resolveFileAttachments, + defineImportMarkdown, + observeViewofValues + } = params; const { importSrc, importMap } = ESMImports(moduleObject, resolveImportPath); return `${importSrc}export default function define(runtime, observer) { const main = runtime.module(); ${ESMAttachments(moduleObject, resolveFileAttachments)} -${ESMVariables(moduleObject, importMap) || ""} +${ESMVariables(moduleObject, importMap, { + defineImportMarkdown, + observeViewofValues +}) || ""} return main; }`; -}; +} function defaultResolveImportPath(path) { const source = extractPath(path); @@ -265,18 +159,31 @@ export class Compiler { constructor(params = {}) { const { resolveFileAttachments = defaultResolveFileAttachments, - resolveImportPath = defaultResolveImportPath + resolveImportPath = defaultResolveImportPath, + defineImportMarkdown = true, + observeViewofValues = true } = params; this.resolveFileAttachments = resolveFileAttachments; this.resolveImportPath = resolveImportPath; + this.defineImportMarkdown = defineImportMarkdown; + this.observeViewofValues = observeViewofValues; } - module(text) { - const m1 = parseModule(text); - return createESModule( - m1, - this.resolveImportPath, - this.resolveFileAttachments - ); + module(text, params = {}) { + let m1 = parseModule(text); + + if (params.treeShake) + m1 = computeShakenCells( + m1, + params.treeShake.targets, + params.treeShake.stdlib + ); + + return createESModule(m1, { + resolveImportPath: this.resolveImportPath, + resolveFileAttachments: this.resolveFileAttachments, + defineImportMarkdown: this.defineImportMarkdown, + observeViewofValues: this.observeViewofValues + }); } notebook(obj) { const cells = obj.nodes.map(({ value }) => { @@ -286,8 +193,12 @@ export class Compiler { }); return createESModule( { cells }, - this.resolveImportPath, - this.resolveFileAttachments + { + resolveImportPath: this.resolveImportPath, + resolveFileAttachments: this.resolveFileAttachments, + defineImportMarkdown: this.defineImportMarkdown, + observeViewofValues: this.observeViewofValues + } ); } } diff --git a/src/interpreter.js b/src/interpreter.js index 4b0c3d1..4ffd3c4 100644 --- a/src/interpreter.js +++ b/src/interpreter.js @@ -1,124 +1,5 @@ -import { parseCell, parseModule, walk } from "@observablehq/parser"; -import { simple } from "acorn-walk"; -import { extractPath } from "./utils"; - -function setupImportCell(cell) { - const specifiers = []; - - if (cell.body.specifiers) - for (const specifier of cell.body.specifiers) { - if (specifier.view) { - specifiers.push({ - name: "viewof " + specifier.imported.name, - alias: "viewof " + specifier.local.name - }); - } else if (specifier.mutable) { - specifiers.push({ - name: "mutable " + specifier.imported.name, - alias: "mutable " + specifier.local.name - }); - } - specifiers.push({ - name: specifier.imported.name, - alias: specifier.local.name - }); - } - // If injections is undefined, do not derive! - const hasInjections = cell.body.injections !== undefined; - const injections = []; - if (hasInjections) - for (const injection of cell.body.injections) { - // This currently behaves like notebooks on observablehq.com - // Commenting out the if & else if blocks result in behavior like Example 3 here: https://observablehq.com/d/7ccad009e4d89969 - if (injection.view) { - injections.push({ - name: "viewof " + injection.imported.name, - alias: "viewof " + injection.local.name - }); - } else if (injection.mutable) { - injections.push({ - name: "mutable " + injection.imported.name, - alias: "mutable " + injection.local.name - }); - } - injections.push({ - name: injection.imported.name, - alias: injection.local.name - }); - } - const importString = `import {${specifiers - .map(specifier => `${specifier.name} as ${specifier.alias}`) - .join(", ")}} ${ - hasInjections - ? `with {${injections - .map(injection => `${injection.name} as ${injection.alias}`) - .join(", ")}} ` - : `` - }from "${cell.body.source.value}"`; - - return { specifiers, hasInjections, injections, importString }; -} - -function setupRegularCell(cell) { - let name = null; - if (cell.id && cell.id.name) name = cell.id.name; - else if (cell.id && cell.id.id && cell.id.id.name) name = cell.id.id.name; - let bodyText = cell.input.substring(cell.body.start, cell.body.end); - const cellReferences = (cell.references || []).map(ref => { - if (ref.type === "ViewExpression") { - return "viewof " + ref.id.name; - } else if (ref.type === "MutableExpression") { - return "mutable " + ref.id.name; - } else return ref.name; - }); - let $count = 0; - let indexShift = 0; - const references = (cell.references || []).map(ref => { - if (ref.type === "ViewExpression") { - const $string = "$" + $count; - $count++; - // replace "viewof X" in bodyText with "$($count)" - simple( - cell.body, - { - ViewExpression(node) { - const start = node.start - cell.body.start; - const end = node.end - cell.body.start; - bodyText = - bodyText.slice(0, start + indexShift) + - $string + - bodyText.slice(end + indexShift); - indexShift += $string.length - (end - start); - } - }, - walk - ); - return $string; - } else if (ref.type === "MutableExpression") { - const $string = "$" + $count; - const $stringValue = $string + ".value"; - $count++; - // replace "mutable Y" in bodyText with "$($count).value" - simple( - cell.body, - { - MutableExpression(node) { - const start = node.start - cell.body.start; - const end = node.end - cell.body.start; - bodyText = - bodyText.slice(0, start + indexShift) + - $stringValue + - bodyText.slice(end + indexShift); - indexShift += $stringValue.length - (end - start); - } - }, - walk - ); - return $string; - } else return ref.name; - }); - return { cellName: name, references, bodyText, cellReferences }; -} +import { parseCell, parseModule } from "@observablehq/parser"; +import { setupRegularCell, setupImportCell, extractPath } from "./utils"; function createRegularCellDefinition(cell) { const { cellName, references, bodyText, cellReferences } = setupRegularCell( diff --git a/src/tree-shake.js b/src/tree-shake.js new file mode 100644 index 0000000..a1cfad8 --- /dev/null +++ b/src/tree-shake.js @@ -0,0 +1,73 @@ +function names(cell) { + if (cell.body && cell.body.specifiers) + return cell.body.specifiers.map( + d => `${d.view ? "viewof " : d.mutable ? "mutable " : ""}${d.local.name}` + ); + + if (cell.id && cell.id.type && cell.id) { + if (cell.id.type === "ViewExpression") return [`viewof ${cell.id.id.name}`]; + if (cell.id.type === "MutableExpression") + return [`mutable ${cell.id.id.name}`]; + if (cell.id.name) return [cell.id.name]; + } + + return []; +} + +function references(cell, stdlibCells) { + if (cell.references) + return cell.references + .map(d => { + if (d.name) return d.name; + if (d.type === "ViewExpression") return `viewof ${d.id.name}`; + if (d.type === "MutableExpression") return `mutable ${d.id.name}`; + return null; + }) + .filter(d => !stdlibCells.has(d)); + + if (cell.body && cell.body.injections) + return cell.body.injections + .map( + d => + `${d.view ? "viewof " : d.mutable ? "mutable " : ""}${ + d.imported.name + }` + ) + .filter(d => !stdlibCells.has(d)); + + return []; +} + +function getCellRefs(module, stdlibCells) { + const cells = []; + for (const cell of module.cells) { + const ns = names(cell); + const refs = references(cell, stdlibCells); + if (!ns || !ns.length) continue; + for (const name of ns) { + cells.push([name, refs]); + if (name.startsWith("viewof ")) + cells.push([name.substring("viewof ".length), [name]]); + } + } + return new Map(cells); +} + +export function computeShakenCells(module, targets, stdlibCells) { + const cellRefs = getCellRefs(module, stdlibCells); + + const embed = new Set(); + const todo = targets.slice(); + while (todo.length) { + const d = todo.pop(); + embed.add(d); + if (!cellRefs.has(d)) throw Error(`${d} not a defined cell in module`); + const refs = cellRefs.get(d); + for (const ref of refs) if (!embed.has(ref)) todo.push(ref); + } + return { + cells: module.cells.filter( + cell => names(cell).filter(name => embed.has(name)).length + ) + }; +} diff --git a/src/utils.js b/src/utils.js index 395c446..af0e02f 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,3 +1,7 @@ +import { walk } from "@observablehq/parser"; + +import { simple } from "acorn-walk"; + export const extractPath = path => { let source = path; let m; @@ -13,3 +17,121 @@ export const extractPath = path => { source = source.slice(m[0].length); return source; }; + +export function setupImportCell(cell) { + const specifiers = []; + + if (cell.body.specifiers) + for (const specifier of cell.body.specifiers) { + if (specifier.view) { + specifiers.push({ + name: "viewof " + specifier.imported.name, + alias: "viewof " + specifier.local.name + }); + } else if (specifier.mutable) { + specifiers.push({ + name: "mutable " + specifier.imported.name, + alias: "mutable " + specifier.local.name + }); + } + specifiers.push({ + name: specifier.imported.name, + alias: specifier.local.name + }); + } + // If injections is undefined, do not derive! + const hasInjections = cell.body.injections !== undefined; + const injections = []; + if (hasInjections) + for (const injection of cell.body.injections) { + // This currently behaves like notebooks on observablehq.com + // Commenting out the if & else if blocks result in behavior like Example 3 here: https://observablehq.com/d/7ccad009e4d89969 + if (injection.view) { + injections.push({ + name: "viewof " + injection.imported.name, + alias: "viewof " + injection.local.name + }); + } else if (injection.mutable) { + injections.push({ + name: "mutable " + injection.imported.name, + alias: "mutable " + injection.local.name + }); + } + injections.push({ + name: injection.imported.name, + alias: injection.local.name + }); + } + const importString = `import {${specifiers + .map(specifier => `${specifier.name} as ${specifier.alias}`) + .join(", ")}} ${ + hasInjections + ? `with {${injections + .map(injection => `${injection.name} as ${injection.alias}`) + .join(", ")}} ` + : `` + }from "${cell.body.source.value}"`; + + return { specifiers, hasInjections, injections, importString }; +} + +export function setupRegularCell(cell) { + let name = null; + if (cell.id && cell.id.name) name = cell.id.name; + else if (cell.id && cell.id.id && cell.id.id.name) name = cell.id.id.name; + let bodyText = cell.input.substring(cell.body.start, cell.body.end); + const cellReferences = (cell.references || []).map(ref => { + if (ref.type === "ViewExpression") { + return "viewof " + ref.id.name; + } else if (ref.type === "MutableExpression") { + return "mutable " + ref.id.name; + } else return ref.name; + }); + let $count = 0; + let indexShift = 0; + const references = (cell.references || []).map(ref => { + if (ref.type === "ViewExpression") { + const $string = "$" + $count; + $count++; + // replace "viewof X" in bodyText with "$($count)" + simple( + cell.body, + { + ViewExpression(node) { + const start = node.start - cell.body.start; + const end = node.end - cell.body.start; + bodyText = + bodyText.slice(0, start + indexShift) + + $string + + bodyText.slice(end + indexShift); + indexShift += $string.length - (end - start); + } + }, + walk + ); + return $string; + } else if (ref.type === "MutableExpression") { + const $string = "$" + $count; + const $stringValue = $string + ".value"; + $count++; + // replace "mutable Y" in bodyText with "$($count).value" + simple( + cell.body, + { + MutableExpression(node) { + const start = node.start - cell.body.start; + const end = node.end - cell.body.start; + bodyText = + bodyText.slice(0, start + indexShift) + + $stringValue + + bodyText.slice(end + indexShift); + indexShift += $stringValue.length - (end - start); + } + }, + walk + ); + return $string; + } else return ref.name; + }); + return { cellName: name, references, bodyText, cellReferences }; +} From 600c85bbac8119a1329f366615ffd686c260c987 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 15 Apr 2021 09:56:34 -0700 Subject: [PATCH 04/20] 0.6.0-alpha.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 332c088..b5605d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.0", + "version": "0.6.0-alpha.1", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 5cafa8eb18c3730010f52a15dd5b37a7c924465d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Fri, 16 Apr 2021 17:48:46 -0700 Subject: [PATCH 05/20] re add async/generator functions. --- src/interpreter.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/interpreter.js b/src/interpreter.js index 4ffd3c4..ecba90b 100644 --- a/src/interpreter.js +++ b/src/interpreter.js @@ -1,6 +1,11 @@ import { parseCell, parseModule } from "@observablehq/parser"; import { setupRegularCell, setupImportCell, extractPath } from "./utils"; +const AsyncFunction = Object.getPrototypeOf(async function() {}).constructor; +const GeneratorFunction = Object.getPrototypeOf(function*() {}).constructor; +const AsyncGeneratorFunction = Object.getPrototypeOf(async function*() {}) + .constructor; + function createRegularCellDefinition(cell) { const { cellName, references, bodyText, cellReferences } = setupRegularCell( cell @@ -132,7 +137,7 @@ export class Interpreter { return [ module .variable(observer(reference)) - .define(reference, cellReferences, cellFunction), + .define(reference, cellReferences, cellFunction.bind(this)), module .variable(this.observeViewofValues ? observer(cellName) : null) .define(cellName, ["Generators", reference], (G, _) => G.input(_)) @@ -155,7 +160,7 @@ export class Interpreter { return [ module .variable(observer(cellName)) - .define(cellName, cellReferences, cellFunction) + .define(cellName, cellReferences, cellFunction.bind(this)) ]; } } From d3c2e30ec22a0f9bb830c5c2b50132b4f49165e9 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Fri, 16 Apr 2021 17:49:56 -0700 Subject: [PATCH 06/20] v0.6.0-alpha.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b5605d7..2eddec8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.1", + "version": "0.6.0-alpha.2", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 7cee69471340481e519daa49126bf7a45c2808ed Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 18 Apr 2021 22:16:20 -0700 Subject: [PATCH 07/20] Documentation, tests, bug fixes. --- .github/workflows/pull_request.yml | 29 -- README.md | 259 +++------------- src/compiler.js | 26 +- src/interpreter.js | 8 +- test/compileESModule-test.js | 286 ------------------ test/compiler-test.js | 462 +++++++++++++++++++++++------ test/interpreter-test.js | 303 +++++++++++++++++++ 7 files changed, 745 insertions(+), 628 deletions(-) delete mode 100644 .github/workflows/pull_request.yml delete mode 100644 test/compileESModule-test.js create mode 100644 test/interpreter-test.js diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index 4be180a..0000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Pull Request Demos - -on: - pull_request: - types: [opened, reopened, synchronize] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-node@v1.1.0 - - run: npm install - - run: npm run build - - run: npm install -g surge - - run: surge --domain unofficial-observablehq-compiler-demo-${{ github.event.number }}.surge.sh --project . - env: - SURGE_TOKEN: ${{ secrets.SURGE_TOKEN }} - - uses: actions/github-script@0.2.0 - with: - github-token: ${{github.token}} - script: | - await github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'New demo site for this PR deployed to [unofficial-observablehq-compiler-demo-${{ github.event.number }}.surge.sh](https://unofficial-observablehq-compiler-demo-${{ github.event.number }}.surge.sh) !' - }) diff --git a/README.md b/README.md index 57d93b3..fc826be 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,20 @@ -# @alex.garcia/unofficial-observablehq-compiler [![CircleCI](https://circleci.com/gh/asg017/unofficial-observablehq-compiler.svg?style=svg)](https://circleci.com/gh/asg017/unofficial-observablehq-compiler) +# @alex.garcia/unofficial-observablehq-compiler -An unoffical compiler for Observable notebooks (glue between the Observable parser and runtime) +An unoffical compiler _and_ interpreter for Observable notebooks (the glue between the Observable parser and runtime) -This compiler will compile "observable syntax" into "javascript syntax". -For example - +This library has two parts: The Interpreter and the Compiler. The Interpreter will interpret "Observable syntax" into "javascript syntax" live in a javascript environment. For example: ```javascript -import compiler from "@alex.garcia/unofficial-observablehq-compiler"; +import { Intepreter } from "@alex.garcia/unofficial-observablehq-compiler"; import { Inspector, Runtime } from "@observablehq/runtime"; -const compile = new compiler.Compiler(); +async function main() { + const runtime = new Runtime(); + const main = runtime.module(); + const observer = Inspector.into(document.body); + const interpret = new Intepreter({ module: main, observer }); -compile.module(` + await interpret.module(` import {text} from '@jashkenas/inputs' viewof name = text({ @@ -21,240 +24,60 @@ viewof name = text({ md\`Hello **\${name}**, it's nice to meet you!\` -`).then(define => { - const runtime = new Runtime(); - - const module = runtime.module(define, Inpsector.into(document.body)); -}); +`); +} +main(); ``` +The Compiler will compile "Observable syntax" into javascript source code (as an ES module). + For more live examples and functionality, take a look at the [announcement notebook](https://observablehq.com/d/74f872c4fde62e35) and this [test page](https://github.com/asg017/unofficial-observablehq-compiler/blob/master/test/test.html). ## API Reference -### Compiler - -# new Compiler(resolve = defaultResolver, fileAttachmentsResolve = name => name, resolvePath = defaultResolvePath) [<>](https://github.com/asg017/unofficial-observablehq-compiler/blob/master/src/compiler.js#L119 "Source") - -Returns a new compiler. `resolve` is an optional function that, given a `path` -string, will resolve a new define function for a new module. This is used when -the compiler comes across an import statement - for example: - -```javascript -import {chart} from "@d3/bar-chart" -``` - -In this case, `resolve` gets called with `path="@d3/bar-chart"`. The `defaultResolver` -function will lookup the given path on observablehq.com and return the define -function to define that notebook. - -For example, if you have your own set of notebooks on some other server, you -could use something like: - -```javascript -const resolve = path => - import(`other.server.com/notebooks/${path}.js`).then( - module => module.default - ); - -const compile = new Compiler(resolve); -``` - -`fileAttachmentsResolve` is an optional function from strings to URLs which is used as a resolve function in the standard library's FileAttachments function. For example, if you wanted to reference `example.com/my_file.png` in a cell which reads: - -```javascript -await FileAttachment("my_file.png").url(); -``` - -Then you could compile this cell with: - -```javascript -const fileAttachmentsResolve = name => `example.com/${name}`; - -const compile = new Compiler(, fileAttachmentsResolve); -``` - -By default, `fileAtachmentsResolve` simply returns the same string, so you would have to use valid absolute or relative URLs in your `FileAttachment`s. - -`resolvePath` is an optional function from strings to URLs which is used to turn the strings in `import` cells to URLs in [`compile.moduleToESModule`](#compile_moduleToESModule) and [`compile.notebookToESModule`](#compile_notebookToESModule). For instance, if those functions encounter this cell: -```javascript -import {chart} from "@d3/bar-chart" -``` -then `resolvePath` is called with `path="@d3/bar-chart"` and the resulting URL is included in the static `import` statements at the beginning of the generated ES module source. - -#compile.module(contents) - -Returns a define function. `contents` is a string that defines a "module", which -is a list of "cells" (both defintions from [@observablehq/parser](https://github.com/observablehq/parser)). -It must be compatible with [`parseModule`](https://github.com/observablehq/parser#parseModule). This fetches all imports so it is asynchronous. - -For example: - -```javascript -const define = await compile.module(`a = 1 -b = 2 -c = a + b`); -``` - -You can now use `define` with the Observable [runtime](https://github.com/observablehq/runtime): - -```javascript -const runtime = new Runtime(); -const main = runtime.module(define, Inspector.into(document.body)); -``` - -#compile.notebook(object) - -Returns a define function. `object` is a "notebook JSON object" as used by the -ObservableHQ notebook app to display notebooks. Such JSON files are served by -the API endpoint at `https://api.observablehq.com/document/:slug` (see the -[`observable-client`](https://github.com/mootari/observable-client) for a -convenient way to authenticate and make requests). - -`compile.notebook` requires that `object` has a field named `"nodes"` -consisting of an array of cell objects. Each of the cell objects must have a -field `"value"` consisting of a string with the source code for that cell. - -The notebook JSON objects also ordinarily contain some other metadata fields, -e.g. `"id"`, `"slug"`, `"owner"`, etc. which are currently ignored by the -compiler. Similarly, the cell objects in `"nodes"` ordinarily contain `"id"` and -`"pinned"` fields which are also unused here. - -This fetches all imports so it is asynchronous. +### new Interpreter(_params_) -For example: +`Interpreter` is a class that encompasses all logic to interpret Observable js code. _params_ is an optional object with the following allowed configuration: -```javascript -const define = await compile.notebook({ - nodes: [{ value: "a = 1" }, { value: "b = 2" }, { value: "c = a + b" }] -}); -``` - -You can now use `define` with the Observable [runtime](https://github.com/observablehq/runtime): - -```javascript -const runtime = new Runtime(); -const main = runtime.module(define, Inspector.into(document.body)); -``` - -#compile.cell(contents) - -Returns an object that has `define` and `redefine` functions that would define or redefine variables in the given cell to a specified module. `contents` is input for the [`parseCell`](https://github.com/observablehq/parser#parseCell) function. If the cell is not an ImportDeclaration, then the `redefine` functions can be used to redefine previously existing variables in a module. This is an asynchronous function because if the cell is an import, the imported notebook is fetched. - -```javascript -let define, redefine; - -define = await compile.module(`a = 1; -b = 2; - -c = a + b`); +- `resolveImportPath`: An async function that, given an import path, resolves to the `define` defintion for that "remote" notebook. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `https://api.observablehq.com/@d3/bar-chart.js?v=3` +- `resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. +- `defineImportMarkdown`: A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. +- `observeViewofValues`: A boolean, whether to pass in an _observer_ for the value of viewof cells. +- `module`: A default Observable runtime [module](https://github.com/observablehq/runtime#modules) that will be used in operations such as `.cell` and `.module`, if no other module is passed in. +- `observer`: A default Observable runtime [observer](https://github.com/observablehq/runtime#observer) that will be used in operations such as `.cell` and `.module`, if no other observer is passed in. -const runtime = new Runtime(); -const main = runtime.module(define, Inspector.into(document.body)); +Keep in mind, there is no sandboxing done, so it has the same security implications as `eval()`. -await main.value("a") // 1 +interpret.**cell**(_source_ [, *module*, *observer*]) -{define, redefine} = await compile.cell(`a = 20`); +Parses the given string _source_ with the Observable parser [`.parseCell()`](https://github.com/observablehq/parser#parseCell) and interprets the source, passing it and the _observer_ along to the Observable _module_. Returns a Promise that resolves to an array of runtime [variables](https://github.com/observablehq/runtime#variables) that were defined when interpreting the source. More than one variable can be defined with a single cell, like with `viewof`, `mutable`, and `import` cells. _source_ can also be a pre-parsed [cell](https://github.com/observablehq/parser#cell) instead of source code. -redefine(main); +interpret.**module**(_source_ [, *module*, *observer*]) -await main.value("a"); // 20 -await main.value("c"); // 22 +Parses the given string _source_ with the Observable parser [`.parseModule()`](https://github.com/observablehq/parser#parseModule) and interprets the source, passing it and the _observer_ along to the Observable _module_. Returns a Promise that resolves to an array of an array of runtime [variables](https://github.com/observablehq/runtime#variables) that were defined when interpreting the source. _source_ can also be a pre-parsed [program](https://github.com/observablehq/parser#program) instead of source code. -define(main); // would throw an error, since a is already defined in main +interpret.**notebook**(_source_ [, *module*, *observer*]) -{define} = await compile.cell(`x = 2`); -define(main); -{define} = await compile.cell(`y = x * 4`); -define(main); +TODO -await main.value("y") // 8 +new **Compiler**(_params_) -``` - -Keep in mind, if you want to use `define` from `compile.cell`, you'll have to provide an `observer` function, which will most likely be the same observer that was used when defining the module. For example: - -```javascript - -let define, redefine; - -define = await compile.module(`a = 1; -b = 2;`); - -const runtime = new Runtime(); -const observer = Inspector.into(document.body); -const main = runtime.module(define, observer); - -{define} = await compile.cell(`c = a + b`); - -define(main, observer); - -``` +`Compiler` is a class that encompasses all logic to compile Observable javascript code into vanilla Javascript code, as an ES module. _params_ is an optional object with the following allowed configuration: -Since `redefine` is done on a module level, an observer is not required. +- `resolveImportPath`: A function that, given an import path, returns a URL to where the notebook's is defined. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `"https://api.observablehq.com/@d3/bar-chart.js?v=3"`. +- `resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. +- `defineImportMarkdown` - A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. +- `observeViewofValues` - A boolean, whether or not to pass in the `observer` function for viewof value cells. Defaults true. +- `observeMutableValues` - A boolean, whether or not to pass in the `observer` function for mutable value cells. Defaults true. -#compile.moduleToESModule(contents) +compile.**module**(_source_) -Returns a string containing the source code of an ES module. This ES module is compiled from the Observable runtime module in the string `contents`. +TODO -For example: +compile.**notebook**(_source_) -```javascript -const src = compile.moduleToESModule(`a = 1 -b = 2 -c = a + b`); -``` - -Now `src` contains the following: - -```javascript -export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer("a")).define("a", function(){return( -1 -)}); - main.variable(observer("b")).define("b", function(){return( -2 -)}); - main.variable(observer("c")).define("c", ["a","b"], function(a,b){return( -a + b -)}); - return main; -} -``` - -#compile.notebookToESModule(object) - -Returns a string containing the source code of an ES module. This ES module is compiled from the Observable runtime module in the notebok object `object`. (See [compile.notebook](#compile_notebook)). - -For example: - -```javascript -const src = compile.notebookToESModule({ - nodes: [{ value: "a = 1" }, { value: "b = 2" }, { value: "c = a + b" }] -}); -``` - -Now `src` contains the following: - -```javascript -export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer("a")).define("a", function(){return( -1 -)}); - main.variable(observer("b")).define("b", function(){return( -2 -)}); - main.variable(observer("c")).define("c", ["a","b"], function(a,b){return( -a + b -)}); - return main; -} -``` +TODO ## License diff --git a/src/compiler.js b/src/compiler.js index 1439ff5..4a30910 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -40,7 +40,11 @@ function ESMAttachments(moduleObject, resolveFileAttachments) { } function ESMVariables(moduleObject, importMap, params) { - const { defineImportMarkdown, observeViewofValues } = params; + const { + defineImportMarkdown, + observeViewofValues, + observeMutableValues + } = params; let childJ = 0; return moduleObject.cells @@ -117,7 +121,9 @@ ${bodyText} const mutableName = `"mutable ${cellName}"`; src += ` main.define(${initialName}, ${cellReferencesString}${cellFunction}); main.variable(observer(${mutableName})).define(${mutableName}, ["Mutable", ${initialName}], (M, _) => new M(_)); - main.variable(observer("${cellName}")).define("${cellName}", [${mutableName}], _ => _.generator);`; + main.variable(${ + observeMutableValues ? `observer("${cellName}")` : `null` + }).define("${cellName}", [${mutableName}], _ => _.generator);`; } else { src += ` main.variable(observer(${cellNameString})).define(${ cellName ? cellNameString + ", " : "" @@ -133,7 +139,8 @@ function createESModule(moduleObject, params = {}) { resolveImportPath, resolveFileAttachments, defineImportMarkdown, - observeViewofValues + observeViewofValues, + observeMutableValues } = params; const { importSrc, importMap } = ESMImports(moduleObject, resolveImportPath); return `${importSrc}export default function define(runtime, observer) { @@ -141,7 +148,8 @@ function createESModule(moduleObject, params = {}) { ${ESMAttachments(moduleObject, resolveFileAttachments)} ${ESMVariables(moduleObject, importMap, { defineImportMarkdown, - observeViewofValues + observeViewofValues, + observeMutableValues }) || ""} return main; }`; @@ -161,12 +169,14 @@ export class Compiler { resolveFileAttachments = defaultResolveFileAttachments, resolveImportPath = defaultResolveImportPath, defineImportMarkdown = true, - observeViewofValues = true + observeViewofValues = true, + observeMutableValues = true } = params; this.resolveFileAttachments = resolveFileAttachments; this.resolveImportPath = resolveImportPath; this.defineImportMarkdown = defineImportMarkdown; this.observeViewofValues = observeViewofValues; + this.observeMutableValues = observeMutableValues; } module(text, params = {}) { let m1 = parseModule(text); @@ -182,7 +192,8 @@ export class Compiler { resolveImportPath: this.resolveImportPath, resolveFileAttachments: this.resolveFileAttachments, defineImportMarkdown: this.defineImportMarkdown, - observeViewofValues: this.observeViewofValues + observeViewofValues: this.observeViewofValues, + observeMutableValues: this.observeMutableValues }); } notebook(obj) { @@ -197,7 +208,8 @@ export class Compiler { resolveImportPath: this.resolveImportPath, resolveFileAttachments: this.resolveFileAttachments, defineImportMarkdown: this.defineImportMarkdown, - observeViewofValues: this.observeViewofValues + observeViewofValues: this.observeViewofValues, + observeMutableValues: this.observeMutableValues } ); } diff --git a/src/interpreter.js b/src/interpreter.js index ecba90b..66dab14 100644 --- a/src/interpreter.js +++ b/src/interpreter.js @@ -50,7 +50,8 @@ export class Interpreter { resolveImportPath = defaultResolveImportPath, resolveFileAttachments = defaultResolveFileAttachments, defineImportMarkdown = true, - observeViewofValues = true + observeViewofValues = true, + observeMutableValues = true } = params; // can't be this.module bc of async module(). @@ -62,6 +63,7 @@ export class Interpreter { this.resolveFileAttachments = resolveFileAttachments; this.defineImportMarkdown = defineImportMarkdown; this.observeViewofValues = observeViewofValues; + this.observeMutableValues = observeMutableValues; } async module(input, module, observer) { @@ -69,6 +71,7 @@ export class Interpreter { observer = observer || this.defaultObserver; if (!module) throw Error("No module provided."); + if (!observer) throw Error("No observer provided."); const parsedModule = parseModule(input); const cellPromises = []; @@ -84,6 +87,7 @@ export class Interpreter { observer = observer || this.defaultObserver; if (!module) throw Error("No module provided."); + if (!observer) throw Error("No observer provided."); let cell; if (typeof input === "string") { @@ -153,7 +157,7 @@ export class Interpreter { .variable(observer(mutableName)) .define(mutableName, ["Mutable", initialName], (M, _) => new M(_)), module - .variable(observer(cellName)) + .variable(this.observeMutableValues ? observer(cellName) : null) .define(cellName, [mutableName], _ => _.generator) ]; } else { diff --git a/test/compileESModule-test.js b/test/compileESModule-test.js deleted file mode 100644 index b1973cf..0000000 --- a/test/compileESModule-test.js +++ /dev/null @@ -1,286 +0,0 @@ -const test = require("tape"); -const compiler = require("../dist/index"); - -test("ES module: simple", async t => { - const compile = new compiler.Compiler(); - const src = compile.moduleToESModule(` -a = 1 - -b = 2 - -c = a + b - -d = { - yield 1; - yield 2; - yield 3; -} - -{ - await d; -} - -viewof e = { - let output = {}; - let listeners = []; - output.value = 10; - output.addEventListener = (listener) => listeners.push(listener);; - output.removeEventListener = (listener) => { - listeners = listeners.filter(l => l !== listener); - }; - return output; -} - `); - t.equal(src, `export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer("a")).define("a", function(){return( -1 -)}); - main.variable(observer("b")).define("b", function(){return( -2 -)}); - main.variable(observer("c")).define("c", ["a","b"], function(a,b){return( -a + b -)}); - main.variable(observer("d")).define("d", function*() -{ - yield 1; - yield 2; - yield 3; -} -); - main.variable(observer()).define(["d"], async function(d) -{ - await d; -} -); - main.variable(observer("viewof e")).define("viewof e", function() -{ - let output = {}; - let listeners = []; - output.value = 10; - output.addEventListener = (listener) => listeners.push(listener);; - output.removeEventListener = (listener) => { - listeners = listeners.filter(l => l !== listener); - }; - return output; -} -); - main.variable(observer("e")).define("e", ["Generators", "viewof e"], (G, _) => G.input(_)); - return main; -}`); - - t.end(); -}); - -test("ES module: imports", async t => { - const compile = new compiler.Compiler(); - const src = compile.moduleToESModule(`import {a} from "b" -b = { - return 3*a -} -import {c as bc} from "b" -import {d} with {b as cb} from "c" -`); - - t.equal(src, `import define1 from "https://api.observablehq.com/b.js?v=3"; -import define2 from "https://api.observablehq.com/c.js?v=3"; - -export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {a as a} from "b" -~~~\` - ); - const child1 = runtime.module(define1); - main.import("a", "a", child1); - main.variable(observer("b")).define("b", ["a"], function(a) -{ - return 3*a -} -); - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {c as bc} from "b" -~~~\` - ); - const child2 = runtime.module(define1); - main.import("c", "bc", child2); - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {d as d} with {b as cb} from "c" -~~~\` - ); - const child3 = runtime.module(define2).derive([{"name":"b","alias":"cb"}], main); - main.import("d", "d", child3); - return main; -}`); - - t.end(); -}); - - -test("ES module: custom resolvePath function", async t => { - const resolvePath = name => `https://gist.github.com/${name}`; - const compile = new compiler.Compiler(undefined, undefined, resolvePath); - const src = compile.moduleToESModule(`import {a} from "b" -b = { - return 3*a -} -import {c as bc} from "b" -import {d} with {b as cb} from "c" -`); - - t.equal(src, `import define1 from "https://gist.github.com/b"; -import define2 from "https://gist.github.com/c"; - -export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {a as a} from "b" -~~~\` - ); - const child1 = runtime.module(define1); - main.import("a", "a", child1); - main.variable(observer("b")).define("b", ["a"], function(a) -{ - return 3*a -} -); - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {c as bc} from "b" -~~~\` - ); - const child2 = runtime.module(define1); - main.import("c", "bc", child2); - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {d as d} with {b as cb} from "c" -~~~\` - ); - const child3 = runtime.module(define2).derive([{"name":"b","alias":"cb"}], main); - main.import("d", "d", child3); - return main; -}`); - - t.end(); -}); - -test("ES module: viewof + mutable", async t => { - const compile = new compiler.Compiler(); - const src = compile.moduleToESModule(`viewof a = { - const div = html\`\`; - div.value = 3; - return div; -} -mutable b = 3 -{ - return b*b -} -d = { - mutable b++; - return a + b; -} -import {viewof v as w, mutable m} from "notebook"`); - - t.equal(src, `import define1 from "https://api.observablehq.com/notebook.js?v=3"; - -export default function define(runtime, observer) { - const main = runtime.module(); - - main.variable(observer("viewof a")).define("viewof a", ["html"], function(html) -{ - const div = html\`\`; - div.value = 3; - return div; -} -); - main.variable(observer("a")).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); - main.define("initial b", function(){return( -3 -)}); - main.variable(observer("mutable b")).define("mutable b", ["Mutable", "initial b"], (M, _) => new M(_)); - main.variable(observer("b")).define("b", ["mutable b"], _ => _.generator); - main.variable(observer()).define(["b"], function(b) -{ - return b*b -} -); - main.variable(observer("d")).define("d", ["mutable b","a","b"], function($0,a,b) -{ - $0.value++; - return a + b; -} -); - main.variable(observer()).define( - null, - ["md"], - md => md\`~~~javascript -import {viewof v as viewof w, v as w, mutable m as mutable m, m as m} from "notebook" -~~~\` - ); - const child1 = runtime.module(define1); - main.import("viewof v", "viewof w", child1); - main.import("v", "w", child1); - main.import("mutable m", "mutable m", child1); - main.import("m", "m", child1); - return main; -}`); - - t.end(); -}); - -test("ES module: FileAttachment", async t => { - const compile = new compiler.Compiler(); - const src = compile.moduleToESModule(`md\`Here's a cell with a file attachment! \``); - - t.equal(src, `export default function define(runtime, observer) { - const main = runtime.module(); - const fileAttachments = new Map([["image.png","image.png"]]); - main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); - main.variable(observer()).define(["md","FileAttachment"], async function(md,FileAttachment){return( -md\`Here's a cell with a file attachment! \` -)}); - return main; -}`); - - t.end(); -}); - -test("ES module: custom fileAttachmentsResolve", async t => { - const fileAttachmentsResolve = name => `https://example.com/${name}`; - const compile = new compiler.Compiler(undefined, fileAttachmentsResolve); - const src = compile.moduleToESModule(`md\`Here's a cell with a file attachment! \``); - - t.equal(src, `export default function define(runtime, observer) { - const main = runtime.module(); - const fileAttachments = new Map([["image.png","https://example.com/image.png"]]); - main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); - main.variable(observer()).define(["md","FileAttachment"], async function(md,FileAttachment){return( -md\`Here's a cell with a file attachment! \` -)}); - return main; -}`); - - t.end(); -}); - - diff --git a/test/compiler-test.js b/test/compiler-test.js index 6c14f7a..690ae6c 100644 --- a/test/compiler-test.js +++ b/test/compiler-test.js @@ -1,91 +1,381 @@ const test = require("tape"); -const runtime = require("@observablehq/runtime"); -const compiler = require("../dist/index"); - -test("compiler", async t => { - const rt = new runtime.Runtime(); - const compile = new compiler.Compiler(); - const define = await compile.module(` -a = 1 - -b = 2 - -c = a + b - -d = { - yield 1; - yield 2; - yield 3; -} - -viewof e = { - let output = {}; - let listeners = []; - output.value = 10; - output.addEventListener = (listener) => listeners.push(listener);; - output.removeEventListener = (listener) => { - listeners = listeners.filter(l => l !== listener); - }; - return output; -} - `); - const main = rt.module(define); - await rt._compute(); - - t.equal(await main.value("a"), 1); - t.equal(await main.value("b"), 2); - t.equal(await main.value("c"), 3); - - t.equal(await main.value("d"), 1); - t.equal(await main.value("d"), 2); - t.equal(await main.value("d"), 3); - - t.equal(await main.value("e"), 10); - t.deepEqual(Object.keys(await main.value("viewof e")), [ - "value", - "addEventListener", - "removeEventListener" - ]); - - const { redefine: aRedefine } = await compile.cell(`a = 10`); - aRedefine(main); - await rt._compute(); - t.equal(await main.value("a"), 10); - t.equal(await main.value("c"), 12); - - const { define: xDefine } = await compile.cell(`x = y - 1`); - xDefine(main, () => true); - await rt._compute(); - - try { - await main.value("x"); - t.fail(); - } catch (error) { - t.equal(error.constructor, runtime.RuntimeError); +const { Runtime } = require("@observablehq/runtime"); +const { Compiler } = require("../dist/index"); + +test("Compiler: simple", async t => { + const compile = new Compiler(); + const runtime = new Runtime(); + + const src = compile.module(`a = 1; b = 2; c = a + b; + d = {yield 1; yield 2; yield 3;}; e = await Promise.resolve(40);`); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("a")).define("a", function(){return( +1 +)}); + main.variable(observer("b")).define("b", function(){return( +2 +)}); + main.variable(observer("c")).define("c", ["a","b"], function(a,b){return( +a + b +)}); + main.variable(observer("d")).define("d", function*() +{yield 1; yield 2; yield 3;} +); + main.variable(observer("e")).define("e", async function(){return( +await Promise.resolve(40) +)}); + return main; +}` + ); + const define = eval(`(${src.substring("export default ".length)})`); + const main = runtime.module(define); + + t.equals(await main.value("a"), 1); + t.equals(await main.value("b"), 2); + t.equals(await main.value("c"), 3); + t.equals(await main.value("d"), 1); + t.equals(await main.value("e"), 40); + + t.end(); +}); + +test("Compiler: viewof cells", async t => { + const compile = new Compiler(); + const runtime = new Runtime(); + + const src = compile.module( + `viewof a = ({name: 'alex', value: 101, addEventListener: () => {} })` + ); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("viewof a")).define("viewof a", function(){return( +{name: 'alex', value: 101, addEventListener: () => {} } +)}); + main.variable(observer("a")).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); + return main; +}` + ); + const define = eval(`(${src.substring("export default ".length)})`); + const main = runtime.module(define); + + t.equals(await main.value("a"), 101); + + t.end(); +}); + +test("Compiler: mutable cells", async t => { + const compile = new Compiler(); + const runtime = new Runtime(); + + const src = compile.module(`mutable a = 200; _ = (mutable a = 202)`); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.define("initial a", function(){return( +200 +)}); + main.variable(observer("mutable a")).define("mutable a", ["Mutable", "initial a"], (M, _) => new M(_)); + main.variable(observer("a")).define("a", ["mutable a"], _ => _.generator); + main.variable(observer("_")).define("_", ["mutable a"], function($0){return( +$0.value = 202 +)}); + return main; +}` + ); + const define = eval(`(${src.substring("export default ".length)})`); + const main = runtime.module(define); + + t.equals(await main.value("a"), 200); + t.equals(await main.value("_"), 202); + t.equals(await main.value("a"), 202); + + t.end(); +}); + +test("Compiler: import cells", async t => { + const compile = new Compiler({ + resolveImportPath: d => `https://example.com/${d}` + }); + + const src = compile.module(`import {a as A, b as B, c as C} from "alpha";`); + + t.equal( + src, + `import define1 from "https://example.com/alpha"; + +export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer()).define( + null, + ["md"], + md => md\`~~~javascript +import {a as A, b as B, c as C} from "alpha" +~~~\` + ); + const child1 = runtime.module(define1); + main.import("a", "A", child1); + main.import("b", "B", child1); + main.import("c", "C", child1); + return main; +}` + ); + + t.end(); +}); + +// defineImportMarkdown +test("Compiler: defineImportMarkdown", async t => { + let compile = new Compiler({ defineImportMarkdown: true }); + let src = compile.module(`import {a} from "whatever";`); + + t.equal( + src, + `import define1 from "https://api.observablehq.com/whatever.js?v=3"; + +export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer()).define( + null, + ["md"], + md => md\`~~~javascript +import {a as a} from "whatever" +~~~\` + ); + const child1 = runtime.module(define1); + main.import("a", "a", child1); + return main; +}` + ); + + compile = new Compiler({ defineImportMarkdown: false }); + src = compile.module(`import {a} from "whatever";`); + t.equal( + src, + `import define1 from "https://api.observablehq.com/whatever.js?v=3"; + +export default function define(runtime, observer) { + const main = runtime.module(); + + const child1 = runtime.module(define1); + main.import("a", "a", child1); + return main; +}` + ); + + t.end(); +}); + +// observeViewofValues +test("Compiler: observeViewofValues", async t => { + let compile = new Compiler({ observeViewofValues: true }); + let src = compile.module( + "viewof a = ({value: 100, addEventListener: () => {}, removeEventListener: () => {}})" + ); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("viewof a")).define("viewof a", function(){return( +{value: 100, addEventListener: () => {}, removeEventListener: () => {}} +)}); + main.variable(observer("a")).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); + return main; +}` + ); + + compile = new Compiler({ observeViewofValues: false }); + src = compile.module( + "viewof a = ({value: 100, addEventListener: () => {}, removeEventListener: () => {}})" + ); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("viewof a")).define("viewof a", function(){return( +{value: 100, addEventListener: () => {}, removeEventListener: () => {}} +)}); + main.variable(null).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); + return main; +}` + ); + + t.end(); +}); + +test("Compiler: observeMutableValues", async t => { + let compile = new Compiler({ observeMutableValues: true }); + let src = compile.module("mutable a = 0x100"); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.define("initial a", function(){return( +0x100 +)}); + main.variable(observer("mutable a")).define("mutable a", ["Mutable", "initial a"], (M, _) => new M(_)); + main.variable(observer("a")).define("a", ["mutable a"], _ => _.generator); + return main; +}` + ); + + compile = new Compiler({ observeMutableValues: false }); + src = compile.module("mutable a = 0x100"); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.define("initial a", function(){return( +0x100 +)}); + main.variable(observer("mutable a")).define("mutable a", ["Mutable", "initial a"], (M, _) => new M(_)); + main.variable(null).define("a", ["mutable a"], _ => _.generator); + return main; +}` + ); + + t.end(); +}); + +test("Compiler: resolveFileAttachments", async t => { + function resolveFileAttachments(name) { + return `https://example.com/${name}`; } + let compile = new Compiler({ resolveFileAttachments }); + let src = compile.module('a = FileAttachment("a")'); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + const fileAttachments = new Map([["a","https://example.com/a"]]); + main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); + main.variable(observer("a")).define("a", ["FileAttachment"], function(FileAttachment){return( +FileAttachment("a") +)}); + return main; +}` + ); + + t.end(); +}); + +test("Compiler: treeShake", async t => { + let src; + const initSrc = ` + viewof a = html\`\`; b = 2; c = a + b; + d = 2; e = 4; f = d + e; + height = c; + import {chart} with {height} from "@d3/bar-chart";`; + const compile = new Compiler({ defineImportMarkdown: false }); + + src = compile.module(initSrc, { + treeShake: { + targets: ["f"], + stdlib: new Set(["html"]) + } + }); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("d")).define("d", function(){return( +2 +)}); + main.variable(observer("e")).define("e", function(){return( +4 +)}); + main.variable(observer("f")).define("f", ["d","e"], function(d,e){return( +d + e +)}); + return main; +}` + ); + + src = compile.module(initSrc, { + treeShake: { + targets: ["f", "a"], + stdlib: new Set(["html"]) + } + }); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("viewof a")).define("viewof a", ["html"], function(html){return( +html\`\` +)}); + main.variable(observer("a")).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); + main.variable(observer("d")).define("d", function(){return( +2 +)}); + main.variable(observer("e")).define("e", function(){return( +4 +)}); + main.variable(observer("f")).define("f", ["d","e"], function(d,e){return( +d + e +)}); + return main; +}` + ); + + src = compile.module(initSrc, { + treeShake: { + targets: ["chart"], + stdlib: new Set(["html"]) + } + }); + + t.equal( + src, + `import define1 from "https://api.observablehq.com/@d3/bar-chart.js?v=3"; + +export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("viewof a")).define("viewof a", ["html"], function(html){return( +html\`\` +)}); + main.variable(observer("a")).define("a", ["Generators", "viewof a"], (G, _) => G.input(_)); + main.variable(observer("b")).define("b", function(){return( +2 +)}); + main.variable(observer("c")).define("c", ["a","b"], function(a,b){return( +a + b +)}); + main.variable(observer("height")).define("height", ["c"], function(c){return( +c +)}); + const child1 = runtime.module(define1).derive([{"name":"height","alias":"height"}], main); + main.import("chart", "chart", child1); + return main; +}` + ); - const { define: yDefine } = await compile.cell(`y = 101`); - yDefine(main, () => true); - await rt._compute(); - - t.equal(await main.value("y"), 101); - t.equal(await main.value("x"), 100); - - const { redefine: eRedefine } = await compile.cell(`viewof e = { - let output = {}; - let listeners = []; - output.value = 20; - output.addEventListener = (listener) => listeners.push(listener);; - output.removeEventListener = (listener) => { - listeners = listeners.filter(l => l !== listener); - }; - return output; - }`); - eRedefine(main); - await rt._compute(); - - t.equal(await main.value("e"), 20); - - rt.dispose(); t.end(); }); diff --git a/test/interpreter-test.js b/test/interpreter-test.js new file mode 100644 index 0000000..2577acf --- /dev/null +++ b/test/interpreter-test.js @@ -0,0 +1,303 @@ +const test = require("tape"); +const { Interpreter } = require("../dist/index"); +const { Runtime, RuntimeError } = require("@observablehq/runtime"); + +test("Interpreter: simple cells", async t => { + const interpret = new Interpreter(); + const runtime = new Runtime(); + const main = runtime.module(); + let observer = () => null; + + const a = await interpret.cell("a = 1", main, observer); + + t.equals(await main.value("a"), 1); + t.equals(a.length, 1); + t.equals(a[0]._module, main); + t.equals(a[0]._value, 1); + + await interpret.module("b = 2; c = a + b;", main, observer); + + t.equals(await main.value("b"), 2); + t.equals(await main.value("c"), 3); + + await interpret.module( + "d = {yield 1; yield 2; yield 3;}; e = await Promise.resolve(40);", + main, + observer + ); + + t.equals(await main.value("d"), 1); + t.equals(await main.value("e"), 40); + + try { + await main.value("x"); + t.fail(); + } catch (error) { + t.equal(error.constructor, RuntimeError); + } + + t.end(); +}); + +test("Interpreter: viewof cells", async t => { + const interpret = new Interpreter(); + const runtime = new Runtime(); + const main = runtime.module(); + let observer = () => null; + + const a = await interpret.cell( + "viewof a = ({name: 'alex', value: 101, addEventListener: () => {} })", + main, + observer + ); + + t.equals(await main.value("a"), 101); + t.equals((await main.value("viewof a")).name, "alex"); + t.equals(a.length, 2); + t.equals(a[0]._name, "viewof a"); + t.equals(a[1]._name, "a"); + + t.end(); +}); + +test("Interpreter: mutable cells", async t => { + const interpret = new Interpreter(); + const runtime = new Runtime(); + const main = runtime.module(); + let observer = () => null; + + const Mutable = await runtime._builtin.value("Mutable"); + + let a = await interpret.cell("mutable a = 200", main, observer); + + t.equals(await main.value("initial a"), 200); + t.equals((await main.value("mutable a")).constructor, Mutable); + t.equals(await main.value("a"), 200); + + t.equals(a.length, 3); + t.equals(a[0]._name, "initial a"); + t.equals(a[1]._name, "mutable a"); + t.equals(a[2]._name, "a"); + + a = await interpret.cell("_ = (mutable a = 202)", main, observer); + + t.equals(a.length, 1); + t.equals(a[0]._name, "_"); + t.equals(await main.value("_"), 202); + t.equals(await main.value("a"), 202); + + t.end(); +}); + +test("Interpreter: import cells", async t => { + function resolveImportPath(name) { + if (name === "alpha") + return function define(runtime, observer) { + const main = runtime.module(); + main.variable(observer("a")).define("a", function() { + return 100; + }); + main.variable(observer("b")).define("b", function() { + return 200; + }); + main.variable(observer("c")).define("c", ["a", "b"], function(a, b) { + return a + b; + }); + return main; + }; + if (name === "delta") + return function define(runtime, observer) { + const main = runtime.module(); + main.variable(observer("d")).define("d", function() { + return 1000; + }); + main.variable(observer("e")).define("e", function() { + return 2000; + }); + main.variable(observer("f")).define("f", ["d", "e"], function(d, e) { + return d - e; + }); + return main; + }; + } + + const interpret = new Interpreter({ resolveImportPath }); + let runtime = new Runtime(); + let main = runtime.module(); + let observer = () => null; + + const alpha = await interpret.cell( + `import {a as A, b as B, c as C} from "alpha";`, + main, + observer + ); + + t.equals(alpha.length, 4); + + // the first is the unnamed "markdown" import cell + t.equals(alpha[0]._name, null); + t.equals(alpha[1]._name, "A"); + t.equals(alpha[2]._name, "B"); + t.equals(alpha[3]._name, "C"); + t.equals(await main.value("A"), 100); + t.equals(await main.value("B"), 200); + t.equals(await main.value("C"), 300); + + runtime.dispose(); + + // fresh runtime/module for next stuff + runtime = new Runtime(); + main = runtime.module(); + + const program = await interpret.module( + ` + import {c as C} from "alpha"; + + import {f as F} with {C as d} from "delta"; + `, + main, + observer + ); + + t.equals(program.length, 2); + t.equals(program[0].length, 2); + t.equals(program[0][0]._name, null); + t.equals(program[0][1]._name, "C"); + t.equals(program[1].length, 2); + t.equals(program[1][0]._name, null); + t.equals(program[1][1]._name, "F"); + // a=100, b=200, C = a + b + t.equals(await main.value("C"), 300); + // d=300 (C), e=2000, f=d-e + t.equals(await main.value("F"), -1700); + + t.end(); +}); +test("Interpreter: defineImportMarkdown", async t => { + function resolveImportPath(name) { + return function define(runtime, observer) { + const main = runtime.module(); + main.variable(observer("a")).define("a", function() { + return 100; + }); + return main; + }; + } + + let interpret = new Interpreter({ + resolveImportPath, + defineImportMarkdown: true + }); + let runtime = new Runtime(); + let main = runtime.module(); + let observer = () => null; + + let a = await interpret.cell("import {a} from 'whatever';", main, observer); + + t.equals(a.length, 2); + t.equals(a[0]._name, null); + t.equals(a[0]._inputs[0]._name, "md"); + t.equals(a[1]._name, "a"); + + runtime.dispose(); + + interpret = new Interpreter({ + resolveImportPath, + defineImportMarkdown: false + }); + runtime = new Runtime(); + main = runtime.module(); + + a = await interpret.cell("import {a} from 'whatever';", main, observer); + + t.equals(a.length, 1); + t.equals(a[0]._name, "a"); + + t.end(); +}); + +test("Interpreter: observeViewofValues", async t => { + let aObserved = false; + let bObserved = false; + + let interpret = new Interpreter({ + observeViewofValues: true + }); + let runtime = new Runtime(); + let main = runtime.module(); + + function observerA(name) { + if (name === "a") aObserved = true; + } + + await interpret.cell( + "viewof a = ({value: 100, addEventListener: () => {}, removeEventListener: () => {}})", + main, + observerA + ); + + t.true(aObserved); + t.equals(await main.value("a"), 100); + runtime.dispose(); + + interpret = new Interpreter({ + observeViewofValues: false + }); + runtime = new Runtime(); + main = runtime.module(); + + function observerB(name) { + if (name === "b") bObserved = true; + } + + await interpret.cell( + "viewof b = ({value: 200, addEventListener: () => {}})", + main, + observerB + ); + + t.false(bObserved); + t.equals(await main.value("b"), 200); + + t.end(); +}); + +test("Interpreter: observeMutableValues", async t => { + let aObserved = false; + let bObserved = false; + + let interpret = new Interpreter({ + observeMutableValues: true + }); + let runtime = new Runtime(); + let main = runtime.module(); + + function observerA(name) { + if (name === "a") aObserved = true; + } + + await interpret.cell("mutable a = 0x100", main, observerA); + + t.true(aObserved); + t.equals(await main.value("a"), 0x100); + runtime.dispose(); + + interpret = new Interpreter({ + observeMutableValues: false + }); + runtime = new Runtime(); + main = runtime.module(); + + function observerB(name) { + if (name === "b") bObserved = true; + } + + await interpret.cell("mutable b = 0x200", main, observerB); + + t.false(bObserved); + t.equals(await main.value("b"), 0x200); + + t.end(); +}); + +// TODO: resolveFileAttachments test. From 2fff87830f1942f79ee603a7b53aebec2f778822 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 18 Apr 2021 22:19:55 -0700 Subject: [PATCH 08/20] v0.6.0-alpha.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2eddec8..3a816b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.2", + "version": "0.6.0-alpha.3", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 6f3f1c8f6ded4c9dc3ce085bcf7ea4b629f8128f Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 18 Apr 2021 22:34:47 -0700 Subject: [PATCH 09/20] Export @observablehq/parser. --- src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.js b/src/index.js index 05395bb..8a7b4f2 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,3 @@ export { Compiler } from "./compiler.js"; export { Interpreter } from "./interpreter.js"; +export * as parser from "@observablehq/parser"; From 72a4fb781f15b5dd7847afd31faf894a808544bc Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 18 Apr 2021 22:35:25 -0700 Subject: [PATCH 10/20] v0.6.0-alpha.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3a816b8..2646c24 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.3", + "version": "0.6.0-alpha.4", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 0fc6e5fab55863c65c38623da56318b8c9abae3b Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 19 Apr 2021 09:01:39 -0700 Subject: [PATCH 11/20] fix: Multiple of the same reference should appear only once. --- src/utils.js | 7 ++++++- test/compiler-test.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index af0e02f..1f6d7ba 100644 --- a/src/utils.js +++ b/src/utils.js @@ -133,5 +133,10 @@ export function setupRegularCell(cell) { return $string; } else return ref.name; }); - return { cellName: name, references, bodyText, cellReferences }; + return { + cellName: name, + references: Array.from(new Set(references)), + bodyText, + cellReferences: Array.from(new Set(cellReferences)) + }; } diff --git a/test/compiler-test.js b/test/compiler-test.js index 690ae6c..705c315 100644 --- a/test/compiler-test.js +++ b/test/compiler-test.js @@ -379,3 +379,32 @@ c t.end(); }); + +test("Compiler: multiple refs", async t => { + const compile = new Compiler(); + const runtime = new Runtime(); + + const src = compile.module(`a = 1; b = a,a,a,a;`); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + + main.variable(observer("a")).define("a", function(){return( +1 +)}); + main.variable(observer("b")).define("b", ["a"], function(a){return( +a,a,a,a +)}); + return main; +}` + ); + const define = eval(`(${src.substring("export default ".length)})`); + const main = runtime.module(define); + + t.equals(await main.value("a"), 1); + t.equals(await main.value("b"), 1); + + t.end(); +}); From 9e477a89b855efebc6f44bdc9e927b7613d50553 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 19 Apr 2021 09:02:30 -0700 Subject: [PATCH 12/20] v0.6.0-alpha.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2646c24..953bb8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.4", + "version": "0.6.0-alpha.5", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 569e4c79e5adf3783e89df22e4817a966879e3f4 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 22 Apr 2021 21:43:17 -0700 Subject: [PATCH 13/20] Relax reqs for treeshaking, rm stdlib parameter. --- src/compiler.js | 7 +------ src/tree-shake.js | 41 +++++++++++++++++++---------------------- test/compiler-test.js | 23 +++++++++++------------ 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/compiler.js b/src/compiler.js index 4a30910..a5f740f 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -181,12 +181,7 @@ export class Compiler { module(text, params = {}) { let m1 = parseModule(text); - if (params.treeShake) - m1 = computeShakenCells( - m1, - params.treeShake.targets, - params.treeShake.stdlib - ); + if (params.treeShake) m1 = computeShakenCells(m1, params.treeShake); return createESModule(m1, { resolveImportPath: this.resolveImportPath, diff --git a/src/tree-shake.js b/src/tree-shake.js index a1cfad8..6d71949 100644 --- a/src/tree-shake.js +++ b/src/tree-shake.js @@ -14,35 +14,29 @@ function names(cell) { return []; } -function references(cell, stdlibCells) { +function references(cell) { if (cell.references) - return cell.references - .map(d => { - if (d.name) return d.name; - if (d.type === "ViewExpression") return `viewof ${d.id.name}`; - if (d.type === "MutableExpression") return `mutable ${d.id.name}`; - return null; - }) - .filter(d => !stdlibCells.has(d)); + return cell.references.map(d => { + if (d.name) return d.name; + if (d.type === "ViewExpression") return `viewof ${d.id.name}`; + if (d.type === "MutableExpression") return `mutable ${d.id.name}`; + return null; + }); if (cell.body && cell.body.injections) - return cell.body.injections - .map( - d => - `${d.view ? "viewof " : d.mutable ? "mutable " : ""}${ - d.imported.name - }` - ) - .filter(d => !stdlibCells.has(d)); + return cell.body.injections.map( + d => + `${d.view ? "viewof " : d.mutable ? "mutable " : ""}${d.imported.name}` + ); return []; } -function getCellRefs(module, stdlibCells) { +function getCellRefs(module) { const cells = []; for (const cell of module.cells) { const ns = names(cell); - const refs = references(cell, stdlibCells); + const refs = references(cell); if (!ns || !ns.length) continue; for (const name of ns) { cells.push([name, refs]); @@ -53,15 +47,18 @@ function getCellRefs(module, stdlibCells) { return new Map(cells); } -export function computeShakenCells(module, targets, stdlibCells) { - const cellRefs = getCellRefs(module, stdlibCells); +export function computeShakenCells(module, targets) { + const cellRefs = getCellRefs(module); const embed = new Set(); const todo = targets.slice(); while (todo.length) { const d = todo.pop(); embed.add(d); - if (!cellRefs.has(d)) throw Error(`${d} not a defined cell in module`); + // happens when 1) d is an stdlib cell, 2) d doesnt have a defintion, + // or 3) d is in the window/global object. Let's be forgiving + // and let it happen + if (!cellRefs.has(d)) continue; const refs = cellRefs.get(d); for (const ref of refs) if (!embed.has(ref)) todo.push(ref); } diff --git a/test/compiler-test.js b/test/compiler-test.js index 705c315..0d2950b 100644 --- a/test/compiler-test.js +++ b/test/compiler-test.js @@ -291,10 +291,7 @@ test("Compiler: treeShake", async t => { const compile = new Compiler({ defineImportMarkdown: false }); src = compile.module(initSrc, { - treeShake: { - targets: ["f"], - stdlib: new Set(["html"]) - } + treeShake: ["f"], }); t.equal( @@ -316,10 +313,7 @@ d + e ); src = compile.module(initSrc, { - treeShake: { - targets: ["f", "a"], - stdlib: new Set(["html"]) - } + treeShake: ["f", "a"], }); t.equal( @@ -345,10 +339,7 @@ d + e ); src = compile.module(initSrc, { - treeShake: { - targets: ["chart"], - stdlib: new Set(["html"]) - } + treeShake: ["chart"], }); t.equal( @@ -377,6 +368,14 @@ c }` ); + // make sure tree shaking an unreference cell just returns a define identity function. + t.equals(compile.module(`a = 1; b = a/1;`, {treeShake:["c"]}), `export default function define(runtime, observer) { + const main = runtime.module(); + + + return main; +}`) + t.end(); }); From 43a8173039674d5aa8f271bf56a5483b29dea67e Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 22 Apr 2021 21:44:01 -0700 Subject: [PATCH 14/20] v0.6.0-alpha.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 953bb8f..144243b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.5", + "version": "0.6.0-alpha.6", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From f241ef94e0c7252f41bece5ceefc9c7371cc1b5c Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 24 Apr 2021 13:55:25 -0700 Subject: [PATCH 15/20] Add new UNSAFE_allowJavascriptFileAttachments param to Compiler. --- README.md | 38 +++++++++++++++++++----- src/compiler.js | 68 ++++++++++++++++++++++++++++++++----------- src/interpreter.js | 2 ++ test/compiler-test.js | 39 +++++++++++++++++++++---- 4 files changed, 118 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index fc826be..8cf95bb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# @alex.garcia/unofficial-observablehq-compiler +# unofficial-observablehq-compiler An unoffical compiler _and_ interpreter for Observable notebooks (the glue between the Observable parser and runtime) @@ -63,13 +63,37 @@ TODO new **Compiler**(_params_) -`Compiler` is a class that encompasses all logic to compile Observable javascript code into vanilla Javascript code, as an ES module. _params_ is an optional object with the following allowed configuration: +`Compiler` is a class that encompasses all logic to compile Observable javascript code into vanilla Javascript code, as an ES module. _params_ is an optional object with the following allowed configuration. -- `resolveImportPath`: A function that, given an import path, returns a URL to where the notebook's is defined. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `"https://api.observablehq.com/@d3/bar-chart.js?v=3"`. -- `resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. -- `defineImportMarkdown` - A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. -- `observeViewofValues` - A boolean, whether or not to pass in the `observer` function for viewof value cells. Defaults true. -- `observeMutableValues` - A boolean, whether or not to pass in the `observer` function for mutable value cells. Defaults true. +`resolveImportPath`: A function that, given an import path, returns a URL to where the notebook's is defined. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `"https://api.observablehq.com/@d3/bar-chart.js?v=3"`. + +`resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. + +`UNSAFE_allowJavascriptFileAttachments` A boolean. When true, the `resolveFileAttachments` function will resolve to raw JavaScript when calculating the value of a FileAttachment reference. This is useful if you need to use `new URL` or `import.meta.url` when determining where a FileAttachment url should resolve too. This is unsafe because the Compiler will not escape any quotes when including it in the compiled output, so do use with extreme caution when dealing with user input. + +```javascript + +// This can be unsafe since FileAttachment names can include quotes. +// Instead, map file attachments names to something deterministic and escape-safe, +// like SHA hashes. +const resolveFileAttachments = name => `new URL("./files/${name}", import.meta.url)` + +Compiled output when: + +// UNSAFE_allowJavascriptFileAttachments == false +const fileAttachments = new Map([["a", "new URL(\"./files/a\", import.meta.url)"]]); + +// UNSAFE_allowJavascriptFileAttachments == true +const fileAttachments = new Map([["a", new URL("./files/a", import.meta.url)]]); + + +``` + +`defineImportMarkdown` - A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. + +`observeViewofValues` - A boolean, whether or not to pass in the `observer` function for viewof value cells. Defaults true. + +`observeMutableValues` - A boolean, whether or not to pass in the `observer` function for mutable value cells. Defaults true. compile.**module**(_source_) diff --git a/src/compiler.js b/src/compiler.js index a5f740f..238d16d 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -12,6 +12,8 @@ function ESMImports(moduleObject, resolveImportPath) { continue; const defineName = `define${++j}`; + // TODO get cell specififiers here and pass in as 2nd param to resolveImportPath + // need to use same logic as tree-shake name()s const fromPath = resolveImportPath(body.source.value); importMap.set(body.source.value, { defineName, fromPath }); importSrc += `import ${defineName} from "${fromPath}";\n`; @@ -21,21 +23,44 @@ function ESMImports(moduleObject, resolveImportPath) { return { importSrc, importMap }; } -function ESMAttachments(moduleObject, resolveFileAttachments) { - const attachmentMapEntries = []; - // loop over cells with fileAttachments - for (const cell of moduleObject.cells) { - if (cell.fileAttachments.size === 0) continue; - // add filenames and resolved URLs to array - for (const file of cell.fileAttachments.keys()) - attachmentMapEntries.push([file, resolveFileAttachments(file)]); +// by default, file attachments get resolved like: +// [ ["a", "https://example.com/files/a"] ] +// but sometimes, you'll want to write JS code when resolving +// instead of being limiting by strings. The third param +// enables that, allowing for resolving like: +// [ ["a", new URL("./files/a", import.meta.url)] ] +function ESMAttachments( + moduleObject, + resolveFileAttachments, + UNSAFE_allowJavascriptFileAttachments = false +) { + let mapValue; + if (UNSAFE_allowJavascriptFileAttachments) { + const attachmentMapEntries = []; + for (const cell of moduleObject.cells) { + if (cell.fileAttachments.size === 0) continue; + for (const file of cell.fileAttachments.keys()) + attachmentMapEntries.push([file, resolveFileAttachments(file)]); + } + if (attachmentMapEntries.length) + mapValue = `[${attachmentMapEntries + .map(([key, value]) => `[${JSON.stringify(key)}, ${value}]`) + .join(",")}]`; + } else { + const attachmentMapEntries = []; + // loop over cells with fileAttachments + for (const cell of moduleObject.cells) { + if (cell.fileAttachments.size === 0) continue; + // add filenames and resolved URLs to array + for (const file of cell.fileAttachments.keys()) + attachmentMapEntries.push([file, resolveFileAttachments(file)]); + } + if (attachmentMapEntries.length) + mapValue = JSON.stringify(attachmentMapEntries); } - return attachmentMapEntries.length === 0 - ? "" - : ` const fileAttachments = new Map(${JSON.stringify( - attachmentMapEntries - )}); + if (!mapValue) return ""; + return ` const fileAttachments = new Map(${mapValue}); main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name)));`; } @@ -140,12 +165,17 @@ function createESModule(moduleObject, params = {}) { resolveFileAttachments, defineImportMarkdown, observeViewofValues, - observeMutableValues + observeMutableValues, + UNSAFE_allowJavascriptFileAttachments } = params; const { importSrc, importMap } = ESMImports(moduleObject, resolveImportPath); return `${importSrc}export default function define(runtime, observer) { const main = runtime.module(); -${ESMAttachments(moduleObject, resolveFileAttachments)} +${ESMAttachments( + moduleObject, + resolveFileAttachments, + UNSAFE_allowJavascriptFileAttachments +)} ${ESMVariables(moduleObject, importMap, { defineImportMarkdown, observeViewofValues, @@ -170,13 +200,15 @@ export class Compiler { resolveImportPath = defaultResolveImportPath, defineImportMarkdown = true, observeViewofValues = true, - observeMutableValues = true + observeMutableValues = true, + UNSAFE_allowJavascriptFileAttachments = false } = params; this.resolveFileAttachments = resolveFileAttachments; this.resolveImportPath = resolveImportPath; this.defineImportMarkdown = defineImportMarkdown; this.observeViewofValues = observeViewofValues; this.observeMutableValues = observeMutableValues; + this.UNSAFE_allowJavascriptFileAttachments = UNSAFE_allowJavascriptFileAttachments; } module(text, params = {}) { let m1 = parseModule(text); @@ -188,7 +220,9 @@ export class Compiler { resolveFileAttachments: this.resolveFileAttachments, defineImportMarkdown: this.defineImportMarkdown, observeViewofValues: this.observeViewofValues, - observeMutableValues: this.observeMutableValues + observeMutableValues: this.observeMutableValues, + UNSAFE_allowJavascriptFileAttachments: this + .UNSAFE_allowJavascriptFileAttachments }); } notebook(obj) { diff --git a/src/interpreter.js b/src/interpreter.js index 66dab14..36f1d5f 100644 --- a/src/interpreter.js +++ b/src/interpreter.js @@ -99,6 +99,8 @@ export class Interpreter { if (cell.body.type === "ImportDeclaration") { const path = cell.body.source.value; + // TODO get cell specififiers here and pass in as 2nd param to resolveImportPath + // need to use same logic as tree-shake name() const fromModule = await this.resolveImportPath(path); let mdVariable, vars; diff --git a/test/compiler-test.js b/test/compiler-test.js index 0d2950b..cd2fb6a 100644 --- a/test/compiler-test.js +++ b/test/compiler-test.js @@ -281,6 +281,32 @@ FileAttachment("a") t.end(); }); +test("Compiler: resolveFileAttachments UNSAFE_allowJavascriptFileAttachments", async t => { + function resolveFileAttachments(name) { + return `new URL("./path/to/${name}", import.meta.url)`; + } + let compile = new Compiler({ + resolveFileAttachments, + UNSAFE_allowJavascriptFileAttachments: true + }); + let src = compile.module('a = FileAttachment("a")'); + + t.equal( + src, + `export default function define(runtime, observer) { + const main = runtime.module(); + const fileAttachments = new Map([["a", new URL("./path/to/a", import.meta.url)]]); + main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); + main.variable(observer("a")).define("a", ["FileAttachment"], function(FileAttachment){return( +FileAttachment("a") +)}); + return main; +}` + ); + + t.end(); +}); + test("Compiler: treeShake", async t => { let src; const initSrc = ` @@ -291,7 +317,7 @@ test("Compiler: treeShake", async t => { const compile = new Compiler({ defineImportMarkdown: false }); src = compile.module(initSrc, { - treeShake: ["f"], + treeShake: ["f"] }); t.equal( @@ -313,7 +339,7 @@ d + e ); src = compile.module(initSrc, { - treeShake: ["f", "a"], + treeShake: ["f", "a"] }); t.equal( @@ -339,7 +365,7 @@ d + e ); src = compile.module(initSrc, { - treeShake: ["chart"], + treeShake: ["chart"] }); t.equal( @@ -369,12 +395,15 @@ c ); // make sure tree shaking an unreference cell just returns a define identity function. - t.equals(compile.module(`a = 1; b = a/1;`, {treeShake:["c"]}), `export default function define(runtime, observer) { + t.equals( + compile.module(`a = 1; b = a/1;`, { treeShake: ["c"] }), + `export default function define(runtime, observer) { const main = runtime.module(); return main; -}`) +}` + ); t.end(); }); From fac56a2e6516a113888f4807ec08d82b2aad3f5e Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 24 Apr 2021 13:56:13 -0700 Subject: [PATCH 16/20] v0.6.0-alpha.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 144243b..0e1dfe3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.6", + "version": "0.6.0-alpha.7", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From 1f1fd76354828d0110fa3c90be93af9c010188b6 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 24 Apr 2021 17:46:11 -0700 Subject: [PATCH 17/20] Include import specifiers cell names in 2nd param to resolveImportPath. --- README.md | 14 +++++++------- src/compiler.js | 6 +++++- src/interpreter.js | 8 +++++--- test/compiler-test.js | 7 +++++-- test/interpreter-test.js | 25 ++++++++++++++++++++++++- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8cf95bb..1662508 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,12 @@ and this [test page](https://github.com/asg017/unofficial-observablehq-compiler/ `Interpreter` is a class that encompasses all logic to interpret Observable js code. _params_ is an optional object with the following allowed configuration: -- `resolveImportPath`: An async function that, given an import path, resolves to the `define` defintion for that "remote" notebook. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `https://api.observablehq.com/@d3/bar-chart.js?v=3` -- `resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. -- `defineImportMarkdown`: A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. -- `observeViewofValues`: A boolean, whether to pass in an _observer_ for the value of viewof cells. -- `module`: A default Observable runtime [module](https://github.com/observablehq/runtime#modules) that will be used in operations such as `.cell` and `.module`, if no other module is passed in. -- `observer`: A default Observable runtime [observer](https://github.com/observablehq/runtime#observer) that will be used in operations such as `.cell` and `.module`, if no other observer is passed in. +`resolveImportPath(path, specifers)`: An async function that resolves to the `define` definition for a notebook. _path_ is the string provided in the import statement, and _specifiers_ is a array of string if the imported cell name specifiers in the statement (cells inside `import {...}`). _specifiers_ is useful when implementing tree-shaking. For example, `import {chart as CHART} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"` and `specifiers=["chart"]`. Default imports from observablehq.com, eg `https://api.observablehq.com/@d3/bar-chart.js?v=3` +`resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. +`defineImportMarkdown`: A boolean, whether to define a markdown description cell for imports in the notebook. Defaults true. +`observeViewofValues`: A boolean, whether to pass in an _observer_ for the value of viewof cells. +`module`: A default Observable runtime [module](https://github.com/observablehq/runtime#modules) that will be used in operations such as `.cell` and `.module`, if no other module is passed in. +`observer`: A default Observable runtime [observer](https://github.com/observablehq/runtime#observer) that will be used in operations such as `.cell` and `.module`, if no other observer is passed in. Keep in mind, there is no sandboxing done, so it has the same security implications as `eval()`. @@ -65,7 +65,7 @@ new **Compiler**(_params_) `Compiler` is a class that encompasses all logic to compile Observable javascript code into vanilla Javascript code, as an ES module. _params_ is an optional object with the following allowed configuration. -`resolveImportPath`: A function that, given an import path, returns a URL to where the notebook's is defined. For example, `import {chart} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"`. Default imports from observablehq.com, eg `"https://api.observablehq.com/@d3/bar-chart.js?v=3"`. +`resolveImportPath(path, specifers)`: A function that returns a URL to where the notebook is defined. _path_ is the string provided in the import statement, and _specifiers_ is a array of string if the imported cell name specifiers in the statement (cells inside `import {...}`). _specifiers_ is useful when implementing tree-shaking. For example, `import {chart as CHART} from "@d3/bar-chart"` would supply `path="@d3/bar-chart"` and `specifiers=["chart"]`. Default imports from observablehq.com, eg `https://api.observablehq.com/@d3/bar-chart.js?v=3` `resolveFileAttachments`: A function that, given the name of a FileAttachment, returns the URL that the FileAttachment will be fetched from. Defaults to `name => name`. diff --git a/src/compiler.js b/src/compiler.js index 238d16d..c63b7ba 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -14,7 +14,11 @@ function ESMImports(moduleObject, resolveImportPath) { const defineName = `define${++j}`; // TODO get cell specififiers here and pass in as 2nd param to resolveImportPath // need to use same logic as tree-shake name()s - const fromPath = resolveImportPath(body.source.value); + const specifiers = body.specifiers.map(d => { + const prefix = d.view ? "viewof " : d.mutable ? "mutable " : ""; + return `${prefix}${d.imported.name}`; + }); + const fromPath = resolveImportPath(body.source.value, specifiers); importMap.set(body.source.value, { defineName, fromPath }); importSrc += `import ${defineName} from "${fromPath}";\n`; } diff --git a/src/interpreter.js b/src/interpreter.js index 36f1d5f..bdb1bc5 100644 --- a/src/interpreter.js +++ b/src/interpreter.js @@ -99,9 +99,11 @@ export class Interpreter { if (cell.body.type === "ImportDeclaration") { const path = cell.body.source.value; - // TODO get cell specififiers here and pass in as 2nd param to resolveImportPath - // need to use same logic as tree-shake name() - const fromModule = await this.resolveImportPath(path); + const specs = cell.body.specifiers.map(d => { + const prefix = d.view ? "viewof " : d.mutable ? "mutable " : ""; + return `${prefix}${d.imported.name}`; + }); + const fromModule = await this.resolveImportPath(path, specs); let mdVariable, vars; const { diff --git a/test/compiler-test.js b/test/compiler-test.js index cd2fb6a..b67ce79 100644 --- a/test/compiler-test.js +++ b/test/compiler-test.js @@ -106,14 +106,17 @@ $0.value = 202 test("Compiler: import cells", async t => { const compile = new Compiler({ - resolveImportPath: d => `https://example.com/${d}` + resolveImportPath: (name, specifiers) => + `https://example.com/${name}?${specifiers + .map(s => `specifiers=${encodeURIComponent(s)}`) + .join("&")}` }); const src = compile.module(`import {a as A, b as B, c as C} from "alpha";`); t.equal( src, - `import define1 from "https://example.com/alpha"; + `import define1 from "https://example.com/alpha?specifiers=a&specifiers=b&specifiers=c"; export default function define(runtime, observer) { const main = runtime.module(); diff --git a/test/interpreter-test.js b/test/interpreter-test.js index 2577acf..b546ebd 100644 --- a/test/interpreter-test.js +++ b/test/interpreter-test.js @@ -90,7 +90,8 @@ test("Interpreter: mutable cells", async t => { }); test("Interpreter: import cells", async t => { - function resolveImportPath(name) { + function resolveImportPath(name, specifiers) { + const specs = new Set(specifiers); if (name === "alpha") return function define(runtime, observer) { const main = runtime.module(); @@ -173,6 +174,28 @@ test("Interpreter: import cells", async t => { t.end(); }); + +test("Interpreter: resolveImportPath specifiers", async t => { + function resolveImportPath(name, specifiers) { + t.plan(1); + return function define(runtime, observer) { + t.deepEquals(specifiers, ["a", "b", "x"]); + return main; + }; + } + const interpret = new Interpreter({ resolveImportPath }); + let runtime = new Runtime(); + let main = runtime.module(); + let observer = () => null; + + await interpret.cell( + `import {a as A, b as B, x as X} from "alpha";`, + main, + observer + ); + t.end(); +}); + test("Interpreter: defineImportMarkdown", async t => { function resolveImportPath(name) { return function define(runtime, observer) { From 1a12e245a6a053c923c9ba484fef85bbea04144e Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 24 Apr 2021 17:46:52 -0700 Subject: [PATCH 18/20] v0.6.0-alpha.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0e1dfe3..a08e1b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.7", + "version": "0.6.0-alpha.8", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ", From b2f23bf4e4214bfc6bfd3503757db60bcbecd2e4 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 25 Apr 2021 01:28:34 -0700 Subject: [PATCH 19/20] export treeShakeModule, allow modules to be passed in as input in compile.module. --- src/compiler.js | 8 ++++---- src/index.js | 1 + src/tree-shake.js | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/compiler.js b/src/compiler.js index c63b7ba..7c1127c 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -1,6 +1,6 @@ import { parseCell, parseModule } from "@observablehq/parser"; import { setupRegularCell, setupImportCell, extractPath } from "./utils"; -import { computeShakenCells } from "./tree-shake"; +import { treeShakeModule } from "./tree-shake"; function ESMImports(moduleObject, resolveImportPath) { const importMap = new Map(); @@ -214,10 +214,10 @@ export class Compiler { this.observeMutableValues = observeMutableValues; this.UNSAFE_allowJavascriptFileAttachments = UNSAFE_allowJavascriptFileAttachments; } - module(text, params = {}) { - let m1 = parseModule(text); + module(input, params = {}) { + let m1 = typeof input === "string" ? parseModule(input) : input; - if (params.treeShake) m1 = computeShakenCells(m1, params.treeShake); + if (params.treeShake) m1 = treeShakeModule(m1, params.treeShake); return createESModule(m1, { resolveImportPath: this.resolveImportPath, diff --git a/src/index.js b/src/index.js index 8a7b4f2..099b48f 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,4 @@ export { Compiler } from "./compiler.js"; export { Interpreter } from "./interpreter.js"; export * as parser from "@observablehq/parser"; +export { treeShakeModule } from "./tree-shake.js"; diff --git a/src/tree-shake.js b/src/tree-shake.js index 6d71949..6da74af 100644 --- a/src/tree-shake.js +++ b/src/tree-shake.js @@ -47,7 +47,7 @@ function getCellRefs(module) { return new Map(cells); } -export function computeShakenCells(module, targets) { +export function treeShakeModule(module, targets) { const cellRefs = getCellRefs(module); const embed = new Set(); From 0c4a2b7864d163611b81c005f547065761943b79 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sun, 25 Apr 2021 01:29:43 -0700 Subject: [PATCH 20/20] v0.6.0-alpha.9 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a08e1b5..5c24cf4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex.garcia/unofficial-observablehq-compiler", - "version": "0.6.0-alpha.8", + "version": "0.6.0-alpha.9", "description": "An unofficial compiler to bind @observablehq/parser and @observablehq/runtime", "main": "dist/index.js", "author": "Alex Garcia ",