[Notebooks]: Support Jupyter Notebooks (#2048)

* feat(plugins): add jupyter plugin - #2029

* feat-fix(notebook): add required nbformat package - #2029

* test(plugin): use strictEqual - #2029

* feat-fix(plugin): add plugin to registery - #2029

---------

Co-authored-by: Florian Sihler <florian.sihler@uni-ulm.de>

Author: gigalasr

package-lock.json

@@ -198,6 +198,7 @@
   },
   "dependencies": {
     "@eagleoutice/tree-sitter-r": "^1.1.2",
+    "@jupyterlab/nbformat": "^4.5.0",
     "@xmldom/xmldom": "^0.9.7",
     "clipboardy": "^4.0.0",
     "command-line-args": "^6.0.1",

package.json

@@ -0,0 +1,36 @@
+import type { PathLike } from 'fs';
+import { SemVer } from 'semver';
+import type { FlowrAnalyzerContext } from '../../../context/flowr-analyzer-context';
+import type { FlowrFileProvider } from '../../../context/flowr-file';
+import { FlowrAnalyzerFilePlugin } from '../flowr-analyzer-file-plugin';
+import { platformBasename } from '../../../../dataflow/internal/process/functions/call/built-in/built-in-source';
+import { FlowrJupyterFile } from './flowr-jupyter-file';
+
+const IpynbPattern = /\.ipynb$/i;
+
+/**
+ * The plugin provides support for Jupyter (`.ipynb`) files
+ */
+export class FlowrAnalyzerJupyterFilePlugin extends FlowrAnalyzerFilePlugin {
+	public readonly name =    'ipynb-file-plugin';
+	public readonly description = 'Parses Jupyter files';
+	public readonly version = new SemVer('0.1.0');
+	private readonly pattern: RegExp;
+
+	/**
+	 * Creates a new instance of the Jupyter file plugin.
+	 * @param filePattern - The pattern to identify Jupyter files, see {@link IpynbPattern} for the default pattern.
+	 */
+	constructor(filePattern: RegExp = IpynbPattern) {
+		super();
+		this.pattern = filePattern;
+	}
+
+	public applies(file: PathLike): boolean {
+		return this.pattern.test(platformBasename(file.toString()));
+	}
+
+	protected process(_ctx: FlowrAnalyzerContext, arg: FlowrFileProvider<string>): FlowrJupyterFile {
+		return FlowrJupyterFile.from(arg);
+	}
+}

src/project/plugins/file-plugins/notebooks/flowr-analyzer-jupyter-file-plugin.ts

@@ -31,6 +31,6 @@ export class FlowrAnalyzerQmdFilePlugin extends FlowrAnalyzerFilePlugin {
 	}
 
 	protected process(_ctx: FlowrAnalyzerContext, arg: FlowrFileProvider<string>): FlowrRMarkdownFile {
-		return new FlowrRMarkdownFile(arg);
+		return FlowrRMarkdownFile.from(arg);
 	}
-}
+}

src/project/plugins/file-plugins/notebooks/flowr-analyzer-qmd-file-plugin.ts

@@ -0,0 +1,45 @@
+import type { INotebookContent } from '@jupyterlab/nbformat';
+import type { FlowrFileProvider } from '../../../context/flowr-file';
+import { FileRole, FlowrFile } from '../../../context/flowr-file';
+
+
+/**
+ * This decorates a text file and parses its contents as a Jupyter file.
+ * Finnaly, it provides access to the single cells, and all cells fused together as one R file.
+ */
+export class FlowrJupyterFile extends FlowrFile<string> {
+	private readonly wrapped: FlowrFileProvider<string>;
+
+	/**
+	 * Prefer the static {@link FlowrRMarkdownFile.from} method
+	 * @param file - the file to load as R Markdown
+	 */
+	constructor(file: FlowrFileProvider<string>) {
+		super(file.path(), FileRole.Source);
+		this.wrapped = file;
+	}
+
+	/**
+	 * Loads and parses the content of the wrapped file.
+	 * @returns RmdInfo
+	 */
+	protected loadContent(): string {
+		return loadJupyter(this.wrapped.content());
+	}
+
+	public static from(file: FlowrFileProvider<string> | FlowrJupyterFile): FlowrJupyterFile {
+		return file instanceof FlowrJupyterFile ? file : new FlowrJupyterFile(file);
+	}
+}
+
+function loadJupyter(content: string): string {
+	const nb = JSON.parse(content) as INotebookContent;
+
+	return nb.cells.map(cell => {
+		if(cell.cell_type === 'code') {
+			return typeof cell.source === 'object' ? cell.source.join('') : cell.source;
+		} else {
+			return typeof cell.source === 'object' ? cell.source.map(s => `# ${s}`).join('') : `# ${cell.source}`;
+		}
+	}).join('\n');
+}

src/project/plugins/file-plugins/notebooks/flowr-jupyter-file.ts

@@ -8,6 +8,7 @@ import {
 } from './loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin';
 import { FlowrAnalyzerRmdFilePlugin } from './file-plugins/notebooks/flowr-analyzer-rmd-file-plugin';
 import { FlowrAnalyzerQmdFilePlugin } from './file-plugins/notebooks/flowr-analyzer-qmd-file-plugin';
+import { FlowrAnalyzerJupyterFilePlugin } from './file-plugins/notebooks/flowr-analyzer-jupyter-file-plugin';
 
 /**
  * Provides the default set of Flowr Analyzer plugins.
@@ -18,6 +19,7 @@ export function FlowrAnalyzerPluginDefaults(): FlowrAnalyzerPlugin[] {
 		new FlowrAnalyzerPackageVersionsDescriptionFilePlugin(),
 		new FlowrAnalyzerLoadingOrderDescriptionFilePlugin(),
 		new FlowrAnalyzerRmdFilePlugin(),
-		new FlowrAnalyzerQmdFilePlugin()
+		new FlowrAnalyzerQmdFilePlugin(),
+		new FlowrAnalyzerJupyterFilePlugin(),
 	];
-}
+}

src/project/plugins/file-plugins/notebooks/notebook.ts

@@ -9,6 +9,7 @@ import {
 import { FlowrAnalyzerRmdFilePlugin } from './file-plugins/notebooks/flowr-analyzer-rmd-file-plugin';
 import { FlowrAnalyzerQmdFilePlugin } from './file-plugins/notebooks/flowr-analyzer-qmd-file-plugin';
 import { guard } from '../../util/assert';
+import { FlowrAnalyzerJupyterFilePlugin } from './file-plugins/notebooks/flowr-analyzer-jupyter-file-plugin';
 
 /**
  * The built-in Flowr Analyzer plugins that are always available.
@@ -18,7 +19,8 @@ export const BuiltInPlugins = [
 	['versions:description', FlowrAnalyzerPackageVersionsDescriptionFilePlugin],
 	['loading-order:description', FlowrAnalyzerLoadingOrderDescriptionFilePlugin],
 	['file:rmd', FlowrAnalyzerRmdFilePlugin],
-	['file:qmd', FlowrAnalyzerQmdFilePlugin]
+	['file:qmd', FlowrAnalyzerQmdFilePlugin],
+	['file:ipynb', FlowrAnalyzerJupyterFilePlugin],
 ] as const satisfies [string, PluginProducer][];
 
 export type BuiltInFlowrPluginName = typeof BuiltInPlugins[number][0];
@@ -79,4 +81,4 @@ export function makePlugin<T extends BuiltInFlowrPluginName | string>(toRegister
 	const plugin = getPlugin(toRegister, []);
 	guard(plugin !== undefined, () => `Unknown Flowr Analyzer plugin: ${toRegister.toString()}`);
 	return plugin;
-}
+}

src/project/plugins/flowr-analyzer-plugin-defaults.ts

@@ -0,0 +1,14 @@
+import { describe } from 'vitest';
+import { testFileLoadPlugin } from './plugin-test-helper';
+import { FlowrAnalyzerJupyterFilePlugin } from '../../../../src/project/plugins/file-plugins/notebooks/flowr-analyzer-jupyter-file-plugin';
+import { FlowrJupyterFile } from '../../../../src/project/plugins/file-plugins/notebooks/flowr-jupyter-file';
+
+describe('Jupyter-file', async() => {
+	await testFileLoadPlugin(FlowrAnalyzerJupyterFilePlugin, FlowrJupyterFile, 'test/testfiles/notebook/example.ipynb', `x <- 5
+cat(x)
+# # Hello
+# This is a cool test
+y <- c(1,2,3)
+plot(y)
+# Hi`);
+});

src/project/plugins/plugin-registry.ts

@@ -43,8 +43,8 @@ export async function testFileLoadPlugin<F extends ConstructTo<FlowrFile>, P ext
 		await analyzer.parse();
 
 		const files = analyzer.inspectContext().files.getFilesByRole(FileRole.Source);
-		assert(files.length === 1);
+		assert.strictEqual(files.length, 1);
 		assert(files[0] instanceof pluginFileType);
-		assert(files[0].content() === expectedContent);
+		assert.strictEqual(files[0].content(), expectedContent);
 	});
 }