Updating snippet parsing and consolidation to support sub languages

Author: Mathys-Gasnier

utils/consolidateSnippets.js

@@ -1,5 +1,5 @@
 import { exit } from 'process';
-import { parseAllSnippets, reverseSlugify } from './snippetParser.js';
+import { parseAllSnippets, reverseSlugify, slugify } from './snippetParser.js';
 import { join } from 'path';
 import { copyFileSync, writeFileSync } from 'fs';
 
@@ -8,21 +8,39 @@ const indexPath = join(dataPath, '_index.json');
 const iconPath = 'public/icons/';
 const snippetsPath = 'snippets/';
 
-const [ errored, snippets ] = parseAllSnippets();
+const [ errored, languages ] = parseAllSnippets();
 
 if(errored) exit(1);
 
 const index = [];
-for(const [language, categories] of Object.entries(snippets)) {
-    const languageIconPath = join(snippetsPath, language, 'icon.svg');
+for(const language of languages) {
+    copyFileSync(language.icon, join(iconPath, `${slugify(language.name)}.svg`));
 
-    copyFileSync(languageIconPath, join(iconPath, `${language}.svg`));
+    const subIndexes = [];
 
-    index.push({ lang: reverseSlugify(language).toUpperCase(), icon: `/icons/${language}.svg` });
+    for(const subLanguage of language.subLanguages) {
+        const joinedName = `${slugify(language.name)}--${slugify(subLanguage.name)}`;
+        const iconName = `${joinedName}.svg`;
+        const subLanguageFilePath = join(dataPath, `${joinedName}.json`);
+
+        copyFileSync(subLanguage.icon, join(iconPath, iconName));
+        subIndexes.push({
+            name: subLanguage.name.toUpperCase(),
+            icon: `/icons/${iconName}`,
+        });
+
+        writeFileSync(subLanguageFilePath, JSON.stringify(subLanguage.categories, null, 4));
+    }
+
+    index.push({
+        name: language.name.toUpperCase(),
+        icon: `/icons/${slugify(language.name)}.svg`,
+        subLanguages: subIndexes,
+    });
 
-    const languageFilePath = join(dataPath, `${language}.json`);
+    const languageFilePath = join(dataPath, `${slugify(language.name)}.json`);
 
-    writeFileSync(languageFilePath, JSON.stringify(categories, null, 4));
+    writeFileSync(languageFilePath, JSON.stringify(language.categories, null, 4));
 }
 
 writeFileSync(indexPath, JSON.stringify(index, null, 4));

utils/snippetParser.js

@@ -31,13 +31,13 @@ const crlfRegex = /\r\n/gm;
 const propertyRegex = /^\s+([a-zA-Z]+):\s*(.+)/;
 const headerEndCodeStartRegex = /^\s*---\s*```.*\n/;
 const codeRegex = /^(.+)```/s
-function parseSnippet(snippetPath, name, text) {
-    if(crlfRegex.exec(text) !== null) return raise('Found CRLF line endings instead of LF line endings', snippetPath);
+function parseSnippet(path, name, text) {
+    if(crlfRegex.exec(text) !== null) return raise('Found CRLF line endings instead of LF line endings', path);
     let cursor = 0;
 
     const fromCursor = () => text.substring(cursor);
 
-    if(!fromCursor().trim().startsWith('---')) return raise('Missing header start delimiter \'---\'', snippetPath);
+    if(!fromCursor().trim().startsWith('---')) return raise('Missing header start delimiter \'---\'', path);
     cursor += 3;
 
     const properties = {};
@@ -47,19 +47,19 @@ function parseSnippet(snippetPath, name, text) {
         properties[match[1].toLowerCase()] = match[2];
     }
 
-    if(!('title' in properties)) return raise(`Missing 'title' property`, snippetPath);
-    if(!('description' in properties)) return raise(`Missing 'description' property`, snippetPath);
-    if(!('author' in properties)) return raise(`Missing 'author' property`, snippetPath);
-    if(!('tags' in properties)) return raise(`Missing 'tags' property`, snippetPath);
+    if(!('title' in properties)) return raise(`Missing 'title' property`, path);
+    if(!('description' in properties)) return raise(`Missing 'description' property`, path);
+    if(!('author' in properties)) return raise(`Missing 'author' property`, path);
+    if(!('tags' in properties)) return raise(`Missing 'tags' property`, path);
 
-    if(slugify(properties.title) !== name) return raise(`slugifyed 'title' property doesn't match snippet file name`, snippetPath);
+    if(slugify(properties.title) !== name) return raise(`slugifyed 'title' property doesn't match snippet file name`, path);
 
     match = headerEndCodeStartRegex.exec(fromCursor());
-    if(match === null) return raise('Missing header end \'---\' or code start \'```\'', snippetPath);
+    if(match === null) return raise('Missing header end \'---\' or code start \'```\'', path);
     cursor += match[0].length;
 
     match = codeRegex.exec(fromCursor());
-    if(match === null) return raise('Missing code block end \'```\'', snippetPath);
+    if(match === null) return raise('Missing code block end \'```\'', path);
     const code = match[1];
 
     return {
@@ -72,43 +72,63 @@ function parseSnippet(snippetPath, name, text) {
     }
 }
 
+function parseCategory(path, name) {
+    const snippets = [];
+
+    for(const snippet of readdirSync(path)) {
+        const snippetPath = join(path, snippet);
+        const snippetContent = readFileSync(snippetPath).toString();
+        const snippetFileName = snippet.slice(0, -3);
+
+        const snippetData = parseSnippet(snippetPath, snippetFileName, snippetContent);
+        if(!snippetData) continue;
+        snippets.push(snippetData);
+    }
+
+    return {
+        name: reverseSlugify(name),
+        snippets,
+    };
+}
+
+function parseLanguage(path, name, subLanguageOf = null) {
+    const iconPath = join(path, 'icon.svg');
+
+    if(!existsSync(iconPath)) return raise(`icon for '${subLanguageOf ? `${subLanguageOf}/` : '' }${name}' is missing`);
+
+    const categories = [];
+    const subLanguages = [];
+
+    for(const category of readdirSync(path)) {
+        if(category === 'icon.svg') continue;
+        const categoryPath = join(path, category);
+
+        if(category.startsWith('[') && category.endsWith(']')) {
+            if(subLanguageOf !== null) {
+                return raise('Cannot have more than two level of language nesting');
+            }
+            subLanguages.push(parseLanguage(categoryPath, category.slice(1, -1), name));
+        }else {
+            categories.push(parseCategory(categoryPath, category));
+        }
+    }
+
+    return {
+        name: reverseSlugify(name),
+        icon: iconPath,
+        categories, subLanguages,
+    }
+}
+
 const snippetPath = "snippets/";
 export function parseAllSnippets() {
-    const snippets = {};
+    const languages = [];
 
     for(const language of readdirSync(snippetPath)) {
         const languagePath = join(snippetPath, language);
-    
-        const languageIconPath = join(languagePath, 'icon.svg');
-    
-        if(!existsSync(languageIconPath)) {
-            raise(`icon for '${language}' is missing`);
-            continue;
-        }
-        
-        const categories = [];
-        for(const category of readdirSync(languagePath)) {
-            if(category === 'icon.svg') continue;
-            const categoryPath = join(languagePath, category);
-    
-            const categorySnippets = [];
-            for(const snippet of readdirSync(categoryPath)) {
-                const snippetPath = join(categoryPath, snippet);
-                const snippetContent = readFileSync(snippetPath).toString();
-                const snippetFileName = snippet.slice(0, -3);
-    
-                const snippetData = parseSnippet(snippetPath, snippetFileName, snippetContent);
-                if(!snippetData) continue;
-                categorySnippets.push(snippetData);
-            }
-            categories.push({
-                categoryName: reverseSlugify(category),
-                snippets: categorySnippets,
-            });
-        }
 
-        snippets[language] = categories;
+        languages.push(parseLanguage(languagePath, language));
     }
 
-    return [ errored, snippets ];
+    return [ errored, languages ];
 }