Changed the parsing of custom block to parse like Vue3 parser. (#90)

* Support for custom blocks

* update

* Add testcase

* Add testcases

Author: ota-meshi

scripts/update-fixtures-document-fragment.js

@@ -61,8 +61,53 @@ for (const name of TARGETS) {
     const actual = result.services.getDocumentFragment()
 
     const resultPath = path.join(ROOT, `${name}/document-fragment.json`)
+    const tokenRangesPath = path.join(ROOT, `${name}/token-ranges.json`)
+    const treePath = path.join(ROOT, `${name}/tree.json`)
 
     console.log("Update:", name)
 
+    const tokenRanges = getAllTokens(actual).map(t =>
+        source.slice(t.range[0], t.range[1])
+    )
+    const tree = getTree(source, actual)
+
     fs.writeFileSync(resultPath, JSON.stringify(actual, replacer, 4))
+    fs.writeFileSync(tokenRangesPath, JSON.stringify(tokenRanges, replacer, 4))
+    fs.writeFileSync(treePath, JSON.stringify(tree, replacer, 4))
+}
+
+function getAllTokens(fgAst) {
+    const tokenArrays = [fgAst.tokens, fgAst.comments]
+
+    return Array.prototype.concat.apply([], tokenArrays)
+}
+
+/**
+ * Create simple tree.
+ * @param {string} source The source code.
+ * @param {VDocumentFragment} fgAst The root node.
+ * @returns {object} Simple tree.
+ */
+function getTree(source, fgAst) {
+    const stack = []
+    const root = { children: [] }
+    let current = root
+
+    parser.AST.traverseNodes(fgAst, {
+        enterNode(node) {
+            stack.push(current)
+            current.children.push(
+                (current = {
+                    type: node.type,
+                    text: source.slice(node.range[0], node.range[1]),
+                    children: [],
+                })
+            )
+        },
+        leaveNode() {
+            current = stack.pop()
+        },
+    })
+
+    return root.children
 }

src/html/parser.ts

@@ -3,6 +3,7 @@
  * @copyright 2017 Toru Nagashima. All rights reserved.
  * See LICENSE file in root directory for full license.
  */
+import * as path from "path"
 import assert from "assert"
 import last from "lodash/last"
 import findLastIndex from "lodash/findLastIndex"
@@ -150,6 +151,7 @@ export class Parser {
     private tokenizer: IntermediateTokenizer
     private locationCalculator: LocationCalculator
     private parserOptions: ParserOptions
+    private isSFC: boolean
     private document: VDocumentFragment
     private elementStack: VElement[]
     private vPreElement: VElement | null
@@ -230,6 +232,8 @@ export class Parser {
             tokenizer.lineTerminators,
         )
         this.parserOptions = parserOptions
+        this.isSFC =
+            path.extname(parserOptions.filePath || "unknown.vue") === ".vue"
         this.document = {
             type: "VDocumentFragment",
             range: [0, 0],
@@ -516,27 +520,40 @@ export class Parser {
 
         // Update the content type of this element.
         if (namespace === NS.HTML) {
-            if (
-                element.name === "template" &&
-                element.parent.type === "VDocumentFragment"
-            ) {
+            if (element.parent.type === "VDocumentFragment") {
                 const langAttr = element.startTag.attributes.find(
                     a => !a.directive && a.key.name === "lang",
                 ) as VAttribute | undefined
-                const lang =
-                    (langAttr && langAttr.value && langAttr.value.value) ||
-                    "html"
-
-                if (lang !== "html") {
+                const lang = langAttr?.value?.value
+
+                if (element.name === "template") {
+                    if (lang && lang !== "html") {
+                        // It is not an HTML template.
+                        this.tokenizer.state = "RAWTEXT"
+                    }
+                    this.expressionEnabled = true
+                } else if (this.isSFC) {
+                    // Element is Custom Block. e.g. <i18n>
+                    // Referred to the Vue parser. See https://github.com/vuejs/vue-next/blob/cbaa3805064cb581fc2007cf63774c91d39844fe/packages/compiler-sfc/src/parse.ts#L127
+                    if (!lang || lang !== "html") {
+                        // Custom Block is not HTML.
+                        this.tokenizer.state = "RAWTEXT"
+                    }
+                } else {
+                    if (HTML_RCDATA_TAGS.has(element.name)) {
+                        this.tokenizer.state = "RCDATA"
+                    }
+                    if (HTML_RAWTEXT_TAGS.has(element.name)) {
+                        this.tokenizer.state = "RAWTEXT"
+                    }
+                }
+            } else {
+                if (HTML_RCDATA_TAGS.has(element.name)) {
+                    this.tokenizer.state = "RCDATA"
+                }
+                if (HTML_RAWTEXT_TAGS.has(element.name)) {
                     this.tokenizer.state = "RAWTEXT"
                 }
-                this.expressionEnabled = true
-            }
-            if (HTML_RCDATA_TAGS.has(element.name)) {
-                this.tokenizer.state = "RCDATA"
-            }
-            if (HTML_RAWTEXT_TAGS.has(element.name)) {
-                this.tokenizer.state = "RAWTEXT"
             }
         }
     }

src/html/tokenizer.ts

@@ -955,7 +955,7 @@ export class Tokenizer {
                 this.endToken()
                 return "BEFORE_ATTRIBUTE_NAME"
             }
-            if (!isLetter(cp)) {
+            if (!isLetter(cp) && !maybeValidCustomBlock.call(this, cp)) {
                 this.rollbackProvisionalToken()
                 this.appendTokenValue(LESS_THAN_SIGN, "HTMLRawText")
                 this.appendTokenValue(SOLIDUS, "HTMLRawText")
@@ -973,6 +973,16 @@ export class Tokenizer {
 
             cp = this.consumeNextCodePoint()
         }
+
+        function maybeValidCustomBlock(this: Tokenizer, nextCp: number) {
+            return (
+                this.currentToken &&
+                this.lastTagOpenToken &&
+                this.lastTagOpenToken.value.startsWith(
+                    this.currentToken.value + String.fromCodePoint(nextCp),
+                )
+            )
+        }
     }
 
     /**

test/document-fragment.js

@@ -75,6 +75,23 @@ describe("services.getDocumentFragment", () => {
                     expected
                 )
             })
+
+            it("should have correct range.", () => {
+                const resultPath = path.join(ROOT, `${name}/token-ranges.json`)
+                const expectedText = fs.readFileSync(resultPath, "utf8")
+                const tokens = getAllTokens(actual).map(t =>
+                    source.slice(t.range[0], t.range[1])
+                )
+                const actualText = JSON.stringify(tokens, null, 4)
+
+                assert.strictEqual(actualText, expectedText)
+            })
         })
     }
 })
+
+function getAllTokens(fgAst) {
+    const tokenArrays = [fgAst.tokens, fgAst.comments]
+
+    return Array.prototype.concat.apply([], tokenArrays)
+}