Improve performance of hidden node parsing

packages/langium/src/parser/cst-node-builder.ts

@@ -54,30 +54,34 @@ export class CstNodeBuilder {
         }
     }
 
-    addHiddenNode(token: IToken): void {
-        const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, true);
-        leafNode.root = this.rootNode;
+    addHiddenNodes(tokens: IToken[]): void {
+        const nodes: LeafCstNode[] = [];
+        for (const token of tokens) {
+            const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, true);
+            leafNode.root = this.rootNode;
+            nodes.push(leafNode);
+        }
         let current: CompositeCstNode = this.current;
         let added = false;
-        // If we are within a composite node, we add the hidden node to the content
+        // If we are within a composite node, we add the hidden nodes to the content
         if (current.content.length > 0) {
-            current.content.push(leafNode);
+            current.content.push(...nodes);
             return;
         }
         while (current.container) {
             const index = current.container.content.indexOf(current);
             if (index > 0) {
-                // Add the hidden node before the current node
-                current.container.content.splice(index, 0, leafNode);
+                // Add the hidden nodes before the current node
+                current.container.content.splice(index, 0, ...nodes);
                 added = true;
                 break;
             }
             current = current.container;
         }
-        // If we arrive at the root node, we add the hidden node at the beginning
-        // This is the case if the hidden node is the first node in the tree
+        // If we arrive at the root node, we add the hidden nodes at the beginning
+        // This is the case if the hidden nodes are the first nodes in the tree
         if (!added) {
-            this.rootNode.content.unshift(leafNode);
+            this.rootNode.content.unshift(...nodes);
         }
     }
 

packages/langium/src/parser/langium-parser.ts

@@ -240,7 +240,7 @@ export class LangiumParser extends AbstractLangiumParser {
             throw new Error(options.rule ? `No rule found with name '${options.rule}'` : 'No main rule available.');
         }
         const result = ruleMethod.call(this.wrapper, {});
-        this.appendHiddenTokens(lexerResult.hidden);
+        this.nodeBuilder.addHiddenNodes(lexerResult.hidden);
         this.unorderedGroups.clear();
         this.lexerResult = undefined;
         return {
@@ -275,12 +275,6 @@ export class LangiumParser extends AbstractLangiumParser {
         };
     }
 
-    private appendHiddenTokens(tokens: IToken[]): void {
-        for (const token of tokens) {
-            this.nodeBuilder.addHiddenNode(token);
-        }
-    }
-
     private getHiddenTokens(token: IToken): IToken[] {
         const hiddenTokens = this.lexerResult!.hidden;
         if (!hiddenTokens.length) {
@@ -300,7 +294,7 @@ export class LangiumParser extends AbstractLangiumParser {
         const token = this.wrapper.wrapConsume(idx, tokenType);
         if (!this.isRecording() && this.isValidToken(token)) {
             const hiddenTokens = this.getHiddenTokens(token);
-            this.appendHiddenTokens(hiddenTokens);
+            this.nodeBuilder.addHiddenNodes(hiddenTokens);
             const leafNode = this.nodeBuilder.buildLeafNode(token, feature);
             const { assignment, isCrossRef } = this.getAssignment(feature);
             const current = this.current;

packages/langium/test/utils/cst-utils.test.ts

@@ -85,11 +85,13 @@ describe('findCommentNode', () => {
     test('Finds correct comment at the start of the file', async () => {
         const text = expandToString`
         /** A */
+        /** B */
+        /** C */
         grammar test
         `;
         const grammar = await parser(text);
         const comment = CstUtils.findCommentNode(grammar.parseResult.value.$cstNode, ['ML_COMMENT']);
-        expect(comment?.text).toBe('/** A */');
+        expect(comment?.text).toBe('/** C */');
     });
 });