No {Ignored} tokens when parsing schema coordinates (#4450)

Author: magicmark

src/index.ts

@@ -230,6 +230,7 @@ export {
   printSourceLocation,
   // Lex
   Lexer,
+  SchemaCoordinateLexer,
   TokenKind,
   // Parse
   parse,
@@ -261,6 +262,7 @@ export {
 
 export type {
   ParseOptions,
+  ParseSchemaCoordinateOptions,
   SourceLocation,
   // Visitor utilities
   ASTVisitor,

src/language/__tests__/lexer-test.ts

@@ -9,7 +9,11 @@ import { inspect } from '../../jsutils/inspect.js';
 import { GraphQLError } from '../../error/GraphQLError.js';
 
 import type { Token } from '../ast.js';
-import { isPunctuatorTokenKind, Lexer } from '../lexer.js';
+import {
+  isPunctuatorTokenKind,
+  Lexer,
+  SchemaCoordinateLexer,
+} from '../lexer.js';
 import { Source } from '../source.js';
 import { TokenKind } from '../tokenKind.js';
 
@@ -1189,6 +1193,33 @@ describe('Lexer', () => {
   });
 });
 
+describe('SchemaCoordinateLexer', () => {
+  it('can be stringified', () => {
+    const lexer = new SchemaCoordinateLexer(new Source('Name.field'));
+    expect(Object.prototype.toString.call(lexer)).to.equal(
+      '[object SchemaCoordinateLexer]',
+    );
+  });
+
+  it('tracks a schema coordinate', () => {
+    const lexer = new SchemaCoordinateLexer(new Source('Name.field'));
+    expect(lexer.advance()).to.contain({
+      kind: TokenKind.NAME,
+      start: 0,
+      end: 4,
+      value: 'Name',
+    });
+  });
+
+  it('forbids ignored tokens', () => {
+    const lexer = new SchemaCoordinateLexer(new Source('\nName.field'));
+    expectToThrowJSON(() => lexer.advance()).to.deep.equal({
+      message: 'Syntax Error: Invalid character: U+000A.',
+      locations: [{ line: 1, column: 1 }],
+    });
+  });
+});
+
 describe('isPunctuatorTokenKind', () => {
   function isPunctuatorToken(text: string) {
     return isPunctuatorTokenKind(lexOne(text).kind);

src/language/__tests__/parser-test.ts

@@ -751,11 +751,11 @@ describe('Parser', () => {
     });
 
     it('rejects Name . Name ( Name : Name )', () => {
-      expect(() => parseSchemaCoordinate('MyType.field(arg: value)'))
+      expect(() => parseSchemaCoordinate('MyType.field(arg:value)'))
         .to.throw()
         .to.deep.include({
           message: 'Syntax Error: Expected ")", found Name "value".',
-          locations: [{ line: 1, column: 19 }],
+          locations: [{ line: 1, column: 18 }],
         });
     });
 

src/language/__tests__/printer-test.ts

@@ -301,16 +301,24 @@ describe('Printer: Query document', () => {
   });
 
   it('prints schema coordinates', () => {
-    expect(print(parseSchemaCoordinate('  Name  '))).to.equal('Name');
-    expect(print(parseSchemaCoordinate('  Name . field '))).to.equal(
-      'Name.field',
-    );
-    expect(print(parseSchemaCoordinate('  Name . field ( arg: )'))).to.equal(
+    expect(print(parseSchemaCoordinate('Name'))).to.equal('Name');
+    expect(print(parseSchemaCoordinate('Name.field'))).to.equal('Name.field');
+    expect(print(parseSchemaCoordinate('Name.field(arg:)'))).to.equal(
       'Name.field(arg:)',
     );
-    expect(print(parseSchemaCoordinate(' @ name  '))).to.equal('@name');
-    expect(print(parseSchemaCoordinate(' @ name (arg:) '))).to.equal(
-      '@name(arg:)',
+    expect(print(parseSchemaCoordinate('@name'))).to.equal('@name');
+    expect(print(parseSchemaCoordinate('@name(arg:)'))).to.equal('@name(arg:)');
+  });
+
+  it('throws syntax error for ignored tokens in schema coordinates', () => {
+    expect(() => print(parseSchemaCoordinate('# foo\nName'))).to.throw(
+      'Syntax Error: Invalid character: "#"',
+    );
+    expect(() => print(parseSchemaCoordinate('\nName'))).to.throw(
+      'Syntax Error: Invalid character: U+000A.',
+    );
+    expect(() => print(parseSchemaCoordinate('Name .field'))).to.throw(
+      'Syntax Error: Invalid character: " "',
     );
   });
 });

src/language/index.ts

@@ -11,7 +11,7 @@ export { Kind } from './kinds.js';
 
 export { TokenKind } from './tokenKind.js';
 
-export { Lexer } from './lexer.js';
+export { Lexer, SchemaCoordinateLexer } from './lexer.js';
 
 export {
   parse,
@@ -20,7 +20,7 @@ export {
   parseType,
   parseSchemaCoordinate,
 } from './parser.js';
-export type { ParseOptions } from './parser.js';
+export type { ParseOptions, ParseSchemaCoordinateOptions } from './parser.js';
 
 export { print } from './printer.js';
 

src/language/lexer.ts

@@ -83,6 +83,27 @@ export class Lexer {
     }
     return token;
   }
+
+  validateIgnoredToken(_position: number): void {
+    /* noop - ignored tokens are ignored */
+  }
+}
+
+/**
+ * As `Lexer`, but forbids ignored tokens as required of schema coordinates.
+ */
+export class SchemaCoordinateLexer extends Lexer {
+  override get [Symbol.toStringTag]() {
+    return 'SchemaCoordinateLexer';
+  }
+
+  override validateIgnoredToken(position: number): void {
+    throw syntaxError(
+      this.source,
+      position,
+      `Invalid character: ${printCodePointAt(this, position)}.`,
+    );
+  }
 }
 
 /**
@@ -217,18 +238,21 @@ function readNextToken(lexer: Lexer, start: number): Token {
       case 0x0009: // \t
       case 0x0020: // <space>
       case 0x002c: // ,
+        lexer.validateIgnoredToken(position);
         ++position;
         continue;
       // LineTerminator ::
       //   - "New Line (U+000A)"
       //   - "Carriage Return (U+000D)" [lookahead != "New Line (U+000A)"]
       //   - "Carriage Return (U+000D)" "New Line (U+000A)"
       case 0x000a: // \n
+        lexer.validateIgnoredToken(position);
         ++position;
         ++lexer.line;
         lexer.lineStart = position;
         continue;
       case 0x000d: // \r
+        lexer.validateIgnoredToken(position);
         if (body.charCodeAt(position + 1) === 0x000a) {
           position += 2;
         } else {
@@ -239,6 +263,7 @@ function readNextToken(lexer: Lexer, start: number): Token {
         continue;
       // Comment
       case 0x0023: // #
+        lexer.validateIgnoredToken(position);
         return readComment(lexer, position);
       // Token ::
       //   - Punctuator

src/language/parser.ts

@@ -70,7 +70,11 @@ import type {
 import { Location, OperationTypeNode } from './ast.js';
 import { DirectiveLocation } from './directiveLocation.js';
 import { Kind } from './kinds.js';
-import { isPunctuatorTokenKind, Lexer } from './lexer.js';
+import {
+  isPunctuatorTokenKind,
+  Lexer,
+  SchemaCoordinateLexer,
+} from './lexer.js';
 import { isSource, Source } from './source.js';
 import { TokenKind } from './tokenKind.js';
 
@@ -114,6 +118,24 @@ export interface ParseOptions {
    * ```
    */
   experimentalFragmentArguments?: boolean | undefined;
+
+  /**
+   * You may override the Lexer class used to lex the source; this is used by
+   * schema coordinates to introduce a lexer that forbids ignored tokens.
+   */
+  Lexer?: typeof Lexer | undefined;
+}
+
+/**
+ * Configuration options to control schema coordinate parser behavior
+ */
+export interface ParseSchemaCoordinateOptions {
+  /**
+   * By default, the parser creates AST nodes that know the location
+   * in the source that they correspond to. This configuration flag
+   * disables that behavior for performance or testing.
+   */
+  noLocation?: boolean | undefined;
 }
 
 /**
@@ -199,9 +221,13 @@ export function parseType(
  */
 export function parseSchemaCoordinate(
   source: string | Source,
-  options?: ParseOptions,
+  options?: ParseSchemaCoordinateOptions,
 ): SchemaCoordinateNode {
-  const parser = new Parser(source, options);
+  // Ignored tokens are excluded syntax for a Schema Coordinate.
+  const parser = new Parser(source, {
+    ...options,
+    Lexer: SchemaCoordinateLexer,
+  });
   parser.expectToken(TokenKind.SOF);
   const coordinate = parser.parseSchemaCoordinate();
   parser.expectToken(TokenKind.EOF);
@@ -227,7 +253,8 @@ export class Parser {
   constructor(source: string | Source, options: ParseOptions = {}) {
     const sourceObj = isSource(source) ? source : new Source(source);
 
-    this._lexer = new Lexer(sourceObj);
+    const LexerClass = options.Lexer ?? Lexer;
+    this._lexer = new LexerClass(sourceObj);
     this._options = options;
     this._tokenCounter = 0;
   }