Skip to content

Commit 64ed7e4

Browse files
JackWang032JackWang032
authored
fix: minimum collect candidates boundary to fix parse performance (#378)
* fix: minimum collect candidates boundary to fix parse performance * fix: fix check-types * fix: remove debugger code
1 parent c0a2854 commit 64ed7e4

File tree

17 files changed

+251
-123
lines changed

17 files changed

+251
-123
lines changed

‎scripts/benchmark.js‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,19 @@ function checkVersion() {
4343
if (semver.lt(currentVersion, MIN_VERSION)) {
4444
console.error(
4545
chalk.bold.red(
46-
`Current Node.js version (v${currentVersion}) is lower than required version (v${semver.major(MIN_VERSION)}.x)`
46+
`Current Node.js version (v${currentVersion}) is lower than required version (v${semver.major(
47+
MIN_VERSION
48+
)}.x)`
4749
)
4850
);
4951
return false;
5052
} else {
5153
if (isRelease && semver.lt(currentVersion, RELEASE_VERSION)) {
5254
console.error(
5355
chalk.bold.red(
54-
`Node.js version v${semver.major(RELEASE_VERSION)}.x+ is required for release benchmark!`
56+
`Node.js version v${semver.major(
57+
RELEASE_VERSION
58+
)}.x+ is required for release benchmark!`
5559
)
5660
);
5761
return false;
@@ -81,7 +85,9 @@ function prompt() {
8185
'Cold start' +
8286
(isNodeVersionOk
8387
? ''
84-
: ` (Only supported on Node.js v${semver.major(RECOMMENDED_VERSION)}.x+)`),
88+
: ` (Only supported on Node.js v${semver.major(
89+
RECOMMENDED_VERSION
90+
)}.x+)`),
8591
value: 'cold',
8692
disabled: !isNodeVersionOk,
8793
},

‎src/parser/common/basicSQL.ts‎

Lines changed: 69 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,13 @@ export abstract class BasicSQL<
203203
return this._parseErrors;
204204
}
205205

206+
/**
207+
* Get the input string that has been parsed.
208+
*/
209+
public getParsedInput(): string {
210+
return this._parsedInput;
211+
}
212+
206213
/**
207214
* Get all Tokens of input string，'<EOF>' is not included.
208215
* @param input source string
@@ -252,35 +259,35 @@ export abstract class BasicSQL<
252259
}
253260

254261
/**
255-
* Get suggestions of syntax and token at caretPosition
256-
* @param input source string
257-
* @param caretPosition caret position, such as cursor position
258-
* @returns suggestion
262+
* Get a minimum boundary parser near tokenIndex.
263+
* @param input source string.
264+
* @param tokenIndex start from which index to minimize the boundary.
265+
* @param originParseTree the parse tree need to be minimized, default value is the result of parsing `input`.
266+
* @returns minimum parser info
259267
*/
260-
public getSuggestionAtCaretPosition(
268+
public getMinimumParserInfo(
261269
input: string,
262-
caretPosition: CaretPosition
263-
): Suggestions | null {
264-
const splitListener = this.splitListener;
265-
266-
this.parseWithCache(input);
267-
if (!this._parseTree) return null;
268-
269-
let sqlParserIns = this._parser;
270-
const allTokens = this.getAllTokens(input);
271-
let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens);
272-
let c3Context: ParserRuleContext = this._parseTree;
273-
let tokenIndexOffset: number = 0;
270+
tokenIndex: number,
271+
originParseTree?: ParserRuleContext | null
272+
) {
273+
if (arguments.length <= 2) {
274+
this.parseWithCache(input);
275+
originParseTree = this._parseTree;
276+
}
274277

275-
if (!caretTokenIndex && caretTokenIndex !== 0) return null;
278+
if (!originParseTree || !input?.length) return null;
276279

280+
const splitListener = this.splitListener;
277281
/**
278282
* Split sql by statement.
279283
* Try to collect candidates in as small a range as possible.
280284
*/
281-
this.listen(splitListener, this._parseTree);
285+
this.listen(splitListener, originParseTree);
282286
const statementCount = splitListener.statementsContext?.length;
283287
const statementsContext = splitListener.statementsContext;
288+
let tokenIndexOffset = 0;
289+
let sqlParserIns = this._parser;
290+
let parseTree = originParseTree;
284291

285292
// If there are multiple statements.
286293
if (statementCount > 1) {
@@ -305,14 +312,14 @@ export abstract class BasicSQL<
305312
const isNextCtxValid =
306313
index === statementCount - 1 || !statementsContext[index + 1]?.exception;
307314

308-
if (ctx.stop && ctx.stop.tokenIndex < caretTokenIndex && isPrevCtxValid) {
315+
if (ctx.stop && ctx.stop.tokenIndex < tokenIndex && isPrevCtxValid) {
309316
startStatement = ctx;
310317
}
311318

312319
if (
313320
ctx.start &&
314321
!stopStatement &&
315-
ctx.start.tokenIndex > caretTokenIndex &&
322+
ctx.start.tokenIndex > tokenIndex &&
316323
isNextCtxValid
317324
) {
318325
stopStatement = ctx;
@@ -329,7 +336,7 @@ export abstract class BasicSQL<
329336
* compared to the tokenIndex in the whole input
330337
*/
331338
tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0;
332-
caretTokenIndex = caretTokenIndex - tokenIndexOffset;
339+
tokenIndex = tokenIndex - tokenIndexOffset;
333340

334341
/**
335342
* Reparse the input fragment，
@@ -349,17 +356,54 @@ export abstract class BasicSQL<
349356
parser.errorHandler = new ErrorStrategy();
350357

351358
sqlParserIns = parser;
352-
c3Context = parser.program();
359+
parseTree = parser.program();
353360
}
354361

362+
return {
363+
parser: sqlParserIns,
364+
parseTree,
365+
tokenIndexOffset,
366+
newTokenIndex: tokenIndex,
367+
};
368+
}
369+
370+
/**
371+
* Get suggestions of syntax and token at caretPosition
372+
* @param input source string
373+
* @param caretPosition caret position, such as cursor position
374+
* @returns suggestion
375+
*/
376+
public getSuggestionAtCaretPosition(
377+
input: string,
378+
caretPosition: CaretPosition
379+
): Suggestions | null {
380+
this.parseWithCache(input);
381+
382+
if (!this._parseTree) return null;
383+
384+
const allTokens = this.getAllTokens(input);
385+
let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens);
386+
387+
if (!caretTokenIndex && caretTokenIndex !== 0) return null;
388+
389+
const minimumParser = this.getMinimumParserInfo(input, caretTokenIndex);
390+
391+
if (!minimumParser) return null;
392+
393+
const {
394+
parser: sqlParserIns,
395+
tokenIndexOffset,
396+
newTokenIndex,
397+
parseTree: c3Context,
398+
} = minimumParser;
355399
const core = new CodeCompletionCore(sqlParserIns);
356400
core.preferredRules = this.preferredRules;
357401

358-
const candidates = core.collectCandidates(caretTokenIndex, c3Context);
402+
const candidates = core.collectCandidates(newTokenIndex, c3Context);
359403
const originalSuggestions = this.processCandidates(
360404
candidates,
361405
allTokens,
362-
caretTokenIndex,
406+
newTokenIndex,
363407
tokenIndexOffset
364408
);
365409

‎src/parser/common/parseErrorListener.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import {
1010
InputMismatchException,
1111
NoViableAltException,
1212
} from 'antlr4ng';
13-
import { LOCALE_TYPE } from './types';
1413
import { transform } from './transform';
14+
import { BasicSQL } from './basicSQL';
1515

1616
/**
1717
* Converted from {@link SyntaxError}.
@@ -48,10 +48,19 @@ export type ErrorListener = (parseError: ParseError, originalError: SyntaxError)
4848

4949
export abstract class ParseErrorListener implements ANTLRErrorListener {
5050
private _errorListener: ErrorListener;
51-
private locale: LOCALE_TYPE;
51+
protected preferredRules: Set<number>;
52+
protected get locale() {
53+
return this.parserContext.locale;
54+
}
55+
protected parserContext: BasicSQL;
5256

53-
constructor(errorListener: ErrorListener, locale: LOCALE_TYPE = 'en_US') {
54-
this.locale = locale;
57+
constructor(
58+
errorListener: ErrorListener,
59+
parserContext: BasicSQL,
60+
preferredRules: Set<number>
61+
) {
62+
this.parserContext = parserContext;
63+
this.preferredRules = preferredRules;
5564
this._errorListener = errorListener;
5665
}
5766

‎src/parser/flink/flinkErrorListener.ts‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
import { CodeCompletionCore } from 'antlr4-c3';
2-
import { ErrorListener, ParseErrorListener } from '../common/parseErrorListener';
2+
import { ParseErrorListener } from '../common/parseErrorListener';
33
import { Parser, Token } from 'antlr4ng';
44
import { FlinkSqlParser } from '../../lib/flink/FlinkSqlParser';
5-
import { LOCALE_TYPE } from '../common/types';
65

76
export class FlinkErrorListener extends ParseErrorListener {
8-
private preferredRules: Set<number>;
9-
107
private objectNames: Map<number, string> = new Map([
118
[FlinkSqlParser.RULE_catalogPath, 'catalog'],
129
[FlinkSqlParser.RULE_catalogPathCreate, 'catalog'],
@@ -22,22 +19,34 @@ export class FlinkErrorListener extends ParseErrorListener {
2219
[FlinkSqlParser.RULE_columnNameCreate, 'column'],
2320
]);
2421

25-
constructor(errorListener: ErrorListener, preferredRules: Set<number>, locale: LOCALE_TYPE) {
26-
super(errorListener, locale);
27-
this.preferredRules = preferredRules;
28-
}
29-
3022
public getExpectedText(parser: Parser, token: Token) {
3123
let expectedText = '';
24+
const input = this.parserContext.getParsedInput();
3225

26+
/**
27+
* Get the program context.
28+
* When called error listener, `this._parseTree` is still `undefined`,
29+
* so we can't use cached parseTree in `getMinimumParserInfo`
30+
*/
3331
let currentContext = parser.context ?? undefined;
3432
while (currentContext?.parent) {
3533
currentContext = currentContext.parent;
3634
}
3735

38-
const core = new CodeCompletionCore(parser);
36+
const parserInfo = this.parserContext.getMinimumParserInfo(
37+
input,
38+
token.tokenIndex,
39+
currentContext
40+
);
41+
42+
if (!parserInfo) return '';
43+
44+
const { parser: c3Parser, newTokenIndex, parseTree: c3Context } = parserInfo;
45+
46+
const core = new CodeCompletionCore(c3Parser);
3947
core.preferredRules = this.preferredRules;
40-
const candidates = core.collectCandidates(token.tokenIndex, currentContext);
48+
49+
const candidates = core.collectCandidates(newTokenIndex, c3Context);
4150

4251
if (candidates.rules.size) {
4352
const result: string[] = [];

‎src/parser/flink/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
3939
return new FlinkSqlSplitListener();
4040
}
4141

42-
protected createErrorListener(_errorListener: ErrorListener) {
43-
return new FlinkErrorListener(_errorListener, this.preferredRules, this.locale);
42+
protected createErrorListener(_errorListener: ErrorListener): FlinkErrorListener {
43+
const parserContext = this;
44+
return new FlinkErrorListener(_errorListener, parserContext, this.preferredRules);
4445
}
4546

4647
protected createEntityCollector(input: string, caretTokenIndex?: number) {

‎src/parser/hive/hiveErrorListener.ts‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
import { CodeCompletionCore } from 'antlr4-c3';
2-
import { ErrorListener, ParseErrorListener } from '../common/parseErrorListener';
2+
import { ParseErrorListener } from '../common/parseErrorListener';
33
import { Parser, Token } from 'antlr4ng';
44
import { HiveSqlParser } from '../../lib/hive/HiveSqlParser';
5-
import { LOCALE_TYPE } from '../common/types';
65

76
export class HiveErrorListener extends ParseErrorListener {
8-
private preferredRules: Set<number>;
9-
107
private objectNames: Map<number, string> = new Map([
118
[HiveSqlParser.RULE_dbSchemaName, 'database'],
129
[HiveSqlParser.RULE_dbSchemaNameCreate, 'database'],
@@ -21,22 +18,34 @@ export class HiveErrorListener extends ParseErrorListener {
2118
[HiveSqlParser.RULE_columnNameCreate, 'column'],
2219
]);
2320

24-
constructor(errorListener: ErrorListener, preferredRules: Set<number>, locale: LOCALE_TYPE) {
25-
super(errorListener, locale);
26-
this.preferredRules = preferredRules;
27-
}
28-
2921
public getExpectedText(parser: Parser, token: Token) {
3022
let expectedText = '';
23+
const input = this.parserContext.getParsedInput();
3124

25+
/**
26+
* Get the program context.
27+
* When called error listener, `this._parseTree` is still `undefined`,
28+
* so we can't use cached parseTree in `getMinimumParserInfo`
29+
*/
3230
let currentContext = parser.context ?? undefined;
3331
while (currentContext?.parent) {
3432
currentContext = currentContext.parent;
3533
}
3634

37-
const core = new CodeCompletionCore(parser);
35+
const parserInfo = this.parserContext.getMinimumParserInfo(
36+
input,
37+
token.tokenIndex,
38+
currentContext
39+
);
40+
41+
if (!parserInfo) return '';
42+
43+
const { parser: c3Parser, newTokenIndex, parseTree: c3Context } = parserInfo;
44+
45+
const core = new CodeCompletionCore(c3Parser);
3846
core.preferredRules = this.preferredRules;
39-
const candidates = core.collectCandidates(token.tokenIndex, currentContext);
47+
48+
const candidates = core.collectCandidates(newTokenIndex, c3Context);
4049

4150
if (candidates.rules.size) {
4251
const result: string[] = [];

‎src/parser/hive/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@ export class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, HiveSqlParse
4040
return new HiveSqlSplitListener();
4141
}
4242

43-
protected createErrorListener(_errorListener: ErrorListener) {
44-
return new HiveErrorListener(_errorListener, this.preferredRules, this.locale);
43+
protected createErrorListener(_errorListener: ErrorListener): HiveErrorListener {
44+
const parserContext = this;
45+
return new HiveErrorListener(_errorListener, parserContext, this.preferredRules);
4546
}
4647

4748
protected createEntityCollector(input: string, caretTokenIndex?: number) {

0 commit comments

Comments
 (0)