Fixes tests.

Author: hediet

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/beforeEditPositionMapper.ts

@@ -26,7 +26,6 @@ export class BeforeEditPositionMapper {
 	*/
 	constructor(
 		edits: readonly TextEditInfo[],
-		private readonly documentLength: Length,
 	) {
 		this.edits = edits.map(edit => TextEditInfoCache.from(edit));
 	}
@@ -41,12 +40,16 @@ export class BeforeEditPositionMapper {
 
 	/**
 	 * @param offset Must be equal to or greater than the last offset this method has been called with.
+	 * Returns null if there is no edit anymore.
 	*/
-	getDistanceToNextChange(offset: Length): Length {
+	getDistanceToNextChange(offset: Length): Length | null {
 		this.adjustNextEdit(offset);
 
 		const nextEdit = this.edits[this.nextEditIdx];
-		const nextChangeOffset = nextEdit ? this.translateOldToCur(nextEdit.offsetObj) : this.documentLength;
+		const nextChangeOffset = nextEdit ? this.translateOldToCur(nextEdit.offsetObj) : null;
+		if (nextChangeOffset === null) {
+			return null;
+		}
 
 		return lengthDiffNonNegative(offset, nextChangeOffset);
 	}

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts

@@ -131,7 +131,7 @@ export class BracketPairsTree extends Disposable {
 			this.astWithTokens = this.parseDocumentFromTextBuffer(this.queuedTextEdits, this.astWithTokens, false);
 			this.queuedTextEdits = [];
 		}
-		if (this.queuedTextEditsForInitialAstWithoutTokens) {
+		if (this.queuedTextEditsForInitialAstWithoutTokens.length > 0) {
 			if (this.initialAstWithoutTokens) {
 				this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer(this.queuedTextEditsForInitialAstWithoutTokens, this.initialAstWithoutTokens, false);
 			}

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts

@@ -53,7 +53,7 @@ class Parser {
 		}
 
 		this.oldNodeReader = oldNode ? new NodeReader(oldNode) : undefined;
-		this.positionMapper = new BeforeEditPositionMapper(edits, tokenizer.length);
+		this.positionMapper = new BeforeEditPositionMapper(edits);
 	}
 
 	parseDocument(): AstNode {
@@ -104,9 +104,11 @@ class Parser {
 	private tryReadChildFromCache(openedBracketIds: SmallImmutableSet<number>): AstNode | undefined {
 		if (this.oldNodeReader) {
 			const maxCacheableLength = this.positionMapper.getDistanceToNextChange(this.tokenizer.offset);
-			if (!lengthIsZero(maxCacheableLength)) {
+			if (maxCacheableLength === null || !lengthIsZero(maxCacheableLength)) {
 				const cachedNode = this.oldNodeReader.readLongestNodeAt(this.positionMapper.getOffsetBeforeChange(this.tokenizer.offset), curNode => {
-					if (!lengthLessThan(curNode.length, maxCacheableLength)) {
+					// The edit could extend the ending token, thus we cannot re-use nodes that touch the edit.
+					// If there is no edit anymore, we can re-use the node in any case.
+					if (maxCacheableLength !== null && !lengthLessThan(curNode.length, maxCacheableLength)) {
 						// Either the node contains edited text or touches edited text.
 						// In the latter case, brackets might have been extended (`end` -> `ending`), so even touching nodes cannot be reused.
 						return false;

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts

@@ -81,7 +81,7 @@ export class TextBufferTokenizer implements Tokenizer {
 	}
 
 	get length() {
-		return toLength(this.textBufferLineCount, this.textBufferLastLineLength);
+		return toLength(this.textBufferLineCount - 1, this.textBufferLastLineLength);
 	}
 
 	getText() {

src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts

@@ -11,13 +11,27 @@ import { BracketPairInfo } from 'vs/editor/common/textModelBracketPairs';
 import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry';
 import { createModelServices, instantiateTextModel } from 'vs/editor/test/common/testTextModel';
 import { TextModel } from 'vs/editor/common/model/textModel';
+import { TokenInfo, TokenizedDocument } from 'vs/editor/test/common/model/bracketPairColorizer/tokenizer.test';
+import { ILanguageService } from 'vs/editor/common/languages/language';
+import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes';
+import { TokenizationRegistry } from 'vs/editor/common/languages';
 
 suite('Bracket Pair Colorizer - getBracketPairsInRange', () => {
 
 	function createTextModelWithColorizedBracketPairs(store: DisposableStore, text: string): TextModel {
 		const languageId = 'testLanguage';
 		const instantiationService = createModelServices(store);
 		const languageConfigurationService = instantiationService.get(ILanguageConfigurationService);
+		const languageService = instantiationService.get(ILanguageService);
+		languageService.registerLanguage({
+			id: languageId,
+		});
+
+		const encodedMode1 = languageService.languageIdCodec.encodeLanguageId(languageId);
+		const document = new TokenizedDocument([
+			new TokenInfo(text, encodedMode1, StandardTokenType.Other, true)
+		]);
+		store.add(TokenizationRegistry.register(languageId, document.getTokenizationSupport()));
 
 		store.add(languageConfigurationService.register(languageId, {
 			colorizedBracketPairs: [
@@ -26,13 +40,15 @@ suite('Bracket Pair Colorizer - getBracketPairsInRange', () => {
 				['(', ')'],
 			]
 		}));
-		return store.add(instantiateTextModel(instantiationService, text, languageId));
+		const textModel = store.add(instantiateTextModel(instantiationService, text, languageId));
+		return textModel;
 	}
 
 	test('Basic 1', () => {
 		disposeOnReturn(store => {
 			const doc = new AnnotatedDocument(`{ ( [] ¹ ) [ ² { } ] () } []`);
 			const model = createTextModelWithColorizedBracketPairs(store, doc.text);
+			model.tokenization.getLineTokens(1).getLanguageId(0);
 			assert.deepStrictEqual(
 				model.bracketPairs
 					.getBracketPairsInRange(doc.range(1, 2))

src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts

@@ -124,7 +124,7 @@ function tokenToObj(token: Token, offset: Length, model: TextModel, keyProvider:
 	};
 }
 
-class TokenizedDocument {
+export class TokenizedDocument {
 	private readonly tokensByLine: readonly TokenInfo[][];
 	constructor(tokens: TokenInfo[]) {
 		const tokensByLine = new Array<TokenInfo[]>();
@@ -189,7 +189,7 @@ class TokenizedDocument {
 	}
 }
 
-class TokenInfo {
+export class TokenInfo {
 	constructor(
 		public readonly text: string,
 		public readonly languageId: LanguageId,