TokenIterator - allow skipping TOKEN_PHPDOC_EOL anywhere for Doctrine annotations

Author: ondrejmirtes

src/Parser/TokenIterator.php

@@ -22,6 +22,9 @@ class TokenIterator
 	/** @var int[] */
 	private $savePoints = [];
 
+	/** @var list<int> */
+	private $skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
+
 	/**
 	 * @param list<array{string, int, int}> $tokens
 	 */
@@ -30,11 +33,7 @@ public function __construct(array $tokens, int $index = 0)
 		$this->tokens = $tokens;
 		$this->index = $index;
 
-		if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] !== Lexer::TOKEN_HORIZONTAL_WS) {
-			return;
-		}
-
-		$this->index++;
+		$this->skipIrrelevantTokens();
 	}
 
 
@@ -131,12 +130,7 @@ public function consumeTokenType(int $tokenType): void
 		}
 
 		$this->index++;
-
-		if (($this->tokens[$this->index][Lexer::TYPE_OFFSET] ?? -1) !== Lexer::TOKEN_HORIZONTAL_WS) {
-			return;
-		}
-
-		$this->index++;
+		$this->skipIrrelevantTokens();
 	}
 
 
@@ -150,12 +144,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
 		}
 
 		$this->index++;
-
-		if (($this->tokens[$this->index][Lexer::TYPE_OFFSET] ?? -1) !== Lexer::TOKEN_HORIZONTAL_WS) {
-			return;
-		}
-
-		$this->index++;
+		$this->skipIrrelevantTokens();
 	}
 
 
@@ -167,10 +156,7 @@ public function tryConsumeTokenValue(string $tokenValue): bool
 		}
 
 		$this->index++;
-
-		if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) {
-			$this->index++;
-		}
+		$this->skipIrrelevantTokens();
 
 		return true;
 	}
@@ -184,10 +170,7 @@ public function tryConsumeTokenType(int $tokenType): bool
 		}
 
 		$this->index++;
-
-		if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) {
-			$this->index++;
-		}
+		$this->skipIrrelevantTokens();
 
 		return true;
 	}
@@ -217,12 +200,34 @@ public function joinUntil(int ...$tokenType): string
 	public function next(): void
 	{
 		$this->index++;
+		$this->skipIrrelevantTokens();
+	}
 
-		if ($this->tokens[$this->index][Lexer::TYPE_OFFSET] !== Lexer::TOKEN_HORIZONTAL_WS) {
+
+	private function skipIrrelevantTokens(): void
+	{
+		if (!isset($this->tokens[$this->index])) {
 			return;
 		}
 
-		$this->index++;
+		while (in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $this->skippedTokenTypes, true)) {
+			if (!isset($this->tokens[$this->index + 1])) {
+				break;
+			}
+			$this->index++;
+		}
+	}
+
+
+	public function addEndOfLineToSkippedTokens(): void
+	{
+		$this->skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS, Lexer::TOKEN_PHPDOC_EOL];
+	}
+
+
+	public function removeEndOfLineFromSkippedTokens(): void
+	{
+		$this->skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
 	}
 
 	/** @phpstan-impure */