PhpDocParser: tags must be at the beginning of the line

Author: JanTvrdik

src/Ast/PhpDoc/PhpDocNode.php

@@ -142,7 +142,7 @@ public function getMethodTagValues(): array
 
 	public function __toString(): string
 	{
-		return '/**' . implode('', $this->children) . '*/';
+		return "/**\n * " . implode("\n * ", $this->children) . '*/';
 	}
 
 }

src/Lexer/Lexer.php

@@ -26,6 +26,7 @@ class Lexer
 	const TOKEN_OPEN_PHPDOC = 15;
 	const TOKEN_CLOSE_PHPDOC = 16;
 	const TOKEN_PHPDOC_TAG = 17;
+	const TOKEN_PHPDOC_EOL = 26;
 	const TOKEN_FLOAT = 18;
 	const TOKEN_INTEGER = 19;
 	const TOKEN_SINGLE_QUOTED_STRING = 20;
@@ -34,7 +35,6 @@ class Lexer
 	const TOKEN_THIS_VARIABLE = 23;
 	const TOKEN_VARIABLE = 24;
 	const TOKEN_HORIZONTAL_WS = 25;
-	const TOKEN_EOL = 26;
 	const TOKEN_OTHER = 27;
 	const TOKEN_END = 28;
 
@@ -65,7 +65,6 @@ class Lexer
 		self::TOKEN_THIS_VARIABLE => '\'$this\'',
 		self::TOKEN_VARIABLE => 'TOKEN_VARIABLE',
 		self::TOKEN_HORIZONTAL_WS => 'TOKEN_HORIZONTAL_WS',
-		self::TOKEN_EOL => 'TOKEN_EOL',
 		self::TOKEN_OTHER => 'TOKEN_OTHER',
 		self::TOKEN_END => 'TOKEN_END',
 	];
@@ -130,6 +129,7 @@ private function initialize()
 			self::TOKEN_OPEN_PHPDOC => '/\\*\\*',
 			self::TOKEN_CLOSE_PHPDOC => '\\*/',
 			self::TOKEN_PHPDOC_TAG => '@[a-z-]++',
+			self::TOKEN_PHPDOC_EOL => '\\r?+\\n\\h*+(?:\\*(?!/)\\h*+)?',
 
 			self::TOKEN_FLOAT => '(?:-?[0-9]++\\.[0-9]*+(?:e-?[0-9]++)?)|(?:-?[0-9]*+\\.[0-9]++(?:e-?[0-9]++)?)|(?:-?[0-9]++e-?[0-9]++)',
 			self::TOKEN_INTEGER => '-?[0-9]++',
@@ -141,10 +141,9 @@ private function initialize()
 			self::TOKEN_VARIABLE => '\\$[a-z_\\x7F-\\xFF][0-9a-z_\\x7F-\\xFF]*+',
 
 			self::TOKEN_HORIZONTAL_WS => '\\h++',
-			self::TOKEN_EOL => '(?:\\r?+\\n)++',
 
-			// anything but TOKEN_PHPDOC_TAG or TOKEN_CLOSE_PHPDOC or TOKEN_HORIZONTAL_WS or TOKEN_EOL
-			self::TOKEN_OTHER => '(?:(?!@[a-z-])(?!\\*/)[^\\s])++',
+			// anything but TOKEN_CLOSE_PHPDOC or TOKEN_HORIZONTAL_WS or TOKEN_EOL
+			self::TOKEN_OTHER => '(?:(?!\\*/)[^\\s])++',
 		];
 
 		$this->regexp = '~(' . implode(')|(', $patterns) . ')~Asi';

src/Parser/PhpDocParser.php

@@ -30,13 +30,15 @@ public function __construct(TypeParser $typeParser, ConstExprParser $constantExp
 	public function parse(TokenIterator $tokens): Ast\PhpDoc\PhpDocNode
 	{
 		$tokens->consumeTokenType(Lexer::TOKEN_OPEN_PHPDOC);
+		$tokens->tryConsumeTokenType(Lexer::TOKEN_PHPDOC_EOL);
 
 		$children = [];
-		$children[] = $this->parseText($tokens);
 
-		while (!$tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC) && !$tokens->isCurrentTokenType(Lexer::TOKEN_END)) {
-			$children[] = $this->parseTag($tokens);
-			$children[] = $this->parseText($tokens);
+		if (!$tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC)) {
+			$children[] = $this->parseChild($tokens);
+			while ($tokens->tryConsumeTokenType(Lexer::TOKEN_PHPDOC_EOL) && !$tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC)) {
+				$children[] = $this->parseChild($tokens);
+			}
 		}
 
 		$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_PHPDOC);
@@ -45,10 +47,21 @@ public function parse(TokenIterator $tokens): Ast\PhpDoc\PhpDocNode
 	}
 
 
+	private function parseChild(TokenIterator $tokens): Ast\PhpDoc\PhpDocChildNode
+	{
+		if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG)) {
+			return $this->parseTag($tokens);
+
+		} else {
+			return $this->parseText($tokens);
+		}
+	}
+
+
 	private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode
 	{
-		$text = $tokens->getSkippedHorizontalWhiteSpaceIfAny();
-		$text .= $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_CLOSE_PHPDOC);
+		$text = $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END);
+		$text = rtrim($text, " \t"); // the trimmed characters MUST match Lexer::TOKEN_HORIZONTAL_WS
 
 		return new Ast\PhpDoc\PhpDocTextNode($text);
 	}
@@ -223,6 +236,7 @@ private function parseOptionalVariableName(TokenIterator $tokens): string
 		return $parameterName;
 	}
 
+
 	private function parseRequiredVariableName(TokenIterator $tokens): string
 	{
 		$parameterName = $tokens->currentTokenValue();
@@ -242,14 +256,7 @@ private function parseOptionalDescription(TokenIterator $tokens, bool $limitStar
 			}
 		}
 
-		$description = $tokens->joinUntil(
-			Lexer::TOKEN_EOL,
-			Lexer::TOKEN_PHPDOC_TAG,
-			Lexer::TOKEN_CLOSE_PHPDOC
-		);
-
-		// the trimmed characters MUST match Lexer::TOKEN_HORIZONTAL_WS
-		return rtrim($description, " \t");
+		return $this->parseText($tokens)->text;
 	}
 
 }

src/Parser/TokenIterator.php

@@ -125,7 +125,7 @@ public function getSkippedHorizontalWhiteSpaceIfAny(): string
 	public function joinUntil(int ...$tokenType): string
 	{
 		$s = '';
-		while (isset($this->tokens[$this->index]) && !in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $tokenType, true)) {
+		while (!in_array($this->tokens[$this->index][Lexer::TYPE_OFFSET], $tokenType, true)) {
 			$s .= $this->tokens[$this->index++][Lexer::VALUE_OFFSET];
 		}
 		return $s;