From 2108d702baa4883362a8824def66b96733b8cf82 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 00:39:25 +0000 Subject: [PATCH 01/10] Update actions/upload-pages-artifact action to v2 --- .github/workflows/apiref.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/apiref.yml b/.github/workflows/apiref.yml index 4b3abbb0..7adcb330 100644 --- a/.github/workflows/apiref.yml +++ b/.github/workflows/apiref.yml @@ -42,7 +42,7 @@ jobs: run: "cp apigen/favicon.png docs/favicon.png" - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v2 with: path: 'docs' From 1c9f85331cf81954605a243269823cc2743efbfa Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Thu, 20 Jul 2023 17:16:15 +0200 Subject: [PATCH 02/10] Open 1.23.x-dev --- .github/workflows/apiref.yml | 2 +- .github/workflows/backward-compatibility.yml | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/test-slevomat-coding-standard.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/apiref.yml b/.github/workflows/apiref.yml index 7adcb330..d3d55f98 100644 --- a/.github/workflows/apiref.yml +++ b/.github/workflows/apiref.yml @@ -5,7 +5,7 @@ name: "Deploy API Reference" on: push: branches: - - "1.22.x" + - "1.23.x" concurrency: group: "pages" diff --git a/.github/workflows/backward-compatibility.yml b/.github/workflows/backward-compatibility.yml index 103e3470..eb78a350 100644 --- a/.github/workflows/backward-compatibility.yml +++ b/.github/workflows/backward-compatibility.yml @@ -6,7 +6,7 @@ on: pull_request: push: branches: - - "1.22.x" + - "1.23.x" jobs: backward-compatibility: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 658c2de1..61dd4466 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ on: pull_request: push: branches: - - "1.22.x" + - "1.23.x" jobs: lint: diff --git a/.github/workflows/test-slevomat-coding-standard.yml b/.github/workflows/test-slevomat-coding-standard.yml index f8bc2912..abd90452 100644 --- a/.github/workflows/test-slevomat-coding-standard.yml +++ b/.github/workflows/test-slevomat-coding-standard.yml @@ -6,7 +6,7 @@ on: pull_request: push: branches: - - "1.22.x" + - "1.23.x" jobs: tests: From ebf9e48d610416cdf05794d78682251ae6df0c4d Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Thu, 20 Jul 2023 17:17:55 +0200 Subject: [PATCH 03/10] Update merge-maintained-branch.yml --- .github/workflows/merge-maintained-branch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge-maintained-branch.yml b/.github/workflows/merge-maintained-branch.yml index 6ea17262..3aa2b0b3 100644 --- a/.github/workflows/merge-maintained-branch.yml +++ b/.github/workflows/merge-maintained-branch.yml @@ -5,7 +5,7 @@ name: Merge maintained branch on: push: branches: - - "1.21.x" + - "1.22.x" jobs: merge: @@ -19,5 +19,5 @@ jobs: with: github_token: "${{ secrets.PHPSTAN_BOT_TOKEN }}" source_ref: ${{ github.ref }} - target_branch: '1.22.x' + target_branch: '1.23.x' commit_message_template: 'Merge branch {source_ref} into {target_branch}' From 395b6dbd6163d2260686ce3f237134edda4ba0dc Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Fri, 21 Jul 2023 15:41:32 +0200 Subject: [PATCH 04/10] Document what parseText is doing --- src/Parser/PhpDocParser.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Parser/PhpDocParser.php b/src/Parser/PhpDocParser.php index c92b8093..09171294 100644 --- a/src/Parser/PhpDocParser.php +++ b/src/Parser/PhpDocParser.php @@ -214,9 +214,11 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode { $text = ''; + // if the next token is EOL, everything below is skipped and empty string is returned while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END); + // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { break; } @@ -224,11 +226,15 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode $tokens->pushSavePoint(); $tokens->next(); + // if we're at EOL, check what's next + // if next is a PHPDoc tag, EOL, or end of PHPDoc, stop if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END)) { $tokens->rollback(); break; } + // otherwise if the next is text, continue building the description string + $tokens->dropSavePoint(); $text .= $tokens->getDetectedNewline() ?? "\n"; } @@ -241,9 +247,11 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) { $text = ''; + // if the next token is EOL, everything below is skipped and empty string is returned while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END); + // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { if (!$tokens->isPrecededByHorizontalWhitespace()) { return trim($text . $this->parseText($tokens)->text, " \t"); @@ -281,11 +289,15 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) $tokens->pushSavePoint(); $tokens->next(); + // if we're at EOL, check what's next + // if next is a PHPDoc tag, EOL, or end of PHPDoc, stop if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END)) { $tokens->rollback(); break; } + // otherwise if the next is text, continue building the description string + $tokens->dropSavePoint(); $text .= $tokens->getDetectedNewline() ?? "\n"; } From f9311f0a7e9b6f8c55d533870b90a74194671e15 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 23 Jul 2023 16:24:20 +0200 Subject: [PATCH 05/10] Small refactoring --- src/Parser/PhpDocParser.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Parser/PhpDocParser.php b/src/Parser/PhpDocParser.php index 09171294..92804c57 100644 --- a/src/Parser/PhpDocParser.php +++ b/src/Parser/PhpDocParser.php @@ -214,9 +214,11 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode { $text = ''; + $endTokens = [Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + // if the next token is EOL, everything below is skipped and empty string is returned while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END); + $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(...$endTokens); // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { @@ -228,7 +230,7 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode // if we're at EOL, check what's next // if next is a PHPDoc tag, EOL, or end of PHPDoc, stop - if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END)) { + if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, ...$endTokens)) { $tokens->rollback(); break; } @@ -247,9 +249,11 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) { $text = ''; + $endTokens = [Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + // if the next token is EOL, everything below is skipped and empty string is returned while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END); + $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, ...$endTokens); // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { @@ -291,7 +295,7 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) // if we're at EOL, check what's next // if next is a PHPDoc tag, EOL, or end of PHPDoc, stop - if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END)) { + if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, ...$endTokens)) { $tokens->rollback(); break; } From 5164f16dcfb1e86b506239d7d27f3cb25cbbca91 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Fri, 21 Jul 2023 15:41:44 +0200 Subject: [PATCH 06/10] New option to attach text between tags as description to the tag above --- src/Parser/PhpDocParser.php | 21 +++-- tests/PHPStan/Parser/PhpDocParserTest.php | 99 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 5 deletions(-) diff --git a/src/Parser/PhpDocParser.php b/src/Parser/PhpDocParser.php index 92804c57..c407c8b0 100644 --- a/src/Parser/PhpDocParser.php +++ b/src/Parser/PhpDocParser.php @@ -49,6 +49,9 @@ class PhpDocParser /** @var bool */ private $useIndexAttributes; + /** @var bool */ + private $textBetweenTagsBelongsToDescription; + /** * @param array{lines?: bool, indexes?: bool} $usedAttributes */ @@ -58,7 +61,8 @@ public function __construct( bool $requireWhitespaceBeforeDescription = false, bool $preserveTypeAliasesWithInvalidTypes = false, array $usedAttributes = [], - bool $parseDoctrineAnnotations = false + bool $parseDoctrineAnnotations = false, + bool $textBetweenTagsBelongsToDescription = false ) { $this->typeParser = $typeParser; @@ -68,6 +72,7 @@ public function __construct( $this->parseDoctrineAnnotations = $parseDoctrineAnnotations; $this->useLinesAttributes = $usedAttributes['lines'] ?? false; $this->useIndexAttributes = $usedAttributes['indexes'] ?? false; + $this->textBetweenTagsBelongsToDescription = $textBetweenTagsBelongsToDescription; } @@ -215,10 +220,13 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode $text = ''; $endTokens = [Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + if ($this->textBetweenTagsBelongsToDescription) { + $endTokens = [Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + } // if the next token is EOL, everything below is skipped and empty string is returned - while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(...$endTokens); + while ($this->textBetweenTagsBelongsToDescription || !$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { + $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { @@ -250,10 +258,13 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) $text = ''; $endTokens = [Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + if ($this->textBetweenTagsBelongsToDescription) { + $endTokens = [Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; + } // if the next token is EOL, everything below is skipped and empty string is returned - while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, ...$endTokens); + while ($this->textBetweenTagsBelongsToDescription || !$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { + $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { diff --git a/tests/PHPStan/Parser/PhpDocParserTest.php b/tests/PHPStan/Parser/PhpDocParserTest.php index 817ada88..32eb518b 100644 --- a/tests/PHPStan/Parser/PhpDocParserTest.php +++ b/tests/PHPStan/Parser/PhpDocParserTest.php @@ -6682,4 +6682,103 @@ public function testDoctrine( $this->assertEquals($expectedAnnotations, $parser->parse($input, $label), $label); } + /** + * @return iterable + */ + public function dataTextBetweenTagsBelongsToDescription(): iterable + { + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' * paramA description' . PHP_EOL . + ' * @param int $b' . PHP_EOL . + ' * paramB description' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' paramA description')), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$b', PHP_EOL . ' paramB description')), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a aaaa' . PHP_EOL . + ' * bbbb' . PHP_EOL . + ' *' . PHP_EOL . + ' * ccc' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', 'aaaa' . PHP_EOL . ' bbbb' . PHP_EOL . PHP_EOL . 'ccc')), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @ORM\Column()' . PHP_EOL . + ' * bbbb' . PHP_EOL . + ' *' . PHP_EOL . + ' * ccc' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@ORM\Column', new DoctrineTagValueNode(new DoctrineAnnotation('@ORM\Column', []), PHP_EOL . ' bbbb' . PHP_EOL . PHP_EOL . 'ccc')), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @ORM\Column() aaaa' . PHP_EOL . + ' * bbbb' . PHP_EOL . + ' *' . PHP_EOL . + ' * ccc' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@ORM\Column', new DoctrineTagValueNode(new DoctrineAnnotation('@ORM\Column', []), 'aaaa' . PHP_EOL . ' bbbb' . PHP_EOL . PHP_EOL . 'ccc')), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @ORM\Column() aaaa' . PHP_EOL . + ' * bbbb' . PHP_EOL . + ' *' . PHP_EOL . + ' * ccc' . PHP_EOL . + ' * @param int $b' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@ORM\Column', new DoctrineTagValueNode(new DoctrineAnnotation('@ORM\Column', []), 'aaaa' . PHP_EOL . ' bbbb' . PHP_EOL . PHP_EOL . 'ccc')), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$b', '')), + ]), + ]; + } + + /** + * @dataProvider dataTextBetweenTagsBelongsToDescription + */ + public function testTextBetweenTagsBelongsToDescription( + string $input, + PhpDocNode $expectedPhpDocNode + ): void + { + $constExprParser = new ConstExprParser(); + $typeParser = new TypeParser($constExprParser); + $phpDocParser = new PhpDocParser($typeParser, $constExprParser, true, true, [], true, true); + + $tokens = new TokenIterator($this->lexer->tokenize($input)); + $actualPhpDocNode = $phpDocParser->parse($tokens); + + $this->assertEquals($expectedPhpDocNode, $actualPhpDocNode); + $this->assertSame((string) $expectedPhpDocNode, (string) $actualPhpDocNode); + $this->assertSame(Lexer::TOKEN_END, $tokens->currentTokenType()); + } + } From c44a8e5f5da48bcdc5daf3b6839b5808316b44f9 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 23 Jul 2023 21:50:09 +0200 Subject: [PATCH 07/10] More tests --- tests/PHPStan/Parser/PhpDocParserTest.php | 40 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/PHPStan/Parser/PhpDocParserTest.php b/tests/PHPStan/Parser/PhpDocParserTest.php index 32eb518b..4446776b 100644 --- a/tests/PHPStan/Parser/PhpDocParserTest.php +++ b/tests/PHPStan/Parser/PhpDocParserTest.php @@ -2654,11 +2654,11 @@ public function provideSingleLinePhpDocData(): Iterator } /** - * @return array + * @return iterable> */ - public function provideMultiLinePhpDocData(): array + public function provideMultiLinePhpDocData(): iterable { - return [ + yield from [ [ 'multi-line with two tags', '/** @@ -3560,6 +3560,40 @@ public function provideMultiLinePhpDocData(): array ]), ], ]; + + yield [ + 'Empty lines before end', + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', '')), + new PhpDocTextNode(''), + new PhpDocTextNode(''), + ]), + ]; + + yield [ + 'Empty lines before end 2', + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' * test' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', '')), + new PhpDocTextNode(''), + new PhpDocTextNode(''), + new PhpDocTextNode('test'), + ]), + ]; } public function provideTemplateTagsData(): Iterator From e64b19323c35d19c8f075ab120639acae7b62348 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 23 Jul 2023 21:51:36 +0200 Subject: [PATCH 08/10] Fix trailing newlines in PHPDocs with textBetweenTagsBelongsToDescription=true --- src/Parser/PhpDocParser.php | 41 +++++- tests/PHPStan/Parser/PhpDocParserTest.php | 144 ++++++++++++++++++++++ 2 files changed, 183 insertions(+), 2 deletions(-) diff --git a/src/Parser/PhpDocParser.php b/src/Parser/PhpDocParser.php index c407c8b0..d31b0672 100644 --- a/src/Parser/PhpDocParser.php +++ b/src/Parser/PhpDocParser.php @@ -14,6 +14,7 @@ use function array_key_exists; use function array_values; use function count; +use function rtrim; use function str_replace; use function trim; @@ -224,15 +225,28 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode $endTokens = [Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; } + $savepoint = false; + // if the next token is EOL, everything below is skipped and empty string is returned while ($this->textBetweenTagsBelongsToDescription || !$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); + $tmpText = $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); + $text .= $tmpText; // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { break; } + if ($this->textBetweenTagsBelongsToDescription) { + if (!$savepoint) { + $tokens->pushSavePoint(); + $savepoint = true; + } elseif ($tmpText !== '') { + $tokens->dropSavePoint(); + $savepoint = false; + } + } + $tokens->pushSavePoint(); $tokens->next(); @@ -249,6 +263,11 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode $text .= $tokens->getDetectedNewline() ?? "\n"; } + if ($savepoint) { + $tokens->rollback(); + $text = rtrim($text, $tokens->getDetectedNewline() ?? "\n"); + } + return new Ast\PhpDoc\PhpDocTextNode(trim($text, " \t")); } @@ -262,9 +281,12 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) $endTokens = [Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END]; } + $savepoint = false; + // if the next token is EOL, everything below is skipped and empty string is returned while ($this->textBetweenTagsBelongsToDescription || !$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { - $text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); + $tmpText = $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_DOCTRINE_TAG, Lexer::TOKEN_PHPDOC_EOL, ...$endTokens); + $text .= $tmpText; // stop if we're not at EOL - meaning it's the end of PHPDoc if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) { @@ -301,6 +323,16 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) break; } + if ($this->textBetweenTagsBelongsToDescription) { + if (!$savepoint) { + $tokens->pushSavePoint(); + $savepoint = true; + } elseif ($tmpText !== '') { + $tokens->dropSavePoint(); + $savepoint = false; + } + } + $tokens->pushSavePoint(); $tokens->next(); @@ -317,6 +349,11 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) $text .= $tokens->getDetectedNewline() ?? "\n"; } + if ($savepoint) { + $tokens->rollback(); + $text = rtrim($text, $tokens->getDetectedNewline() ?? "\n"); + } + return trim($text, " \t"); } diff --git a/tests/PHPStan/Parser/PhpDocParserTest.php b/tests/PHPStan/Parser/PhpDocParserTest.php index 4446776b..7b74388d 100644 --- a/tests/PHPStan/Parser/PhpDocParserTest.php +++ b/tests/PHPStan/Parser/PhpDocParserTest.php @@ -6736,6 +6736,21 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable ]), ]; + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' *' . PHP_EOL . + ' * @param int $b' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', '')), + new PhpDocTextNode(''), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$b', '')), + ]), + ]; + yield [ '/**' . PHP_EOL . ' * Real description' . PHP_EOL . @@ -6793,6 +6808,135 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$b', '')), ]), ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', '')), + new PhpDocTextNode(''), + new PhpDocTextNode(''), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' * test' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . PHP_EOL . PHP_EOL . 'test')), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a test' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', 'test')), + new PhpDocTextNode(''), + new PhpDocTextNode(''), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' * test' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test')), + new PhpDocTextNode(''), + //new PhpDocTextNode(''), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' * test' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test')), + new PhpDocTextNode(''), + new PhpDocTextNode(''), + //new PhpDocTextNode(''), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' * test' . PHP_EOL . + ' *' . PHP_EOL . + ' * test 2' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), + //new PhpDocTextNode(''), + ]), + ]; + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a' . PHP_EOL . + ' * test' . PHP_EOL . + ' *' . PHP_EOL . + ' * test 2' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), + new PhpDocTextNode(''), + //new PhpDocTextNode(''), + ]), + ]; + + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @ORM\Column()' . PHP_EOL . + ' * test' . PHP_EOL . + ' *' . PHP_EOL . + ' * test 2' . PHP_EOL . + ' *' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@ORM\Column', new DoctrineTagValueNode(new DoctrineAnnotation('@ORM\Column', []), PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), + new PhpDocTextNode(''), + //new PhpDocTextNode(''), + ]), + ]; } /** From fa451018528fb6ebd4e8e9eea0f8c74a9bc9bc12 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 23 Jul 2023 23:45:19 +0200 Subject: [PATCH 09/10] One more test --- tests/PHPStan/Parser/PhpDocParserTest.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/PHPStan/Parser/PhpDocParserTest.php b/tests/PHPStan/Parser/PhpDocParserTest.php index 7b74388d..d911ec17 100644 --- a/tests/PHPStan/Parser/PhpDocParserTest.php +++ b/tests/PHPStan/Parser/PhpDocParserTest.php @@ -6838,6 +6838,19 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable ]), ]; + yield [ + '/**' . PHP_EOL . + ' * Real description' . PHP_EOL . + ' * @param int $a test' . PHP_EOL . + ' *' . PHP_EOL . + ' */', + new PhpDocNode([ + new PhpDocTextNode('Real description'), + new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', 'test')), + new PhpDocTextNode(''), + ]), + ]; + yield [ '/**' . PHP_EOL . ' * Real description' . PHP_EOL . From a2b24135c35852b348894320d47b3902a94bc494 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 23 Jul 2023 23:28:58 +0200 Subject: [PATCH 10/10] Fix missing newline --- src/Parser/PhpDocParser.php | 4 ++-- tests/PHPStan/Parser/PhpDocParserTest.php | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Parser/PhpDocParser.php b/src/Parser/PhpDocParser.php index d31b0672..7b56e3bd 100644 --- a/src/Parser/PhpDocParser.php +++ b/src/Parser/PhpDocParser.php @@ -243,7 +243,7 @@ private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode $savepoint = true; } elseif ($tmpText !== '') { $tokens->dropSavePoint(); - $savepoint = false; + $tokens->pushSavePoint(); } } @@ -329,7 +329,7 @@ private function parseOptionalDescriptionAfterDoctrineTag(TokenIterator $tokens) $savepoint = true; } elseif ($tmpText !== '') { $tokens->dropSavePoint(); - $savepoint = false; + $tokens->pushSavePoint(); } } diff --git a/tests/PHPStan/Parser/PhpDocParserTest.php b/tests/PHPStan/Parser/PhpDocParserTest.php index d911ec17..ebf8e4a5 100644 --- a/tests/PHPStan/Parser/PhpDocParserTest.php +++ b/tests/PHPStan/Parser/PhpDocParserTest.php @@ -6878,7 +6878,7 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocTextNode('Real description'), new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test')), new PhpDocTextNode(''), - //new PhpDocTextNode(''), + new PhpDocTextNode(''), ]), ]; @@ -6896,7 +6896,7 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test')), new PhpDocTextNode(''), new PhpDocTextNode(''), - //new PhpDocTextNode(''), + new PhpDocTextNode(''), ]), ]; @@ -6912,7 +6912,7 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocNode([ new PhpDocTextNode('Real description'), new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), - //new PhpDocTextNode(''), + new PhpDocTextNode(''), ]), ]; yield [ @@ -6929,7 +6929,7 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocTextNode('Real description'), new PhpDocTagNode('@param', new ParamTagValueNode(new IdentifierTypeNode('int'), false, '$a', PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), new PhpDocTextNode(''), - //new PhpDocTextNode(''), + new PhpDocTextNode(''), ]), ]; @@ -6947,7 +6947,7 @@ public function dataTextBetweenTagsBelongsToDescription(): iterable new PhpDocTextNode('Real description'), new PhpDocTagNode('@ORM\Column', new DoctrineTagValueNode(new DoctrineAnnotation('@ORM\Column', []), PHP_EOL . ' test' . PHP_EOL . PHP_EOL . 'test 2')), new PhpDocTextNode(''), - //new PhpDocTextNode(''), + new PhpDocTextNode(''), ]), ]; }