Skip to content

Commit 2092c64

Browse files
BlaBlaHumanSpace Team
BlaBlaHuman
authored and
Space Team
committed
[KDoc lexer] Do not consider the last indent of fenced KDoc code blocks as a part of these blocks
When there is a fenced code block (surrounded by ```), the fence has to be placed on a new line. The spacing between the asterisk on this line and the closing fence was considered a part of this code block (`KDocTokens.CODE_BLOCK_TEXT`). That's senseless, as no meaningful content can be placed there, otherwise, the code block is parsed incorrectly (in other editors / according to MD guides). Now these spaces are viewed as `KDocTokens.TEXT`. ^KT-79783 ^KT-75215
1 parent 524c674 commit 2092c64

File tree

11 files changed

+78
-64
lines changed

11 files changed

+78
-64
lines changed

‎compiler/multiplatform-parsing/common/src/org/jetbrains/kotlin/kmp/lexer/KDoc.flex‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ IDENTIFIER = {PLAIN_IDENTIFIER} | `[^`\n]+`
6868
QUALIFIED_NAME = {IDENTIFIER} ([\.] {IDENTIFIER}?)* // Handle incorrect/incomplete qualifiers for correct resolving
6969
CODE_LINK=\[{QUALIFIED_NAME}\]
7070
CODE_FENCE_START=("```" | "~~~").*
71+
// `org.jetbrains.kotlin.kmp.lexer.KDocLexer` relies on these two types of ending fences.
72+
// If this set is changed, please, update `KDocLexer` accordingly
7173
CODE_FENCE_END=("```" | "~~~")
7274

7375
%%

‎compiler/multiplatform-parsing/common/src/org/jetbrains/kotlin/kmp/lexer/KDocLexer.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55

66
package org.jetbrains.kotlin.kmp.lexer
77

8+
import com.intellij.platform.syntax.SyntaxElementType
89
import com.intellij.platform.syntax.element.SyntaxTokenTypes
10+
import com.intellij.platform.syntax.lexer.Lexer
911
import com.intellij.platform.syntax.syntaxElementTypeSetOf
1012
import com.intellij.platform.syntax.util.lexer.FlexAdapter
1113
import com.intellij.platform.syntax.util.lexer.MergingLexerAdapter
@@ -19,4 +21,20 @@ class KDocLexer : MergingLexerAdapter(
1921
companion object {
2022
val KDOC_TOKENS = syntaxElementTypeSetOf(KDocTokens.TEXT, KDocTokens.CODE_BLOCK_TEXT, SyntaxTokenTypes.WHITE_SPACE)
2123
}
24+
25+
override fun merge(tokenType: SyntaxElementType, lexer: Lexer): SyntaxElementType {
26+
val nextTokenType = lexer.getTokenType()
27+
val nextTokenText = lexer.getTokenText()
28+
if (tokenType == KDocTokens.CODE_BLOCK_TEXT && nextTokenType == KDocTokens.TEXT && (nextTokenText == "```" || nextTokenText == "~~~")) {
29+
lexer.advance()
30+
return KDocTokens.TEXT // Don't treat the trailing line as a part of a code block
31+
} else if (tokenType == KDocTokens.CODE_BLOCK_TEXT || tokenType == KDocTokens.TEXT || tokenType == KtTokens.WHITE_SPACE) {
32+
while (tokenType == lexer.getTokenType()) {
33+
lexer.advance()
34+
}
35+
}
36+
37+
return tokenType
38+
}
39+
2240
}

‎compiler/psi/parser/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ IDENTIFIER = {PLAIN_IDENTIFIER} | `[^`\n]+`
7373
QUALIFIED_NAME = {IDENTIFIER} ([\.] {IDENTIFIER}?)* // Handle incorrect/incomplete qualifiers for correct resolving
7474
CODE_LINK=\[{QUALIFIED_NAME}\]
7575
CODE_FENCE_START=("```" | "~~~").*
76+
// `org.jetbrains.kotlin.kdoc.lexer.KDocLexer` relies on these two types of ending fences.
77+
// If this set is changed, please, update `KDocLexer` accordingly
7678
CODE_FENCE_END=("```" | "~~~")
7779

7880
%%

‎compiler/psi/parser/src/org/jetbrains/kotlin/kdoc/lexer/KDocLexer.java‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,37 @@
1616

1717
package org.jetbrains.kotlin.kdoc.lexer;
1818

19-
import com.intellij.lexer.FlexAdapter;
20-
import com.intellij.lexer.MergingLexerAdapter;
19+
import com.intellij.lexer.*;
2120
import com.intellij.psi.TokenType;
22-
import com.intellij.psi.tree.TokenSet;
21+
import com.intellij.psi.tree.IElementType;
2322

24-
import java.io.Reader;
25-
26-
public class KDocLexer extends MergingLexerAdapter {
27-
private static final TokenSet KDOC_TOKENS = TokenSet.create(KDocTokens.TEXT, KDocTokens.CODE_BLOCK_TEXT, TokenType.WHITE_SPACE);
23+
public class KDocLexer extends MergingLexerAdapterBase {
24+
private final MergeFunction mergeFunction = new KDocLexerMergeFunction();
2825

2926
public KDocLexer() {
30-
super(
31-
new FlexAdapter(
32-
new _KDocLexer((Reader) null)
33-
),
34-
KDOC_TOKENS
35-
);
27+
super(new FlexAdapter(new _KDocLexer(null)));
28+
}
29+
30+
@Override
31+
public MergeFunction getMergeFunction() {
32+
return mergeFunction;
33+
}
34+
35+
private static class KDocLexerMergeFunction implements MergeFunction {
36+
@Override
37+
public IElementType merge(IElementType type, Lexer originalLexer) {
38+
IElementType nextTokenType = originalLexer.getTokenType();
39+
String nextTokenText = originalLexer.getTokenText();
40+
if (type == KDocTokens.CODE_BLOCK_TEXT && nextTokenType == KDocTokens.TEXT && (nextTokenText.equals("```") || nextTokenText.equals("~~~"))) {
41+
originalLexer.advance();
42+
return KDocTokens.TEXT; // Don't treat the trailing line as a part of a code block
43+
} else if (type == KDocTokens.CODE_BLOCK_TEXT || type == KDocTokens.TEXT || type == TokenType.WHITE_SPACE) {
44+
while (type == originalLexer.getTokenType()) {
45+
originalLexer.advance();
46+
}
47+
}
48+
49+
return type;
50+
}
3651
}
3752
}

‎compiler/psi/psi-api/src/org/jetbrains/kotlin/psi/stubs/KotlinStubVersions.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ object KotlinStubVersions {
1313
* if you are not 100% sure it can be avoided.
1414
* Increasing this version will lead to reindexing of all kotlin source files on the first IDE startup with the new version.
1515
* */
16-
const val SOURCE_STUB_VERSION = 205
16+
const val SOURCE_STUB_VERSION = 206
1717

1818
/**
1919
* Binary stub version should be increased if stub format (org.jetbrains.kotlin.psi.stubs.impl) is changed

‎compiler/testData/lexer/kdoc/codeBlocks.txt‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ KDOC_LEADING_ASTERISK ('*')
4545
KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
4646
WHITE_SPACE ('\n ')
4747
KDOC_LEADING_ASTERISK ('*')
48-
KDOC_CODE_BLOCK_TEXT (' ')
49-
KDOC_TEXT ('```')
48+
KDOC_TEXT (' ```')
5049
WHITE_SPACE ('\n ')
5150
KDOC_LEADING_ASTERISK ('*')
5251
KDOC_TEXT (' ')
@@ -64,8 +63,7 @@ KDOC_LEADING_ASTERISK ('*')
6463
KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
6564
WHITE_SPACE ('\n ')
6665
KDOC_LEADING_ASTERISK ('*')
67-
KDOC_CODE_BLOCK_TEXT (' ')
68-
KDOC_TEXT ('~~~')
66+
KDOC_TEXT (' ~~~')
6967
WHITE_SPACE ('\n ')
7068
KDOC_LEADING_ASTERISK ('*')
7169
KDOC_TEXT (' ')

‎compiler/testData/lexer/kdoc/codeBlocksWithVerticalTabs.txt‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ KDOC_LEADING_ASTERISK ('*')
4545
KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
4646
WHITE_SPACE ('\n ')
4747
KDOC_LEADING_ASTERISK ('*')
48-
KDOC_CODE_BLOCK_TEXT (' ')
49-
KDOC_TEXT ('```')
48+
KDOC_TEXT (' ```')
5049
WHITE_SPACE ('\n ')
5150
KDOC_LEADING_ASTERISK ('*')
5251
KDOC_TEXT (' ')
@@ -64,8 +63,7 @@ KDOC_LEADING_ASTERISK ('*')
6463
KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
6564
WHITE_SPACE ('\n ')
6665
KDOC_LEADING_ASTERISK ('*')
67-
KDOC_CODE_BLOCK_TEXT (' ')
68-
KDOC_TEXT ('~~~')
66+
KDOC_TEXT (' ~~~')
6967
WHITE_SPACE ('\n ')
7068
KDOC_LEADING_ASTERISK ('*')
7169
KDOC_TEXT (' ')

‎compiler/testData/psi/kdoc/AtTags.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ KtFile: AtTags.kt
2222
PsiElement(KDOC_CODE_BLOCK_TEXT)(' @notATag and some description')
2323
PsiWhiteSpace('\n ')
2424
PsiElement(KDOC_LEADING_ASTERISK)('*')
25-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
26-
PsiElement(KDOC_TEXT)('```')
25+
PsiElement(KDOC_TEXT)(' ```')
2726
PsiWhiteSpace('\n ')
2827
PsiElement(KDOC_LEADING_ASTERISK)('*')
2928
PsiElement(KDOC_TEXT)(' @')

‎compiler/testData/psi/kdoc/CodeBlocks.tagContent.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ CONTENT:
3737

3838

3939

40-
```
40+
```
4141

4242
ORIGINAL KDOC:
4343
* ```
@@ -97,7 +97,7 @@ line
9797

9898
```
9999
line
100-
```
100+
```
101101

102102
```
103103
line
@@ -109,7 +109,7 @@ line
109109

110110
~~~
111111
line
112-
~~~
112+
~~~
113113

114114
~~~
115115
line
@@ -140,7 +140,7 @@ b
140140
a
141141
``` c
142142

143-
```
143+
```
144144
a
145145
b ```
146146

‎compiler/testData/psi/kdoc/CodeBlocks.txt‎

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ KtFile: CodeBlocks.kt
2020
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line 2')
2121
PsiWhiteSpace('\n ')
2222
PsiElement(KDOC_LEADING_ASTERISK)('*')
23-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
24-
PsiElement(KDOC_TEXT)('```')
23+
PsiElement(KDOC_TEXT)(' ```')
2524
PsiWhiteSpace('\n ')
2625
PsiElement(KDOC_LEADING_ASTERISK)('*')
2726
PsiWhiteSpace('\n ')
@@ -38,8 +37,7 @@ KtFile: CodeBlocks.kt
3837
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line 2')
3938
PsiWhiteSpace('\n ')
4039
PsiElement(KDOC_LEADING_ASTERISK)('*')
41-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
42-
PsiElement(KDOC_TEXT)('~~~')
40+
PsiElement(KDOC_TEXT)(' ~~~')
4341
PsiWhiteSpace('\n ')
4442
PsiElement(KDOC_LEADING_ASTERISK)('*')
4543
PsiWhiteSpace('\n ')
@@ -58,8 +56,7 @@ KtFile: CodeBlocks.kt
5856
PsiElement(KDOC_LEADING_ASTERISK)('*')
5957
PsiWhiteSpace('\n ')
6058
PsiElement(KDOC_LEADING_ASTERISK)('*')
61-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
62-
PsiElement(KDOC_TEXT)('```')
59+
PsiElement(KDOC_TEXT)(' ```')
6360
PsiWhiteSpace('\n ')
6461
PsiElement(KDOC_END)('*/')
6562
PsiWhiteSpace('\n\n')
@@ -83,8 +80,7 @@ KtFile: CodeBlocks.kt
8380
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line 4')
8481
PsiWhiteSpace('\n ')
8582
PsiElement(KDOC_LEADING_ASTERISK)('*')
86-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
87-
PsiElement(KDOC_TEXT)('```')
83+
PsiElement(KDOC_TEXT)(' ```')
8884
PsiWhiteSpace('\n ')
8985
PsiElement(KDOC_LEADING_ASTERISK)('*')
9086
PsiElement(KDOC_TEXT)(' ~~~')
@@ -102,8 +98,7 @@ KtFile: CodeBlocks.kt
10298
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line 4')
10399
PsiWhiteSpace('\n ')
104100
PsiElement(KDOC_LEADING_ASTERISK)('*')
105-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
106-
PsiElement(KDOC_TEXT)('~~~')
101+
PsiElement(KDOC_TEXT)(' ~~~')
107102
PsiWhiteSpace('\n ')
108103
PsiElement(KDOC_END)('*/')
109104
PsiWhiteSpace('\n\n')
@@ -118,8 +113,7 @@ KtFile: CodeBlocks.kt
118113
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
119114
PsiWhiteSpace('\n ')
120115
PsiElement(KDOC_LEADING_ASTERISK)('*')
121-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
122-
PsiElement(KDOC_TEXT)('```')
116+
PsiElement(KDOC_TEXT)(' ```')
123117
PsiWhiteSpace('\n ')
124118
PsiElement(KDOC_LEADING_ASTERISK)('*')
125119
PsiWhiteSpace('\n ')
@@ -130,8 +124,7 @@ KtFile: CodeBlocks.kt
130124
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
131125
PsiWhiteSpace('\n ')
132126
PsiElement(KDOC_LEADING_ASTERISK)('*')
133-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
134-
PsiElement(KDOC_TEXT)('```')
127+
PsiElement(KDOC_TEXT)(' ```')
135128
PsiWhiteSpace('\n ')
136129
PsiElement(KDOC_LEADING_ASTERISK)('*')
137130
PsiWhiteSpace('\n ')
@@ -142,8 +135,7 @@ KtFile: CodeBlocks.kt
142135
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
143136
PsiWhiteSpace('\n ')
144137
PsiElement(KDOC_LEADING_ASTERISK)('*')
145-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
146-
PsiElement(KDOC_TEXT)('```')
138+
PsiElement(KDOC_TEXT)(' ```')
147139
PsiWhiteSpace('\n ')
148140
PsiElement(KDOC_LEADING_ASTERISK)('*')
149141
PsiWhiteSpace('\n ')
@@ -154,8 +146,7 @@ KtFile: CodeBlocks.kt
154146
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
155147
PsiWhiteSpace('\n ')
156148
PsiElement(KDOC_LEADING_ASTERISK)('*')
157-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
158-
PsiElement(KDOC_TEXT)('~~~')
149+
PsiElement(KDOC_TEXT)(' ~~~')
159150
PsiWhiteSpace('\n ')
160151
PsiElement(KDOC_LEADING_ASTERISK)('*')
161152
PsiWhiteSpace('\n ')
@@ -166,8 +157,7 @@ KtFile: CodeBlocks.kt
166157
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
167158
PsiWhiteSpace('\n ')
168159
PsiElement(KDOC_LEADING_ASTERISK)('*')
169-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
170-
PsiElement(KDOC_TEXT)('~~~')
160+
PsiElement(KDOC_TEXT)(' ~~~')
171161
PsiWhiteSpace('\n ')
172162
PsiElement(KDOC_LEADING_ASTERISK)('*')
173163
PsiWhiteSpace('\n ')
@@ -178,8 +168,7 @@ KtFile: CodeBlocks.kt
178168
PsiElement(KDOC_CODE_BLOCK_TEXT)(' line')
179169
PsiWhiteSpace('\n ')
180170
PsiElement(KDOC_LEADING_ASTERISK)('*')
181-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
182-
PsiElement(KDOC_TEXT)('~~~')
171+
PsiElement(KDOC_TEXT)(' ~~~')
183172
PsiWhiteSpace('\n ')
184173
PsiElement(KDOC_END)('*/')
185174
PsiWhiteSpace('\n\n')
@@ -204,8 +193,7 @@ KtFile: CodeBlocks.kt
204193
PsiElement(KDOC_LEADING_ASTERISK)('*')
205194
PsiWhiteSpace('\n ')
206195
PsiElement(KDOC_LEADING_ASTERISK)('*')
207-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
208-
PsiElement(KDOC_TEXT)('```')
196+
PsiElement(KDOC_TEXT)(' ```')
209197
PsiWhiteSpace('\n ')
210198
PsiElement(KDOC_LEADING_ASTERISK)('*')
211199
PsiElement(KDOC_TEXT)(' a')
@@ -216,8 +204,7 @@ KtFile: CodeBlocks.kt
216204
PsiElement(KDOC_LEADING_ASTERISK)('*')
217205
PsiWhiteSpace('\n ')
218206
PsiElement(KDOC_LEADING_ASTERISK)('*')
219-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
220-
PsiElement(KDOC_TEXT)('```')
207+
PsiElement(KDOC_TEXT)(' ```')
221208
PsiWhiteSpace('\n ')
222209
PsiElement(KDOC_LEADING_ASTERISK)('*')
223210
PsiElement(KDOC_TEXT)(' a')
@@ -241,8 +228,7 @@ KtFile: CodeBlocks.kt
241228
PsiElement(KDOC_CODE_BLOCK_TEXT)(' Fenced code block')
242229
PsiWhiteSpace('\n ')
243230
PsiElement(KDOC_LEADING_ASTERISK)('*')
244-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
245-
PsiElement(KDOC_TEXT)('```')
231+
PsiElement(KDOC_TEXT)(' ```')
246232
PsiWhiteSpace('\n ')
247233
PsiElement(KDOC_LEADING_ASTERISK)('*')
248234
PsiWhiteSpace('\n ')
@@ -258,8 +244,7 @@ KtFile: CodeBlocks.kt
258244
PsiElement(KDOC_CODE_BLOCK_TEXT)(' Not well formed code block')
259245
PsiWhiteSpace('\n ')
260246
PsiElement(KDOC_LEADING_ASTERISK)('*')
261-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
262-
PsiElement(KDOC_TEXT)('```')
247+
PsiElement(KDOC_TEXT)(' ```')
263248
PsiWhiteSpace('\n ')
264249
PsiElement(KDOC_END)('*/')
265250
PsiWhiteSpace('\n\n\n')
@@ -274,8 +259,7 @@ KtFile: CodeBlocks.kt
274259
PsiElement(KDOC_CODE_BLOCK_TEXT)(' A code line with incorrect ending fence ```')
275260
PsiWhiteSpace('\n ')
276261
PsiElement(KDOC_LEADING_ASTERISK)('*')
277-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
278-
PsiElement(KDOC_TEXT)('```')
262+
PsiElement(KDOC_TEXT)(' ```')
279263
PsiWhiteSpace('\n ')
280264
PsiElement(KDOC_LEADING_ASTERISK)('*')
281265
PsiWhiteSpace('\n ')
@@ -295,8 +279,7 @@ KtFile: CodeBlocks.kt
295279
PsiElement(KDOC_CODE_BLOCK_TEXT)(' A code line with different fences')
296280
PsiWhiteSpace('\n ')
297281
PsiElement(KDOC_LEADING_ASTERISK)('*')
298-
PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
299-
PsiElement(KDOC_TEXT)('```')
282+
PsiElement(KDOC_TEXT)(' ```')
300283
PsiWhiteSpace('\n ')
301284
PsiElement(KDOC_LEADING_ASTERISK)('*')
302285
PsiWhiteSpace('\n ')

0 commit comments

Comments
 (0)