Handle implicit and malformed code blocks by adding explicit markers
Summary:
A bunch of issues throw off the KDoc formatter when code blocks happen with three-backticks in the middle of the line, or also code blocks with no three-backticks.
This normalizes our handling of it, and adds ``` into code blocks when the lexer decided a token is a code block, but we did not see an explicit marker.
Reviewed By: cgrushko
Differential Revision: D20515472
fbshipit-source-id: a7dceeaccc249f7bceda488bb32991d1757e98ab
diff --git a/core/src/main/java/com/facebook/ktfmt/kdoc/KDocFormatter.kt b/core/src/main/java/com/facebook/ktfmt/kdoc/KDocFormatter.kt
index 3bb13a8..a4382cb 100644
--- a/core/src/main/java/com/facebook/ktfmt/kdoc/KDocFormatter.kt
+++ b/core/src/main/java/com/facebook/ktfmt/kdoc/KDocFormatter.kt
@@ -86,8 +86,6 @@
KDocTokens.TEXT -> {
if (tokenText.isBlank()) {
tokens.add(Token(WHITESPACE, " "))
- } else if (tokenText == "```") {
- tokens.add(Token(CODE_BLOCK_MARKER, tokenText))
} else {
val words = tokenText.trim().split(" +".toRegex())
var first = true
@@ -102,6 +100,8 @@
// END_KDOC properly. We want to recover in such cases
if (word == "*/") {
tokens.add(Token(END_KDOC, word))
+ } else if (word == "```") {
+ tokens.add(Token(CODE_BLOCK_MARKER, word))
} else {
tokens.add(Token(LITERAL, word))
tokens.add(Token(WHITESPACE, " "))
@@ -145,7 +145,7 @@
TABLE_CLOSE_TAG -> output.writeTableClose(token)
TAG -> output.writeTag(token)
CODE -> output.writeCodeLine(token)
- CODE_BLOCK_MARKER -> output.writeCodeBlockMarker(token)
+ CODE_BLOCK_MARKER -> output.writeExplicitCodeBlockMarker(token)
BLANK_LINE -> output.requestBlankLine()
WHITESPACE -> output.requestWhitespace()
LITERAL -> output.writeLiteral(token)
diff --git a/core/src/main/java/com/facebook/ktfmt/kdoc/KDocWriter.kt b/core/src/main/java/com/facebook/ktfmt/kdoc/KDocWriter.kt
index d991dbf..8f51519 100644
--- a/core/src/main/java/com/facebook/ktfmt/kdoc/KDocWriter.kt
+++ b/core/src/main/java/com/facebook/ktfmt/kdoc/KDocWriter.kt
@@ -26,6 +26,7 @@
import com.facebook.ktfmt.kdoc.KDocWriter.RequestedWhitespace.NEWLINE
import com.facebook.ktfmt.kdoc.KDocWriter.RequestedWhitespace.NONE
import com.facebook.ktfmt.kdoc.KDocWriter.RequestedWhitespace.WHITESPACE
+import com.facebook.ktfmt.kdoc.Token.Type.CODE_BLOCK_MARKER
import com.facebook.ktfmt.kdoc.Token.Type.HEADER_OPEN_TAG
import com.facebook.ktfmt.kdoc.Token.Type.LIST_ITEM_OPEN_TAG
import com.facebook.ktfmt.kdoc.Token.Type.PARAGRAPH_OPEN_TAG
@@ -86,12 +87,14 @@
}
fun writeEndJavadoc() {
+ requestCloseCodeBlockMarker()
output.append("\n")
appendSpaces(blockIndent + 1)
output.append("*/")
}
fun writeListItemOpen(token: Token) {
+ requestCloseCodeBlockMarker()
requestNewline()
if (continuingListItemOfInnermostList) {
@@ -153,25 +156,41 @@
}
fun writeCodeLine(token: Token) {
- if (!inCodeBlock) {
- requestNewline()
- }
+ requestOpenCodeBlockMarker()
requestNewline()
if (token.value.isNotEmpty()) {
writeToken(token)
}
}
- fun writeCodeBlockMarker(token: Token) {
- if (!inCodeBlock) {
- requestNewline()
+ /** Adds a code block marker if we are not in a code block currently */
+ private fun requestCloseCodeBlockMarker() {
+ if (inCodeBlock) {
+ this.requestedWhitespace = NEWLINE
+ writeExplicitCodeBlockMarker(Token(CODE_BLOCK_MARKER, "```"))
+ inCodeBlock = false
}
+ }
+
+ /** Adds a code block marker if we are in a code block currently */
+ private fun requestOpenCodeBlockMarker() {
+ if (!inCodeBlock) {
+ this.requestedWhitespace = NEWLINE
+ writeExplicitCodeBlockMarker(Token(CODE_BLOCK_MARKER, "```"))
+ inCodeBlock = true
+ }
+ }
+
+ fun writeExplicitCodeBlockMarker(token: Token) {
+ requestNewline()
writeToken(token)
requestNewline()
inCodeBlock = !inCodeBlock
}
fun writeLiteral(token: Token) {
+ requestCloseCodeBlockMarker()
+
writeToken(token)
}
@@ -301,7 +320,7 @@
}
private fun innerIndent(): Int {
- return continuingListItemCount.value() * 4 + continuingListCount.value() * 2
+ return 0
}
// If this is a hotspot, keep a String of many spaces around, and call append(string, start, end).
diff --git a/core/src/test/java/com/facebook/ktfmt/FormatterKtTest.kt b/core/src/test/java/com/facebook/ktfmt/FormatterKtTest.kt
index 9fecdcf..5181946 100644
--- a/core/src/test/java/com/facebook/ktfmt/FormatterKtTest.kt
+++ b/core/src/test/java/com/facebook/ktfmt/FormatterKtTest.kt
@@ -847,6 +847,81 @@
|""".trimMargin())
@Test
+ fun `formatting kdoc lists with line wraps breaks and merges correctly`() {
+ val code =
+ """
+ |/**
+ | * Here are some fruit I like:
+ | * - Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana
+ | * - Apple Apple Apple Apple
+ | * Apple Apple
+ | *
+ | * This is another paragraph
+ | */
+ |""".trimMargin()
+ val expected =
+ """
+ |/**
+ | * Here are some fruit I like:
+ | * - Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana
+ | * Banana Banana Banana Banana Banana
+ | * - Apple Apple Apple Apple Apple Apple
+ | *
+ | * This is another paragraph
+ | */
+ |""".trimMargin()
+ assertThatFormatting(code).isEqualTo(expected)
+ }
+
+ @Test
+ fun `too many spaces on list continuation mean it's a code block, so mark it accordingly`() {
+ val code =
+ """
+ |/**
+ | * Here are some fruit I like:
+ | * - Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana
+ | * Banana Banana Banana Banana Banana
+ | */
+ |""".trimMargin()
+ val expected =
+ """
+ |/**
+ | * Here are some fruit I like:
+ | * - Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana Banana
+ | * ```
+ | * Banana Banana Banana Banana Banana
+ | * ```
+ | */
+ |""".trimMargin()
+ assertThatFormatting(code).isEqualTo(expected)
+ }
+
+ @Test
+ fun `add explicit code markers around indented code`() {
+ val code =
+ """
+ |/**
+ | * This is a code example:
+ | *
+ | * this_is_code()
+ | *
+ | * This is not code again
+ | */
+ |""".trimMargin()
+ val expected =
+ """
+ |/**
+ | * This is a code example:
+ | * ```
+ | * this_is_code()
+ | * ```
+ | * This is not code again
+ | */
+ |""".trimMargin()
+ assertThatFormatting(code).isEqualTo(expected)
+ }
+
+ @Test
fun `formatting kdoc preserves lists of asterisks`() =
assertFormatted(
"""
@@ -2315,29 +2390,58 @@
|""".trimMargin())
@Test
- fun `deal with code blocks starting mid-line`() =
- assertFormatted(
- """
+ fun `handle stray code markers in lines and produce stable output`() {
+ val code =
+ """
|/**
- | * Look code: ``` aaa
+ | * Look! code: ``` aaa
| * fun f() = Unit
| * foo
| * ```
| */
|class MyClass {}
- |""".trimMargin())
+ |""".trimMargin()
+ assertFormatted(format(code))
+ }
@Test
- fun `deal with code blocks starting and ending mid-line`() =
- assertFormatted(
- """
+ fun `code block with triple backtick`() {
+ val code =
+ """
|/**
- | * Look code: ``` aaa
+ | * Look! code:
+ | * ```
+ | * aaa ``` wow
+ | * ```
+ | */
+ |class MyClass {}
+ |""".trimMargin()
+ val expected =
+ """
+ |/**
+ | * Look! code:
+ | * ```
+ | * aaa ``` wow
+ | * ```
+ | */
+ |class MyClass {}
+ |""".trimMargin()
+ assertThatFormatting(code).isEqualTo(expected)
+ }
+
+ @Test
+ fun `when code closer in mid of line, produce stable output`() {
+ val code =
+ """
+ |/**
+ | * Look! code: ``` aaa
| * fun f() = Unit
| * foo ``` wow
| */
|class MyClass {}
- |""".trimMargin())
+ |""".trimMargin()
+ assertFormatted(format(code))
+ }
@Test
fun `handle KDoc with link reference`() =
@@ -2419,12 +2523,13 @@
}
@Test
- fun `do not crash because of malformed KDocs`() =
- assertFormatted(
- """
+ fun `do not crash because of malformed KDocs and produce stable output`() {
+ val code = """
|/** Surprise ``` */
|class MyClass {}
- |""".trimMargin())
+ |""".trimMargin()
+ assertFormatted(format(code))
+ }
@Test
fun `Respect spacing of text after link`() =
