Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-48148][CORE] JSON objects should not be modified when read as STRING #46408

Closed
wants to merge 9 commits into from
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -280,13 +280,32 @@ class JacksonParser(
case VALUE_STRING =>
UTF8String.fromString(parser.getText)

case _ =>
case other =>
// Note that it always tries to convert the data as string without the case of failure.
val writer = new ByteArrayOutputStream()
Utils.tryWithResource(factory.createGenerator(writer, JsonEncoding.UTF8)) {
generator => generator.copyCurrentStructure(parser)
val startLocation = parser.getTokenLocation
startLocation.contentReference().getRawContent match {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there an existing API to get the remaining content as string? Also, would it work with multi-line JSON?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was not able to find such an existing API -- there is JacksonParser.getText but that appears to simply get the current value if it's a string value.

Copy link
Author

@eric-maynard eric-maynard May 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wrt. multiline JSON, I have added a test to cover this. It seems that the content reference is not a byte array when using multiline mode.

case byteArray: Array[Byte] if exactStringParsing =>
other match {
case START_OBJECT =>
parser.skipChildren()
case START_ARRAY =>
parser.skipChildren()
case _ =>
// Do nothing in this case; we've already read the token
}
val endLocation = parser.currentLocation.getByteOffset

UTF8String.fromBytes(
byteArray,
startLocation.getByteOffset.toInt,
endLocation.toInt - (startLocation.getByteOffset.toInt))
case _ =>
val writer = new ByteArrayOutputStream()
Utils.tryWithResource(factory.createGenerator(writer, JsonEncoding.UTF8)) {
generator => generator.copyCurrentStructure(parser)
}
UTF8String.fromBytes(writer.toByteArray)
}
UTF8String.fromBytes(writer.toByteArray)
}

case TimestampType =>
Expand Down Expand Up @@ -429,6 +448,8 @@ class JacksonParser(

private val allowEmptyString = SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_EMPTY_STRING_IN_JSON)

private val exactStringParsing = SQLConf.get.getConf(SQLConf.JSON_EXACT_STRING_PARSING)

/**
* This function throws an exception for failed conversion. For empty string on data types
* except for string and binary types, this also throws an exception.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4249,6 +4249,15 @@ object SQLConf {
.booleanConf
.createWithDefault(true)

val JSON_EXACT_STRING_PARSING =
buildConf("spark.sql.json.enableExactStringParsing")
.internal()
.doc("When set to true, string columns extracted from JSON objects will be extracted " +
"exactly as they appear in the input string, with no changes")
.version("4.0.0")
.booleanConf
.createWithDefault(true)

val LEGACY_CSV_ENABLE_DATE_TIME_PARSING_FALLBACK =
buildConf("spark.sql.legacy.csv.enableDateTimeParsingFallback")
.internal()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3865,6 +3865,65 @@ abstract class JsonSuite
}
}
}

test("SPARK-48148: values are unchanged when read as string") {
withTempPath { path =>
def extractData(
jsonString: String,
expectedInexactData: Seq[String],
expectedExactData: Seq[String],
multiLine: Boolean = false): Unit = {
Seq(jsonString).toDF()
.repartition(1)
.write
.mode("overwrite")
.text(path.getAbsolutePath)

withClue("Exact string parsing") {
withSQLConf(SQLConf.JSON_EXACT_STRING_PARSING.key -> "true") {
val df = spark.read
.schema("data STRING")
.option("multiLine", multiLine.toString)
.json(path.getAbsolutePath)
checkAnswer(df, expectedExactData.map(d => Row(d)))
}
}

withClue("Inexact string parsing") {
withSQLConf(SQLConf.JSON_EXACT_STRING_PARSING.key -> "false") {
val df = spark.read
.schema("data STRING")
.option("multiLine", multiLine.toString)
.json(path.getAbsolutePath)
checkAnswer(df, expectedInexactData.map(d => Row(d)))
}
}
}
extractData(
"""{"data": {"white": "space"}}""",
expectedInexactData = Seq("""{"white":"space"}"""),
expectedExactData = Seq("""{"white": "space"}""")
)
extractData(
"""{"data": ["white", "space"]}""",
expectedInexactData = Seq("""["white","space"]"""),
expectedExactData = Seq("""["white", "space"]""")
)
val granularFloat = "-999.99999999999999999999999999999999995"
extractData(
s"""{"data": {"v": ${granularFloat}}}""",
expectedInexactData = Seq("""{"v":-1000.0}"""),
expectedExactData = Seq(s"""{"v": ${granularFloat}}""")
)
// In multiLine, we fall back to the inexact method:
extractData(
"""{"data": {"white":\n"space"}}""",
expectedInexactData = Seq("""{"white":"space"}"""),
expectedExactData = Seq("""{"white":"space"}"""),
multiLine = true
)
}
}
}

class JsonV1Suite extends JsonSuite {
Expand Down