-
Notifications
You must be signed in to change notification settings - Fork 172
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SNOW-1003775: Fix parsing large string values with jackson databind >…
… 2.15 (#1613)
- Loading branch information
1 parent
e0f4fa2
commit e899319
Showing
8 changed files
with
183 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
src/main/java/net/snowflake/client/core/ObjectMapperFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,38 @@ | ||
| package net.snowflake.client.core; | ||
|
|
||
| import com.fasterxml.jackson.core.StreamReadConstraints; | ||
| import com.fasterxml.jackson.databind.MapperFeature; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import net.snowflake.client.log.SFLogger; | ||
| import net.snowflake.client.log.SFLoggerFactory; | ||
|
|
||
| /** | ||
| * Factor method used to create ObjectMapper instance. All object mapper in JDBC should be created | ||
| * by this method. | ||
| */ | ||
| public class ObjectMapperFactory { | ||
| @SnowflakeJdbcInternalApi | ||
| // Snowflake allows up to 16M string size and returns base64 encoded value that makes it up to 23M | ||
| public static final int DEFAULT_MAX_JSON_STRING_LEN = 23_000_000; | ||
|
|
||
| public static final String MAX_JSON_STRING_LENGTH_JVM = | ||
| "net.snowflake.jdbc.objectMapper.maxJsonStringLength"; | ||
|
|
||
| private static final SFLogger logger = SFLoggerFactory.getLogger(ObjectMapperFactory.class); | ||
|
|
||
| public static ObjectMapper getObjectMapper() { | ||
| ObjectMapper mapper = new ObjectMapper(); | ||
| mapper.configure(MapperFeature.OVERRIDE_PUBLIC_ACCESS_MODIFIERS, false); | ||
| mapper.configure(MapperFeature.CAN_OVERRIDE_ACCESS_MODIFIERS, false); | ||
|
|
||
| // override the maxStringLength value in ObjectMapper | ||
| int maxJsonStringLength = | ||
| SystemUtil.convertSystemPropertyToIntValue( | ||
| MAX_JSON_STRING_LENGTH_JVM, DEFAULT_MAX_JSON_STRING_LEN); | ||
| mapper | ||
| .getFactory() | ||
| .setStreamReadConstraints( | ||
| StreamReadConstraints.builder().maxStringLength(maxJsonStringLength).build()); | ||
| return mapper; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /* | ||
| * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
| */ | ||
|
|
||
| package net.snowflake.client.core; | ||
|
|
||
| import static net.snowflake.client.jdbc.SnowflakeUtil.systemGetProperty; | ||
|
|
||
| import net.snowflake.client.log.SFLogger; | ||
| import net.snowflake.client.log.SFLoggerFactory; | ||
|
|
||
| class SystemUtil { | ||
| private static final SFLogger logger = SFLoggerFactory.getLogger(SystemUtil.class); | ||
|
|
||
| /** | ||
| * Helper function to convert system properties to integers | ||
| * | ||
| * @param systemProperty name of the system property | ||
| * @param defaultValue default value used | ||
| * @return the value of the system property, else the default value | ||
| */ | ||
| static int convertSystemPropertyToIntValue(String systemProperty, int defaultValue) { | ||
| String systemPropertyValue = systemGetProperty(systemProperty); | ||
| int returnVal = defaultValue; | ||
| if (systemPropertyValue != null) { | ||
| try { | ||
| returnVal = Integer.parseInt(systemPropertyValue); | ||
| } catch (NumberFormatException ex) { | ||
| logger.info( | ||
| "Failed to parse the system parameter {} with value {}", | ||
| systemProperty, | ||
| systemPropertyValue); | ||
| } | ||
| } | ||
| return returnVal; | ||
| } | ||
| } |
92 changes: 92 additions & 0 deletions
92
src/test/java/net/snowflake/client/core/ObjectMapperTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| /* | ||
| * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
| */ | ||
|
|
||
| package net.snowflake.client.core; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.sql.SQLException; | ||
| import java.util.ArrayList; | ||
| import java.util.Base64; | ||
| import java.util.Collection; | ||
| import java.util.List; | ||
| import net.snowflake.client.jdbc.SnowflakeUtil; | ||
| import org.junit.After; | ||
| import org.junit.Assert; | ||
| import org.junit.Test; | ||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.Parameterized; | ||
|
|
||
| @RunWith(Parameterized.class) | ||
| public class ObjectMapperTest { | ||
| private static final int jacksonDefaultMaxStringLength = 20_000_000; | ||
|
|
||
| @Parameterized.Parameters(name = "lobSizeInMB={0}, maxJsonStringLength={1}") | ||
| public static Collection<Object[]> data() { | ||
| int[] lobSizeInMB = new int[] {16, 16, 32, 64, 128}; | ||
| // maxJsonStringLength to be set for the corresponding LOB size | ||
| int[] maxJsonStringLengths = | ||
| new int[] {jacksonDefaultMaxStringLength, 23_000_000, 45_000_000, 90_000_000, 180_000_000}; | ||
| List<Object[]> ret = new ArrayList<>(); | ||
| for (int i = 0; i < lobSizeInMB.length; i++) { | ||
| ret.add(new Object[] {lobSizeInMB[i], maxJsonStringLengths[i]}); | ||
| } | ||
| return ret; | ||
| } | ||
|
|
||
| private final int lobSizeInBytes; | ||
| private final int maxJsonStringLength; | ||
|
|
||
| @After | ||
| public void clearProperty() { | ||
| System.clearProperty(ObjectMapperFactory.MAX_JSON_STRING_LENGTH_JVM); | ||
| } | ||
|
|
||
| public ObjectMapperTest(int lobSizeInMB, int maxJsonStringLength) { | ||
| // convert LOB size from MB to bytes | ||
| this.lobSizeInBytes = lobSizeInMB * 1024 * 1024; | ||
| this.maxJsonStringLength = maxJsonStringLength; | ||
| System.setProperty( | ||
| ObjectMapperFactory.MAX_JSON_STRING_LENGTH_JVM, Integer.toString(maxJsonStringLength)); | ||
| } | ||
|
|
||
| @Test | ||
| public void testInvalidMaxJsonStringLength() throws SQLException { | ||
| System.setProperty(ObjectMapperFactory.MAX_JSON_STRING_LENGTH_JVM, "abc"); | ||
| // calling getObjectMapper() should log the exception but not throw | ||
| // default maxJsonStringLength value will be used | ||
| ObjectMapper mapper = ObjectMapperFactory.getObjectMapper(); | ||
| int stringLengthInMapper = mapper.getFactory().streamReadConstraints().getMaxStringLength(); | ||
| Assert.assertEquals(ObjectMapperFactory.DEFAULT_MAX_JSON_STRING_LEN, stringLengthInMapper); | ||
| } | ||
|
|
||
| @Test | ||
| public void testObjectMapperWithLargeJsonString() { | ||
| ObjectMapper mapper = ObjectMapperFactory.getObjectMapper(); | ||
| try { | ||
| JsonNode jsonNode = mapper.readTree(generateBase64EncodedJsonString(lobSizeInBytes)); | ||
| Assert.assertNotNull(jsonNode); | ||
| } catch (Exception e) { | ||
| // exception is expected when jackson's default maxStringLength value is used while retrieving | ||
| // 16M string data | ||
| assertEquals(jacksonDefaultMaxStringLength, maxJsonStringLength); | ||
| } | ||
| } | ||
|
|
||
| private String generateBase64EncodedJsonString(int numChar) { | ||
| StringBuilder jsonStr = new StringBuilder(); | ||
| String largeStr = SnowflakeUtil.randomAlphaNumeric(numChar); | ||
|
|
||
| // encode the string and put it into a JSON formatted string | ||
| jsonStr.append("[\"").append(encodeStringToBase64(largeStr)).append("\"]"); | ||
| return jsonStr.toString(); | ||
| } | ||
|
|
||
| private String encodeStringToBase64(String stringToBeEncoded) { | ||
| return Base64.getEncoder().encodeToString(stringToBeEncoded.getBytes(StandardCharsets.UTF_8)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters