Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix | Send bulk data as unicode when destination column is nchar/nvarchar #1193

Merged
merged 11 commits into from
Dec 6, 2019
Merged

Fix | Send bulk data as unicode when destination column is nchar/nvarchar #1193

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
82c8f47
Fix | Fix the scenario when bulk copy is performed from unicode char/…
ulvii Nov 22, 2019
1278487
Fix | handle more scenarios
ulvii Nov 26, 2019
07778aa
Fix | Exclude streaming
ulvii Nov 26, 2019
044fcfc
Fix | cleanup
ulvii Nov 26, 2019
aae551d
Fix | Cleanup comments
ulvii Nov 26, 2019
4daaf0c
Fix | Add tests
ulvii Nov 27, 2019
e1e093c
Fix | Remove unused imports
ulvii Nov 27, 2019
29e2efe
Fix | Fix DM test
ulvii Nov 27, 2019
5c4910e
Fix | Handle NVARCHARMAX
ulvii Dec 6, 2019
461f155
Fix | Add test for nvracharmax
ulvii Dec 6, 2019
4788fd6
Fix | Fix assert in testDriverDM
ulvii Dec 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
<dbcp2.version>2.6.0</dbcp2.version>
<slf4j.nop.version>1.7.26</slf4j.nop.version>
<gemini.mock.version>2.1.0.RELEASE</gemini.mock.version>
<h2.version>1.4.200</h2.version>

<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>${project.build.sourceEncoding}</project.reporting.outputEncoding>
Expand Down Expand Up @@ -194,6 +195,12 @@
<version>${gemini.mock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
142 changes: 94 additions & 48 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import java.util.logging.Level;

import javax.sql.RowSet;

import microsoft.sql.DateTimeOffset;


Expand Down Expand Up @@ -944,11 +943,13 @@ private void writeTypeInfo(TDSWriter tdsWriter, int srcJdbcType, int srcScale, i
tdsWriter.writeByte(TDSType.GUID.byteValue());
tdsWriter.writeByte((byte) 0x10);
} else {
// BIGCHARTYPE
tdsWriter.writeByte(TDSType.BIGCHAR.byteValue());

tdsWriter.writeShort((short) (srcPrecision));

if (unicodeConversionRequired(srcJdbcType, destSSType)) {
tdsWriter.writeByte(TDSType.NCHAR.byteValue());
tdsWriter.writeShort(isBaseType ? (short) (srcPrecision) : (short) (2 * srcPrecision));
} else {
tdsWriter.writeByte(TDSType.BIGCHAR.byteValue());
tdsWriter.writeShort((short) (srcPrecision));
}
collation.writeCollation(tdsWriter);
}
break;
Expand All @@ -961,12 +962,20 @@ private void writeTypeInfo(TDSWriter tdsWriter, int srcJdbcType, int srcScale, i

case java.sql.Types.LONGVARCHAR:
case java.sql.Types.VARCHAR: // 0xA7
// BIGVARCHARTYPE
tdsWriter.writeByte(TDSType.BIGVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
if (unicodeConversionRequired(srcJdbcType, destSSType)) {
tdsWriter.writeByte(TDSType.NVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
} else {
tdsWriter.writeShort(isBaseType ? (short) (srcPrecision) : (short) (2 * srcPrecision));
}
} else {
tdsWriter.writeShort((short) (srcPrecision));
tdsWriter.writeByte(TDSType.BIGVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
} else {
tdsWriter.writeShort((short) (srcPrecision));
}
}
collation.writeCollation(tdsWriter);
break;
Expand Down Expand Up @@ -1183,6 +1192,7 @@ private String getDestTypeFromSrcType(int srcColIndx, int destColIndx,
int srcPrecision;

bulkJdbcType = srcColumnMetadata.get(srcColIndx).jdbcType;

// For char/varchar precision is the size.
bulkPrecision = srcPrecision = srcColumnMetadata.get(srcColIndx).precision;
int destPrecision = destColumnMetadata.get(destColIndx).precision;
Expand Down Expand Up @@ -1271,23 +1281,34 @@ private String getDestTypeFromSrcType(int srcColIndx, int destColIndx,
return "numeric(" + bulkPrecision + ", " + bulkScale + ")";

case microsoft.sql.Types.GUID:
case java.sql.Types.CHAR:
// For char the value has to be between 0 to 8000.
return "char(" + bulkPrecision + ")";

case java.sql.Types.CHAR:
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
return "nchar(" + bulkPrecision + ")";
} else {
return "char(" + bulkPrecision + ")";
}
case java.sql.Types.NCHAR:
return "NCHAR(" + bulkPrecision + ")";

case java.sql.Types.LONGVARCHAR:
case java.sql.Types.VARCHAR:
// Here the actual size of the varchar is used from the source table.
// Doesn't need to match with the exact size of data or with the destination column size.
if (isStreaming) {
return "varchar(max)";
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
if (isStreaming) {
return "nvarchar(max)";
} else {
return "nvarchar(" + bulkPrecision + ")";
}
} else {
return "varchar(" + bulkPrecision + ")";
if (isStreaming) {
return "varchar(max)";
} else {
return "varchar(" + bulkPrecision + ")";
}
}

// For INSERT BULK operations, XMLTYPE is to be sent as NVARCHAR(N) or NVARCHAR(MAX) data type.
// An error is produced if XMLTYPE is specified.
case java.sql.Types.LONGNVARCHAR:
Expand Down Expand Up @@ -2147,19 +2168,23 @@ else if (null != sourceCryptoMeta) {
} else {
reader = new StringReader(colValue.toString());
}

if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)
|| (SSType.VARBINARYMAX == destSSType) || (SSType.IMAGE == destSSType)) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true,
null);
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
// writeReader is unicode.
tdsWriter.writeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true);
} else {
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, false,
destCollation.getCharset());
} else {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, false,
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)
|| (SSType.VARBINARYMAX == destSSType) || (SSType.IMAGE == destSSType)) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true,
null);
} else {
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, destCollation.getCharset());
} else {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, null);
}
}
}
reader.close();
Expand All @@ -2168,33 +2193,41 @@ else if (null != sourceCryptoMeta) {
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
}
} else // Non-PLP
{
} else {
if (null == colValue) {
writeNullToTdsWriter(tdsWriter, bulkJdbcType, isStreaming);
} else {
String colValueStr = colValue.toString();
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)) {
byte[] bytes = null;
try {
bytes = ParameterUtils.HexToBin(colValueStr);
} catch (SQLServerException e) {
throw new SQLServerException(
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
tdsWriter.writeShort((short) bytes.length);
tdsWriter.writeBytes(bytes);
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
int stringLength = colValue.toString().length();
byte[] typevarlen = new byte[2];
typevarlen[0] = (byte) (2 * stringLength & 0xFF);
typevarlen[1] = (byte) ((2 * stringLength >> 8) & 0xFF);
tdsWriter.writeBytes(typevarlen);
tdsWriter.writeString(colValue.toString());
} else {
tdsWriter.writeShort((short) (colValueStr.length()));
// converting string into destination collation using Charset
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)) {
byte[] bytes = null;
try {
bytes = ParameterUtils.HexToBin(colValueStr);
} catch (SQLServerException e) {
throw new SQLServerException(
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
tdsWriter.writeShort((short) bytes.length);
tdsWriter.writeBytes(bytes);
} else {
tdsWriter.writeShort((short) (colValueStr.length()));
// converting string into destination collation using Charset

SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeBytes(colValueStr
.getBytes(destColumnMetadata.get(destColOrdinal).collation.getCharset()));
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeBytes(colValueStr.getBytes(
destColumnMetadata.get(destColOrdinal).collation.getCharset()));

} else {
tdsWriter.writeBytes(colValueStr.getBytes());
} else {
tdsWriter.writeBytes(colValueStr.getBytes());
}
}
}
}
Expand Down Expand Up @@ -3528,4 +3561,17 @@ protected void setStmtColumnEncriptionSetting(
protected void setDestinationTableMetadata(SQLServerResultSet rs) {
destinationTableMetadata = rs;
}

/**
* For the case when source database stores unicode data in CHAR/VARCHAR and destination column is NCHAR/NVARCHAR.
*
* @param jdbcType
* @param ssType
* @return whether conversion to unicode is required.
*/
private boolean unicodeConversionRequired(int jdbcType, SSType ssType) {
return ((java.sql.Types.CHAR == jdbcType || java.sql.Types.VARCHAR == jdbcType
|| java.sql.Types.LONGNVARCHAR == jdbcType)
&& (SSType.NCHAR == ssType || SSType.NVARCHAR == ssType || SSType.NVARCHARMAX == ssType));
}
}
8 changes: 4 additions & 4 deletions src/test/java/com/microsoft/sqlserver/jdbc/SQLServerDriverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ public class SQLServerDriverTest extends AbstractTest {
*
* @since 1.9
* @throws ClassNotFoundException
* @throws SQLException
*/
@Test
public void testDriverDM() throws ClassNotFoundException {
Stream<Driver> drivers = DriverManager.drivers();
Object[] driversArray = drivers.toArray();
assertEquals(driversArray[0].getClass(), Class.forName(Constants.MSSQL_JDBC_PACKAGE + ".SQLServerDriver"),
public void testDriverDM() throws SQLException, ClassNotFoundException {
Driver driver = DriverManager.getDriver(connectionString);
assertEquals(driver.getClass(), Class.forName(Constants.MSSQL_JDBC_PACKAGE + ".SQLServerDriver"),
TestResource.getResource("R_parrentLoggerNameWrong"));
}

Expand Down
51 changes: 50 additions & 1 deletion src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyColumnMappingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,18 @@
*/
package com.microsoft.sqlserver.jdbc.bulkCopy;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.MessageFormat;
import java.util.Random;

import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
Expand All @@ -20,8 +24,11 @@
import org.junit.runner.RunWith;

import com.microsoft.sqlserver.jdbc.ComparisonUtil;
import com.microsoft.sqlserver.jdbc.RandomUtil;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
import com.microsoft.sqlserver.jdbc.TestResource;
import com.microsoft.sqlserver.jdbc.TestUtils;
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
import com.microsoft.sqlserver.testframework.Constants;
import com.microsoft.sqlserver.testframework.DBConnection;
import com.microsoft.sqlserver.testframework.DBResultSet;
Expand Down Expand Up @@ -359,8 +366,18 @@ public void testInvalidCM() throws SQLException {
}
}

@Test
@DisplayName("BulkCopy:test unicode char/varchar to nchar/nvarchar")
public void testUnicodeCharToNchar() throws SQLException, ClassNotFoundException {
validateMapping("CHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("CHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(max)", "фщыab");
}

/**
* validate if same values are in both source and destination table taking into account 1 extra column in
* Validate if same values are in both source and destination table taking into account 1 extra column in
* destination which should be a copy of first column of source.
*
* @param con
Expand Down Expand Up @@ -405,4 +422,36 @@ private void validateValuesRepetitiveCM(DBConnection con, DBTable sourceTable,
assertTrue(destinationTable.getTotalRows() == numRows);
}
}

private void validateMapping(String sourceType, String destType,
String data) throws SQLException, ClassNotFoundException {
Class.forName("org.h2.Driver");
Random rand = new Random();
String sourceTable = "sourceTable" + rand.nextInt(Integer.MAX_VALUE);
String destTable = TestUtils
.escapeSingleQuotes(AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("destTable")));
try (Connection sourceCon = DriverManager.getConnection("jdbc:h2:~/test", "sa", "");
Connection destCon = DriverManager.getConnection(connectionString);
Statement sourceStmt = sourceCon.createStatement(); Statement destStmt = destCon.createStatement();
SQLServerBulkCopy bulkCopy = new SQLServerBulkCopy(destCon)) {
try {
bulkCopy.setDestinationTableName(destTable);

sourceStmt.executeUpdate("CREATE TABLE " + sourceTable + " (col " + sourceType + ");");
sourceStmt.executeUpdate("INSERT INTO " + sourceTable + " VALUES('" + data + "');");

destStmt.executeUpdate("CREATE TABLE " + destTable + " (col NCHAR(5));");
ResultSet sourceRs = sourceStmt.executeQuery("SELECT * FROM " + sourceTable);
bulkCopy.writeToServer(sourceRs);

ResultSet destRs = destStmt.executeQuery("SELECT * FROM " + destTable);
destRs.next();
String receivedUnicodeData = destRs.getString(1);
assertEquals(data, receivedUnicodeData);
} finally {
sourceStmt.executeUpdate("DROP TABLE " + sourceTable);
TestUtils.dropTableIfExists(destTable, destStmt);
}
}
}
}