Skip to content

Commit

Permalink
Fix | Send bulk data as unicode when destination column is nchar/nvar…
Browse files Browse the repository at this point in the history 
…char (#1193)
  • Loading branch information
@ulvii
ulvii authored Dec 6, 2019
1 parent 68a1b50 commit 5e813b7
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 53 deletions.
7 changes: 7 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
<dbcp2.version>2.6.0</dbcp2.version>
<slf4j.nop.version>1.7.26</slf4j.nop.version>
<gemini.mock.version>2.1.0.RELEASE</gemini.mock.version>
<h2.version>1.4.200</h2.version>

<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>${project.build.sourceEncoding}</project.reporting.outputEncoding>
Expand Down Expand Up @@ -194,6 +195,12 @@
<version>${gemini.mock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
142 changes: 94 additions & 48 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import java.util.logging.Level;

import javax.sql.RowSet;

import microsoft.sql.DateTimeOffset;


Expand Down Expand Up @@ -944,11 +943,13 @@ private void writeTypeInfo(TDSWriter tdsWriter, int srcJdbcType, int srcScale, i
tdsWriter.writeByte(TDSType.GUID.byteValue());
tdsWriter.writeByte((byte) 0x10);
} else {
// BIGCHARTYPE
tdsWriter.writeByte(TDSType.BIGCHAR.byteValue());

tdsWriter.writeShort((short) (srcPrecision));

if (unicodeConversionRequired(srcJdbcType, destSSType)) {
tdsWriter.writeByte(TDSType.NCHAR.byteValue());
tdsWriter.writeShort(isBaseType ? (short) (srcPrecision) : (short) (2 * srcPrecision));
} else {
tdsWriter.writeByte(TDSType.BIGCHAR.byteValue());
tdsWriter.writeShort((short) (srcPrecision));
}
collation.writeCollation(tdsWriter);
}
break;
Expand All @@ -961,12 +962,20 @@ private void writeTypeInfo(TDSWriter tdsWriter, int srcJdbcType, int srcScale, i

case java.sql.Types.LONGVARCHAR:
case java.sql.Types.VARCHAR: // 0xA7
// BIGVARCHARTYPE
tdsWriter.writeByte(TDSType.BIGVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
if (unicodeConversionRequired(srcJdbcType, destSSType)) {
tdsWriter.writeByte(TDSType.NVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
} else {
tdsWriter.writeShort(isBaseType ? (short) (srcPrecision) : (short) (2 * srcPrecision));
}
} else {
tdsWriter.writeShort((short) (srcPrecision));
tdsWriter.writeByte(TDSType.BIGVARCHAR.byteValue());
if (isStreaming) {
tdsWriter.writeShort((short) 0xFFFF);
} else {
tdsWriter.writeShort((short) (srcPrecision));
}
}
collation.writeCollation(tdsWriter);
break;
Expand Down Expand Up @@ -1183,6 +1192,7 @@ private String getDestTypeFromSrcType(int srcColIndx, int destColIndx,
int srcPrecision;

bulkJdbcType = srcColumnMetadata.get(srcColIndx).jdbcType;

// For char/varchar precision is the size.
bulkPrecision = srcPrecision = srcColumnMetadata.get(srcColIndx).precision;
int destPrecision = destColumnMetadata.get(destColIndx).precision;
Expand Down Expand Up @@ -1271,23 +1281,34 @@ private String getDestTypeFromSrcType(int srcColIndx, int destColIndx,
return "numeric(" + bulkPrecision + ", " + bulkScale + ")";

case microsoft.sql.Types.GUID:
case java.sql.Types.CHAR:
// For char the value has to be between 0 to 8000.
return "char(" + bulkPrecision + ")";

case java.sql.Types.CHAR:
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
return "nchar(" + bulkPrecision + ")";
} else {
return "char(" + bulkPrecision + ")";
}
case java.sql.Types.NCHAR:
return "NCHAR(" + bulkPrecision + ")";

case java.sql.Types.LONGVARCHAR:
case java.sql.Types.VARCHAR:
// Here the actual size of the varchar is used from the source table.
// Doesn't need to match with the exact size of data or with the destination column size.
if (isStreaming) {
return "varchar(max)";
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
if (isStreaming) {
return "nvarchar(max)";
} else {
return "nvarchar(" + bulkPrecision + ")";
}
} else {
return "varchar(" + bulkPrecision + ")";
if (isStreaming) {
return "varchar(max)";
} else {
return "varchar(" + bulkPrecision + ")";
}
}

// For INSERT BULK operations, XMLTYPE is to be sent as NVARCHAR(N) or NVARCHAR(MAX) data type.
// An error is produced if XMLTYPE is specified.
case java.sql.Types.LONGNVARCHAR:
Expand Down Expand Up @@ -2147,19 +2168,23 @@ else if (null != sourceCryptoMeta) {
} else {
reader = new StringReader(colValue.toString());
}

if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)
|| (SSType.VARBINARYMAX == destSSType) || (SSType.IMAGE == destSSType)) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true,
null);
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
// writeReader is unicode.
tdsWriter.writeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true);
} else {
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, false,
destCollation.getCharset());
} else {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, false,
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)
|| (SSType.VARBINARYMAX == destSSType) || (SSType.IMAGE == destSSType)) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH, true,
null);
} else {
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, destCollation.getCharset());
} else {
tdsWriter.writeNonUnicodeReader(reader, DataTypes.UNKNOWN_STREAM_LENGTH,
false, null);
}
}
}
reader.close();
Expand All @@ -2168,33 +2193,41 @@ else if (null != sourceCryptoMeta) {
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
}
} else // Non-PLP
{
} else {
if (null == colValue) {
writeNullToTdsWriter(tdsWriter, bulkJdbcType, isStreaming);
} else {
String colValueStr = colValue.toString();
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)) {
byte[] bytes = null;
try {
bytes = ParameterUtils.HexToBin(colValueStr);
} catch (SQLServerException e) {
throw new SQLServerException(
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
tdsWriter.writeShort((short) bytes.length);
tdsWriter.writeBytes(bytes);
if (unicodeConversionRequired(bulkJdbcType, destSSType)) {
int stringLength = colValue.toString().length();
byte[] typevarlen = new byte[2];
typevarlen[0] = (byte) (2 * stringLength & 0xFF);
typevarlen[1] = (byte) ((2 * stringLength >> 8) & 0xFF);
tdsWriter.writeBytes(typevarlen);
tdsWriter.writeString(colValue.toString());
} else {
tdsWriter.writeShort((short) (colValueStr.length()));
// converting string into destination collation using Charset
if ((SSType.BINARY == destSSType) || (SSType.VARBINARY == destSSType)) {
byte[] bytes = null;
try {
bytes = ParameterUtils.HexToBin(colValueStr);
} catch (SQLServerException e) {
throw new SQLServerException(
SQLServerException.getErrString("R_unableRetrieveSourceData"), e);
}
tdsWriter.writeShort((short) bytes.length);
tdsWriter.writeBytes(bytes);
} else {
tdsWriter.writeShort((short) (colValueStr.length()));
// converting string into destination collation using Charset

SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeBytes(colValueStr
.getBytes(destColumnMetadata.get(destColOrdinal).collation.getCharset()));
SQLCollation destCollation = destColumnMetadata.get(destColOrdinal).collation;
if (null != destCollation) {
tdsWriter.writeBytes(colValueStr.getBytes(
destColumnMetadata.get(destColOrdinal).collation.getCharset()));

} else {
tdsWriter.writeBytes(colValueStr.getBytes());
} else {
tdsWriter.writeBytes(colValueStr.getBytes());
}
}
}
}
Expand Down Expand Up @@ -3528,4 +3561,17 @@ protected void setStmtColumnEncriptionSetting(
protected void setDestinationTableMetadata(SQLServerResultSet rs) {
destinationTableMetadata = rs;
}

/**
* For the case when source database stores unicode data in CHAR/VARCHAR and destination column is NCHAR/NVARCHAR.
*
* @param jdbcType
* @param ssType
* @return whether conversion to unicode is required.
*/
private boolean unicodeConversionRequired(int jdbcType, SSType ssType) {
return ((java.sql.Types.CHAR == jdbcType || java.sql.Types.VARCHAR == jdbcType
|| java.sql.Types.LONGNVARCHAR == jdbcType)
&& (SSType.NCHAR == ssType || SSType.NVARCHAR == ssType || SSType.NVARCHARMAX == ssType));
}
}
8 changes: 4 additions & 4 deletions src/test/java/com/microsoft/sqlserver/jdbc/SQLServerDriverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ public class SQLServerDriverTest extends AbstractTest {
*
* @since 1.9
* @throws ClassNotFoundException
* @throws SQLException
*/
@Test
public void testDriverDM() throws ClassNotFoundException {
Stream<Driver> drivers = DriverManager.drivers();
Object[] driversArray = drivers.toArray();
assertEquals(driversArray[0].getClass(), Class.forName(Constants.MSSQL_JDBC_PACKAGE + ".SQLServerDriver"),
public void testDriverDM() throws SQLException, ClassNotFoundException {
Driver driver = DriverManager.getDriver(connectionString);
assertEquals(Class.forName(Constants.MSSQL_JDBC_PACKAGE + ".SQLServerDriver"), driver.getClass(),
TestResource.getResource("R_parrentLoggerNameWrong"));
}

Expand Down
51 changes: 50 additions & 1 deletion src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyColumnMappingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,18 @@
*/
package com.microsoft.sqlserver.jdbc.bulkCopy;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.MessageFormat;
import java.util.Random;

import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
Expand All @@ -20,8 +24,11 @@
import org.junit.runner.RunWith;

import com.microsoft.sqlserver.jdbc.ComparisonUtil;
import com.microsoft.sqlserver.jdbc.RandomUtil;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
import com.microsoft.sqlserver.jdbc.TestResource;
import com.microsoft.sqlserver.jdbc.TestUtils;
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
import com.microsoft.sqlserver.testframework.Constants;
import com.microsoft.sqlserver.testframework.DBConnection;
import com.microsoft.sqlserver.testframework.DBResultSet;
Expand Down Expand Up @@ -359,8 +366,18 @@ public void testInvalidCM() throws SQLException {
}
}

@Test
@DisplayName("BulkCopy:test unicode char/varchar to nchar/nvarchar")
public void testUnicodeCharToNchar() throws SQLException, ClassNotFoundException {
validateMapping("CHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("CHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(5)", "фщыab");
validateMapping("VARCHAR(5)", "NVARCHAR(max)", "фщыab");
}

/**
* validate if same values are in both source and destination table taking into account 1 extra column in
* Validate if same values are in both source and destination table taking into account 1 extra column in
* destination which should be a copy of first column of source.
*
* @param con
Expand Down Expand Up @@ -405,4 +422,36 @@ private void validateValuesRepetitiveCM(DBConnection con, DBTable sourceTable,
assertTrue(destinationTable.getTotalRows() == numRows);
}
}

private void validateMapping(String sourceType, String destType,
String data) throws SQLException, ClassNotFoundException {
Class.forName("org.h2.Driver");
Random rand = new Random();
String sourceTable = "sourceTable" + rand.nextInt(Integer.MAX_VALUE);
String destTable = TestUtils
.escapeSingleQuotes(AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("destTable")));
try (Connection sourceCon = DriverManager.getConnection("jdbc:h2:~/test", "sa", "");
Connection destCon = DriverManager.getConnection(connectionString);
Statement sourceStmt = sourceCon.createStatement(); Statement destStmt = destCon.createStatement();
SQLServerBulkCopy bulkCopy = new SQLServerBulkCopy(destCon)) {
try {
bulkCopy.setDestinationTableName(destTable);

sourceStmt.executeUpdate("CREATE TABLE " + sourceTable + " (col " + sourceType + ");");
sourceStmt.executeUpdate("INSERT INTO " + sourceTable + " VALUES('" + data + "');");

destStmt.executeUpdate("CREATE TABLE " + destTable + " (col NCHAR(5));");
ResultSet sourceRs = sourceStmt.executeQuery("SELECT * FROM " + sourceTable);
bulkCopy.writeToServer(sourceRs);

ResultSet destRs = destStmt.executeQuery("SELECT * FROM " + destTable);
destRs.next();
String receivedUnicodeData = destRs.getString(1);
assertEquals(data, receivedUnicodeData);
} finally {
sourceStmt.executeUpdate("DROP TABLE " + sourceTable);
TestUtils.dropTableIfExists(destTable, destStmt);
}
}
}
}

0 comments on commit 5e813b7

Please sign in to comment.