Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[INLONG-7952][Sort] Mask sensitive message of Flink SQL in the logs #7953

Merged
merged 2 commits into from
May 5, 2023
Merged

[INLONG-7952][Sort] Mask sensitive message of Flink SQL in the logs #7953

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
794857c
[INLONG-7952][Sort] Mask sensitive message of Flink SQL in the logs
May 1, 2023
86eca26
[INLONG-7952][Sort] Fix StringIndexOutOfBoundsException
e-mhui May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions inlong-common/src/main/java/org/apache/inlong/common/util/MaskDataUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public class MaskDataUtils {
"token", "secret_token", "secretToken",
"secret_id", "secretId",
"secret_key", "secretKey",
"public_key", "publicKey");
"public_key", "publicKey",
"gateway.url");
private static final List<String> SEPARATORS = Arrays.asList(":", "=", "\": \"", "\":\"");
private static final List<Character> STOP_CHARACTERS = Arrays.asList('\'', '"');
private static final List<Character> KNOWN_DELIMITERS =
Expand Down Expand Up @@ -117,8 +118,12 @@ public static int maskData(StringBuilder builder, char maskChar, int startPos, i
int idxSeparator;
for (String separator : SEPARATORS) {
idxSeparator = StringUtils.indexOf(builder, separator, keywordStart + keywordLength);
if (idxSeparator == keywordStart + keywordLength) {
charPos = maskStartPosition(keywordStart, keywordLength, separator, builder);
int delimiterPos = keywordStart + keywordLength;
while (delimiterPos < buffLength && isDelimiter(builder.charAt(delimiterPos))) {
delimiterPos++;
}
if (delimiterPos == idxSeparator) {
charPos = maskStartPosition(idxSeparator, separator, builder);

int endPos = detectEnd(builder, buffLength, charPos, keywordUsed, keywordLength, separator);

Expand Down Expand Up @@ -259,19 +264,28 @@ private static boolean keywordStartAtRightPosition(int keywordStart, int pos) {
/**
* the start position of sensitive data
*
* @param keywordStart the start position of keyword
* @param keywordLength the length of keyword
* @param idxSeparator the start position of separator character
* @param separator the separator character of keyword and sensitive data
* @param builder raw data
* @return the start position of sensitive data
*/
private static int maskStartPosition(int keywordStart, int keywordLength, String separator,
StringBuilder builder) {
int charPos = keywordStart + keywordLength + separator.length();
if (Character.isWhitespace(builder.charAt(charPos))) {
private static int maskStartPosition(int idxSeparator, String separator, StringBuilder builder) {
int charPos = idxSeparator + separator.length();
while (charPos < builder.length() && isDelimiter(builder.charAt(charPos))) {
charPos++;
}
return charPos;
}

/**
* mask sensitive message
* @param message raw message
* @return non-sensitive message
*/
public static String maskSensitiveMessage(String message) {
StringBuilder buffer = new StringBuilder(message);
mask(buffer);
return buffer.toString();
}

}
45 changes: 45 additions & 0 deletions inlong-common/src/test/java/org/apache/inlong/common/util/MaskDataUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,49 @@ public void testMaskDataUtils() throws Exception {
assertEquals(masked, buffer.toString());
}

/**
* Remove sensitive message in flink sql
*/
@Test
public void testMaskFlinkSql() {
String unmasked = "CREATE TABLE `table_1`(\n"
+ " PRIMARY KEY (`id`) NOT ENFORCED,\n"
+ " `id` INT,\n"
+ " `name` STRING,\n"
+ " `age` INT)\n"
+ " WITH (\n"
+ " 'inlong.metric.labels' = 'groupId=1&streamId=1&nodeId=1',\n"
+ " 'connector' = 'mysql-cdc-inlong',\n"
+ " 'hostname' = 'localhost',\n"
+ " 'database-name' = 'test',\n"
+ " 'port' = '3306',\n"
+ " 'server-id' = '10011',\n"
+ " 'scan.incremental.snapshot.enabled' = 'true',\n"
+ " 'username' = 'root',\n"
+ " 'password' = 'inlong',\n"
+ " 'table-name' = 'user'\n"
+ ")";

String masked = "CREATE TABLE `table_1`(\n"
+ " PRIMARY KEY (`id`) NOT ENFORCED,\n"
+ " `id` INT,\n"
+ " `name` STRING,\n"
+ " `age` INT)\n"
+ " WITH (\n"
+ " 'inlong.metric.labels' = 'groupId=1&streamId=1&nodeId=1',\n"
+ " 'connector' = 'mysql-cdc-inlong',\n"
+ " 'hostname' = 'localhost',\n"
+ " 'database-name' = 'test',\n"
+ " 'port' = '3306',\n"
+ " 'server-id' = '10011',\n"
+ " 'scan.incremental.snapshot.enabled' = 'true',\n"
+ " 'username' = 'root',\n"
+ " 'password' = '******',\n"
+ " 'table-name' = 'user'\n"
+ ")";
StringBuilder buffer = new StringBuilder(unmasked);
MaskDataUtils.mask(buffer);
assertEquals(masked, buffer.toString());
}

}
10 changes: 6 additions & 4 deletions inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/parser/impl/FlinkSqlParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.inlong.sort.parser.impl;

import static org.apache.inlong.common.util.MaskDataUtils.maskSensitiveMessage;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.table.api.TableEnvironment;
Expand Down Expand Up @@ -283,14 +285,14 @@ private void parseSingleNode(Node node, NodeRelation relation, Map<String, Node>
if (node instanceof ExtractNode) {
log.info("start parse node, node id:{}", node.getId());
String sql = genCreateSql(node);
log.info("node id:{}, create table sql:\n{}", node.getId(), sql);
log.info("node id:{}, create table sql:\n{}", node.getId(), maskSensitiveMessage(sql));
registerTableSql(node, sql);
hasParsedSet.add(node.getId());
} else {
Preconditions.checkNotNull(relation, "relation is null");
if (node instanceof LoadNode) {
String createSql = genCreateSql(node);
log.info("node id:{}, create table sql:\n{}", node.getId(), createSql);
log.info("node id:{}, create table sql:\n{}", node.getId(), maskSensitiveMessage(createSql));
registerTableSql(node, createSql);
hasParsedSet.add(node.getId());
} else if (node instanceof TransformNode) {
Expand All @@ -300,9 +302,9 @@ private void parseSingleNode(Node node, NodeRelation relation, Map<String, Node>
Preconditions.checkState(!transformNode.getFieldRelations().isEmpty(),
"field relations is empty");
String createSql = genCreateSql(node);
log.info("node id:{}, create table sql:\n{}", node.getId(), createSql);
log.info("node id:{}, create table sql:\n{}", node.getId(), maskSensitiveMessage(createSql));
String selectSql = genTransformSelectSql(transformNode, relation, nodeMap);
log.info("node id:{}, tansform sql:\n{}", node.getId(), selectSql);
log.info("node id:{}, transform sql:\n{}", node.getId(), maskSensitiveMessage(selectSql));
registerTableSql(node, createSql + " AS\n" + selectSql);
hasParsedSet.add(node.getId());
}
Expand Down
4 changes: 3 additions & 1 deletion ...ort/sort-core/src/main/java/org/apache/inlong/sort/parser/result/FlinkSqlParseResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.inlong.sort.parser.result;

import static org.apache.inlong.common.util.MaskDataUtils.maskSensitiveMessage;

import com.google.common.base.Preconditions;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -84,7 +86,7 @@ private TableResult executeLoadSqls(List<String> sqls) {

private void executeCreateTableSqls(List<String> sqls) {
for (String sql : sqls) {
log.info("execute createSql:\n{}", sql);
log.info("execute createSql:\n{}", maskSensitiveMessage(sql));
tableEnv.executeSql(sql);
}
}
Expand Down