Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: permanent share and enable special SQL #1445

Merged
merged 8 commits into from
Jun 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions core/src/main/java/datart/core/common/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -131,22 +131,23 @@ public static Boolean isInitialized() {
if (initialized != null) {
return initialized;
}
updateInitialized();
return initialized;
}

public static void updateInitialized() {
UserMapperExt userMapper = getBean(UserMapperExt.class);
if (getCurrMode().equals(PLATFORM)) {
initialized = userMapper.selectUserCount()>0;
return initialized;
}
OrganizationMapperExt orgMapper = getBean(OrganizationMapperExt.class);
List<Organization> organizations = orgMapper.list();
int orgCount = CollectionUtils.size(organizations);
if (orgCount==0) {
initialized = false;
return initialized;
} else if (orgCount==1) {
List<User> users = orgMapper.listOrgMembers(organizations.get(0).getId());
initialized = users.stream().anyMatch(item -> getAdminId().equals(item.getId()));
}
return initialized;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ public Set<Column> readTableColumn(String database, String table) throws SQLExce
}

public String getQueryKey(QueryScript script, ExecuteParam executeParam) throws SqlParseException {
SqlScriptRender render = new SqlScriptRender(script, executeParam, getSqlDialect());
SqlScriptRender render = new SqlScriptRender(script, executeParam, getSqlDialect(), jdbcProperties.isEnableSpecialSql());
return "Q" + DigestUtils.md5Hex(render.render(true, true, true));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import datart.core.base.PageInfo;
import datart.core.data.provider.Dataframe;
import datart.data.provider.calcite.SqlNodeUtils;
import datart.data.provider.script.SqlStringUtils;
import org.apache.commons.lang3.StringUtils;

import java.sql.*;
Expand Down Expand Up @@ -43,7 +43,7 @@ public int executeCountSql(String sql) throws SQLException {

@Override
protected Dataframe execute(String selectSql, PageInfo pageInfo) throws SQLException {
selectSql = SqlNodeUtils.rebuildSqlWithFragment(selectSql);
selectSql = SqlStringUtils.rebuildSqlWithFragment(selectSql);
return super.execute(selectSql, pageInfo);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,10 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class SqlNodeUtils {

public static final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT";

public static SqlBasicCall createSqlBasicCall(SqlOperator sqlOperator, List<SqlNode> sqlNodes) {
if (sqlNodes == null) {
return null;
Expand Down Expand Up @@ -161,24 +157,4 @@ public static String toSql(SqlNode sqlNode, SqlDialect dialect, boolean quoteIde
.withIndentation(0)).getSql();
}

public static String rebuildSqlWithFragment(String sql) {
if (!sql.toLowerCase().startsWith("with")) {
Matcher matcher = Pattern.compile(REG_WITH_SQL_FRAGMENT).matcher(sql);
if (matcher.find()) {
String withFragment = matcher.group();
if (!StringUtils.isEmpty(withFragment)) {
if (withFragment.length() > 6) {
int lastSelectIndex = withFragment.length() - 6;
sql = sql.replace(withFragment, withFragment.substring(lastSelectIndex));
withFragment = withFragment.substring(0, lastSelectIndex);
}
String space = " ";
sql = withFragment + space + sql;
sql = sql.replaceAll(space + "{2,}", space);
}
}
}
return sql;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,17 @@
import org.apache.commons.lang3.StringUtils;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SqlStringUtils {

public static final String REG_SQL_SINGLE_LINE_COMMENT = "-{2,}.*([\r\n])";

public static final String REG_SQL_MULTI_LINE_COMMENT = "/\\*+[\\s\\S]*\\*+/";

public static final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT";

/**
* 替换脚本中的表达式类型变量
*
Expand Down Expand Up @@ -87,4 +91,29 @@ public static String cleanupSql(String sql) {
return sql.trim();
}

/**
* 处理sql with语句
* @param sql
* @return
*/
public static String rebuildSqlWithFragment(String sql) {
if (!sql.toLowerCase().startsWith("with")) {
Matcher matcher = Pattern.compile(REG_WITH_SQL_FRAGMENT).matcher(sql);
if (matcher.find()) {
String withFragment = matcher.group();
if (!StringUtils.isEmpty(withFragment)) {
if (withFragment.length() > 6) {
int lastSelectIndex = withFragment.length() - 6;
sql = sql.replace(withFragment, withFragment.substring(lastSelectIndex));
withFragment = withFragment.substring(0, lastSelectIndex);
}
String space = " ";
sql = withFragment + space + sql;
sql = sql.replaceAll(space + "{2,}", space);
}
}
}
return sql;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,19 @@
import datart.core.data.provider.QueryScript;
import datart.data.provider.DataProviderTestApplication;
import datart.data.provider.base.DataProviderException;
import datart.data.provider.calcite.SqlFragment;
import datart.data.provider.jdbc.SqlScriptRender;
import datart.data.provider.sql.entity.SqlTestEntity;
import datart.data.provider.sql.common.ParamFactory;
import datart.data.provider.sql.examples.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDialect;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;

Expand All @@ -37,6 +44,8 @@
@Slf4j
public class SqlScriptRenderTest {

private static final String T = "DATART_VTABLE";

@Test
public void testNormalSqlTest() throws SqlParseException {
validateTestSql(NormalSqlExamples.sqlList, false);
Expand Down Expand Up @@ -79,7 +88,8 @@ private void validateTestSql(List<SqlTestEntity> list, boolean enableSpecialSql)
SqlScriptRender render = new SqlScriptRender(queryScript, sqlTest.getExecuteParam(), sqlTest.getSqlDialect(), enableSpecialSql);
boolean withExecParam = sqlTest.getExecuteParam()!=null;
String parsedSql = render.render(withExecParam, false, false);
boolean result = parsedSql.equals(sqlTest.getDesireSql());
sqlTest.setDesireSql(covertDesiredSql(sqlTest.getDesireSql(), sqlTest.getSqlDialect()));
boolean result = parsedSql.equalsIgnoreCase(sqlTest.getDesireSql());
if (!result){
Exceptions.msg("sql validate failed! \n" + sqlTest +
" the parsed sql: "+parsedSql);
Expand All @@ -103,4 +113,11 @@ private void validateForbiddenSql(List<SqlTestEntity> list, boolean enableSpecia
}
}

private String covertDesiredSql(String sql, SqlDialect sqlDialect) {
SqlBasicCall sqlBasicCall = new SqlBasicCall(SqlStdOperatorTable.AS
, new SqlNode[]{new SqlFragment("( " + sql + " )"), new SqlIdentifier(T, SqlParserPos.ZERO.withQuoting(true))}
, SqlParserPos.ZERO);
return sqlBasicCall.toSqlString(sqlDialect).getSql().trim();
}

}
9 changes: 9 additions & 0 deletions security/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,15 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-oauth2-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
<dependency>
<groupId>org.bitbucket.b_c</groupId>
<artifactId>jose4j</artifactId>
<version>0.7.12</version>
</dependency>
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>dingtalk</artifactId>
Expand Down
Loading