Skip to content

Commit

Permalink
refactor: Introduced hashing function
Browse files Browse the repository at this point in the history
  • Loading branch information
Julien Ruaux committed Feb 24, 2023
1 parent 9045c8e commit d690fe0
Show file tree
Hide file tree
Showing 9 changed files with 85 additions and 76 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Mode;

import com.redis.smartcache.jdbc.SmartConnection;
import com.redis.smartcache.core.util.CRC32HashingFunction;

public class SqlHashingBenchmark {

private static final String SQL = "SELECT orders.orderNumber, orders.orderDate, orders.requiredDate, orders.shippedDate, orders.status, orders.customerNumber, customers.customerName, orderdetails.productCode, products.productName, orderdetails.quantityOrdered FROM orders JOIN customers ON orders.customerNumber = customers.customerNumber JOIN orderdetails ON orders.orderNumber = orderdetails.orderNumber JOIN products ON orderdetails.productCode = products.productCode WHERE orders.orderNumber = ?";
private static final CRC32HashingFunction CRC32 = new CRC32HashingFunction();

@Benchmark
@BenchmarkMode(Mode.AverageTime)
public void crc32() {
SmartConnection.crc32(SQL);
CRC32.hash(SQL);
}

@Benchmark
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.redis.smartcache.core;

import io.trino.sql.tree.Statement;
import java.util.Set;

public class Query {

Expand All @@ -9,21 +9,25 @@ public class Query {

private final String id;
private final String sql;
private final Statement statement;
private final Set<String> tables;
private long ttl = TTL_NO_CACHING;

public Query(String id, String sql, Statement statement) {
public Query(String id, String sql, Set<String> tables) {
this.id = id;
this.sql = sql;
this.statement = statement;
this.tables = tables;
}

public boolean hasStatement() {
return statement != null;
public String getId() {
return id;
}

public Statement getStatement() {
return statement;
public String getSql() {
return sql;
}

public Set<String> getTables() {
return tables;
}

public long getTtl() {
Expand All @@ -38,12 +42,4 @@ public boolean isCaching() {
return ttl != TTL_NO_CACHING;
}

public String getId() {
return id;
}

public String getSql() {
return sql;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.regex.Pattern;
Expand All @@ -20,8 +18,6 @@

public class QueryRuleSession extends RuleSession<Query, Query> implements PropertyChangeListener {

private static final Collection<String> EMPTY_TABLE_NAMES = Collections.emptyList();

public QueryRuleSession() {
super();
}
Expand Down Expand Up @@ -57,27 +53,20 @@ public void fire(Query query) {
private static Rule<Query, Query> rule(RuleConfig rule) {
Consumer<Query> action = action(rule);
if (rule.getTables() != null) {
return CollectionRule.builder(QueryRuleSession::tableNames, action).exact(rule.getTables());
return CollectionRule.builder(Query::getTables, action).exact(rule.getTables());
}
if (rule.getTablesAll() != null) {
return CollectionRule.builder(QueryRuleSession::tableNames, action).all(rule.getTablesAll());
return CollectionRule.builder(Query::getTables, action).all(rule.getTablesAll());
}
if (rule.getTablesAny() != null) {
return CollectionRule.builder(QueryRuleSession::tableNames, action).any(rule.getTablesAny());
return CollectionRule.builder(Query::getTables, action).any(rule.getTablesAny());
}
if (rule.getRegex() != null) {
return new RegexRule<>(Pattern.compile(rule.getRegex()), Query::getSql, action);
}
return new PredicateRule<>(Predicates.alwaysTrue(), action);
}

private static Collection<String> tableNames(Query query) {
if (query.hasStatement()) {
return TableExtractor.tableNames(query.getStatement());
}
return EMPTY_TABLE_NAMES;
}

private static Consumer<Query> action(RuleConfig rule) {
return s -> s.setTtl(rule.getTtl());
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package com.redis.smartcache.core.util;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.zip.CRC32;

public class CRC32HashingFunction implements HashingFunction {

private static final Charset CHARSET = StandardCharsets.UTF_8;

@Override
public String hash(String string) {
CRC32 crc = new CRC32();
crc.update(string.getBytes(CHARSET));
return String.valueOf(crc.getValue());
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package com.redis.smartcache.core.util;

public interface HashingFunction {

String hash(String string);

}
Original file line number Diff line number Diff line change
@@ -1,31 +1,42 @@
package com.redis.smartcache.core;
package com.redis.smartcache.core.util;

import java.util.Collections;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import io.trino.sql.parser.ParsingException;
import io.trino.sql.parser.ParsingOptions;
import io.trino.sql.parser.SqlParser;
import io.trino.sql.tree.AstVisitor;
import io.trino.sql.tree.Node;
import io.trino.sql.tree.QualifiedName;
import io.trino.sql.tree.Statement;
import io.trino.sql.tree.Table;

public class TableExtractor {
public class SQLParser {

private TableExtractor() {
}
private static final ParsingOptions PARSING_OPTIONS = new ParsingOptions();
private static final Set<String> EMPTY_TABLE_NAMES = Collections.emptySet();
private final SqlParser parser = new SqlParser();

public static Stream<Table> tables(Statement statement) throws ParsingException {
return statement.accept(DepthFirstVisitor.by(new TableVisitor()), null);
public Set<String> extractTableNames(String sql) {
try {
Statement statement = parser.createStatement(sql, PARSING_OPTIONS);
Stream<Table> tables = tables(statement);
return tableNames(tables);
} catch (ParsingException e) {
// This statement cannot be parsed. Only rules like regex will trigger
return EMPTY_TABLE_NAMES;
}
}

public static Set<String> tableNames(Statement statement) {
return tableNames(tables(statement));
private Stream<Table> tables(Statement statement) throws ParsingException {
return statement.accept(DepthFirstVisitor.by(new TableVisitor()), null);
}

public static Set<String> tableNames(Stream<Table> tables) {
private Set<String> tableNames(Stream<Table> tables) {
return tables.map(Table::getName).map(QualifiedName::toString).collect(Collectors.toSet());
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.redis.smartcache.jdbc;

import java.nio.charset.StandardCharsets;
import java.sql.Array;
import java.sql.Blob;
import java.sql.CallableStatement;
Expand All @@ -18,8 +17,8 @@
import java.sql.Struct;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.zip.CRC32;

import javax.sql.rowset.CachedRowSet;
import javax.sql.rowset.RowSetFactory;
Expand All @@ -28,15 +27,15 @@
import com.redis.smartcache.core.Query;
import com.redis.smartcache.core.QueryRuleSession;
import com.redis.smartcache.core.ResultSetCache;
import com.redis.smartcache.core.util.CRC32HashingFunction;
import com.redis.smartcache.core.util.EvictingLinkedHashMap;
import com.redis.smartcache.core.util.HashingFunction;
import com.redis.smartcache.core.util.SQLParser;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.Timer;
import io.trino.sql.parser.ParsingException;
import io.trino.sql.parser.ParsingOptions;
import io.trino.sql.parser.SqlParser;

public class SmartConnection implements Connection {

Expand All @@ -49,11 +48,10 @@ public class SmartConnection implements Connection {
private static final String TAG_MISS = "miss";
private static final String TAG_HIT = "hit";
private static final String TAG_QUERY = "query";
private static final ParsingOptions PARSING_OPTIONS = new ParsingOptions();

private final SqlParser parser = new SqlParser();
private final Map<String, Query> queryCache;

private final HashingFunction hashFunction = new CRC32HashingFunction();
private final SQLParser sqlParser = new SQLParser();
private final Connection connection;
private final QueryRuleSession ruleSession;
private final RowSetFactory rowSetFactory;
Expand Down Expand Up @@ -190,7 +188,8 @@ public void clearWarnings() throws SQLException {

@Override
public Statement createStatement(int rsType, int rsConcurrency) throws SQLException {
return new SmartStatement(this, connection.createStatement(rsType, rsConcurrency));
Statement statement = connection.createStatement(rsType, rsConcurrency);
return new SmartStatement(this, statement);
}

@Override
Expand Down Expand Up @@ -363,12 +362,17 @@ public int getNetworkTimeout() throws SQLException {
return connection.getNetworkTimeout();
}

public String hash(String string) {
return hashFunction.hash(string);
}

public Query getQuery(String sql) {
String id = crc32(sql);
String id = hash(sql);
if (queryCache.containsKey(id)) {
return queryCache.get(id);
}
Query query = new Query(id, sql, parse(sql));
Set<String> tables = sqlParser.extractTableNames(sql);
Query query = new Query(id, sql, tables);
createTimer(METER_QUERY, query);
createTimer(METER_BACKEND, query);
createTimer(METER_CACHE_GET, query);
Expand Down Expand Up @@ -422,22 +426,6 @@ private Counter createCounter(String name, Query query, String... tags) {
return Counter.builder(name).tags(tags(query)).tags(tags).register(meterRegistry);
}

private io.trino.sql.tree.Statement parse(String sql) {
try {
return parser.createStatement(sql, PARSING_OPTIONS);
} catch (ParsingException e) {
// This statement cannot be parsed. Only rules like regex can trigger
return null;
}

}

public static String crc32(String string) {
CRC32 crc = new CRC32();
crc.update(string.getBytes(StandardCharsets.UTF_8));
return String.valueOf(crc.getValue());
}

public Query fireRules(String sql) {
Query query = getQuery(sql);
ruleSession.fire(query);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public SmartPreparedStatement(SmartConnection connection, PreparedStatement stat

@Override
protected String key(Query query) {
return query.getId() + connection.getConfig().getKeySeparator() + SmartConnection.crc32(paramString());
return query.getId() + connection.getConfig().getKeySeparator() + connection.hash(paramString());
}

protected String paramString() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
import com.redis.smartcache.core.Config.RulesetConfig.RuleConfig;
import com.redis.smartcache.core.Query;
import com.redis.smartcache.core.QueryRuleSession;
import com.redis.smartcache.jdbc.SmartConnection;

import io.trino.sql.parser.ParsingOptions;
import io.trino.sql.parser.SqlParser;
import com.redis.smartcache.core.util.CRC32HashingFunction;
import com.redis.smartcache.core.util.HashingFunction;
import com.redis.smartcache.core.util.SQLParser;

class RulesTests {

Expand All @@ -21,8 +20,8 @@ class RulesTests {
private static final String ORDERS = "orders";
private static final String ORDERS_O = ORDERS + " o";

private static final SqlParser PARSER = new SqlParser();
private static final ParsingOptions PARSING_OPTIONS = new ParsingOptions();
private static final SQLParser PARSER = new SQLParser();
private static final HashingFunction HASHING_FUNCTION = new CRC32HashingFunction();

@Test
void testTables() {
Expand Down Expand Up @@ -77,7 +76,7 @@ void testRegex() {
}

private Query query(String sql) {
return new Query(SmartConnection.crc32(sql), sql, PARSER.createStatement(sql, PARSING_OPTIONS));
return new Query(HASHING_FUNCTION.hash(sql), sql, PARSER.extractTableNames(sql));
}

}

0 comments on commit d690fe0

Please sign in to comment.